---
title: "Joining attribute data with geofi data"
author: "Markus Kainu"
date: "`r Sys.Date()`"
output: rmarkdown::html_vignette
vignette: >
  %\VignetteIndexEntry{Joining attribute data with geofi data}
  %\VignetteEncoding{UTF-8}
  %\VignetteEngine{knitr::rmarkdown}
editor_options: 
  chunk_output_type: console
---

```{r setup, include = FALSE}
knitr::opts_chunk$set(
  
  collapse = TRUE,
  comment = "#>",
  message = FALSE, 
  warning = FALSE,
  fig.height = 7, 
  fig.width = 7,
  dpi = 75
)
```

This vignettes provides few examples on how to join attribute data from common sources of attribute data. Here we are using data from three different sources of which two are from [Statistics Finland PxWeb-api](https://pxdata.stat.fi/PXWeb/pxweb/en/), namely [municipality key figures](https://pxdata.stat.fi/PXWeb/pxweb/en/Kuntien_avainluvut/) and [Paavo (Open data by postal code area)](https://pxdata.stat.fi/PXWeb/pxweb/en/Postinumeroalueittainen_avoin_tieto/). Third is [Covid-19 tests and confirmed cases in Finland](https://thl.fi/fi/tilastot-ja-data/aineistot-ja-palvelut/avoin-data/varmistetut-koronatapaukset-suomessa-covid-19-).

**Installation**

`geofi` can be installed from CRAN using

```{r, eval = FALSE}
# install from CRAN
install.packages("geofi")

# Install development version from GitHub
remotes::install_github("ropengov/geofi")
```

```{r}
# Let's first create a function that checks if the suggested 
# packages are available
check_namespaces <- function(pkgs){
  return(all(unlist(sapply(pkgs, requireNamespace,quietly = TRUE))))
}
```


## Municipalities

Municipality data provided by `get_municipalities()`-function contains 77 indicators variables from each of 309 municipalities. Variables can be used either for aggregating data or as keys for joining attribute data. 

### PxWeb-data

In this first example we join municipality level data from Statistics Finland [municipality key figures](https://pxdata.stat.fi/PXWeb/pxweb/en/Kuntien_avainluvut/)

```{r municipality_map}
library(geofi)
muni <- get_municipalities(year = 2019)

libs <- c("pxweb","dplyr","tidyr","janitor","ggplot2")
if (check_namespaces(pkgs = libs)) {
library(pxweb)
pxweb_query_list <-
  list("Alue 2020"=c("*"),
       "Tiedot"=c("*"),
       "Vuosi"=c("2019"))

px_raw <-
  pxweb_get(url = "https://pxdata.stat.fi/PXWeb/api/v1/en/Kuntien_avainluvut/2020/kuntien_avainluvut_2020_aikasarja.px",
            query = pxweb_query_list)

library(dplyr)
library(tidyr)
library(janitor)
library(sf)
px_data <- as_tibble(
  as.data.frame(px_raw, 
                column.name.type = "text", 
                variable.value.type = "text")
  ) %>% setNames(make_clean_names(names(.))) %>% 
  pivot_longer(names_to = "information", values_to = "municipal_key_figures", 3:ncol(.))
px_data
} else {
  message("One or more of the following packages is not available: ", 
          paste(libs, collapse = ", "))
}
```

Once we have the data in long format we can observe the `region_2020`-column.

```{r municipality_map2}
if (check_namespaces(pkgs = libs)) {
count(px_data, region_2020)
  } else {
  message("One or more of the following packages is not available: ", 
          paste(libs, collapse = ", "))
}
```

This is not obvious to all, but have the municipality names in Finnish among other regional breakdowns which allows us to combine the data with spatial data using `municipality_name_fi`-variable. 

```{r}
if (check_namespaces(pkgs = libs)) {
map_data <- right_join(muni, 
                       px_data, 
                       by = c("municipality_name_fi" = "region_2020"))
  } else {
  message("One or more of the following packages is not available: ", 
          paste(libs, collapse = ", "))
}
```

Now we can plot a map showing `Share of Swedish-speakers of the population, %` and `Share of foreign citizens of the population, %` on two panels sharing a scale.

```{r, fig.width = 10, fig.height = 7}
if (check_namespaces(pkgs = libs)) {
library(ggplot2)
map_data %>% 
  filter(grepl("swedish|foreign", information)) %>% 
  ggplot(aes(fill = municipal_key_figures)) + 
  geom_sf() + 
  facet_wrap(~information) +
  theme(legend.position = "top")
  } else {
  message("One or more of the following packages is not available: ", 
          paste(libs, collapse = ", "))
}
```


## Health Districts

in early 2021 we are still troubled by the COVID-19 and the health authorities are counting infections, deaths and vaccinated. Lets pull the daily data from API and compare the names of the health districts both in COVID-19 data and in municipality division from Statistics Finland. 

```{r}
if (FALSE){
  library(readr)
  cols(
    Area = col_character(),
    Time = col_date(format = ""),
    val = col_double()
  ) -> cov_cols
  
  thl_korona_api <- "https://sampo.thl.fi/pivot/prod/en/epirapo/covid19case/fact_epirapo_covid19case.csv?row=dateweek20200101-508804L&column=hcdmunicipality2020-445222L"
  status <- httr::status_code(httr::GET(thl_korona_api))

  xdf_raw <- read_csv2(thl_korona_api, col_types = cov_cols)
  xdf <- xdf_raw %>% 
    # filter(!grepl("Kaikki", Alue)) %>% 
    rename(date = Time, 
           shp = Area, 
           day_cases = val) %>% 
    group_by(shp) %>% 
    arrange(shp,date) %>% 
    filter(!is.na(day_cases)) %>% 
    mutate(total_cases = cumsum(day_cases)) %>% 
    ungroup() %>% 
    group_by(shp) %>% 
    filter(date == max(date, na.rm = TRUE)) %>% 
    ungroup()
} 
xdf <- structure(list(shp = c("Åland", "All areas", "Central Finland Hospital District", 
"Central Ostrobothnia Hospital District", "Helsinki and Uusimaa Hospital District", 
"Itä-Savo Hospital District", "Kainuu Hospital District", "Kanta-Häme Hospital District", 
"Kymenlaakso Hospital District", "Länsi-Pohja Hospital District", 
"Lappi Hospital District", "North Karelia Hospital District", 
"North Ostrobothnia Hospital District", "North Savo Hospital District", 
"Päijät-Häme Hospital District", "Pirkanmaa Hospital District", 
"Satakunta Hospital District", "South Karelia Hospital District", 
"South Ostrobothnia Hospital District", "South Savo Hospital District", 
"Southwest Finland Hospital District", "Vaasa Hospital District"
), date = structure(c(18674, 18674, 18674, 18674, 18674, 18674, 
18674, 18674, 18674, 18674, 18674, 18674, 18674, 18674, 18674, 
18674, 18674, 18674, 18674, 18674, 18674, 18674), class = "Date"), 
    day_cases = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0, 0, 0, 0), total_cases = c(120, 51047, 1850, 177, 
    29519, 141, 227, 873, 854, 478, 465, 516, 2265, 850, 1243, 
    2662, 671, 348, 534, 573, 4803, 1878)), row.names = c(NA, 
-22L), class = c("tbl_df", "tbl", "data.frame"))

xdf %>% 
    count(shp)
  
muni <- get_municipalities(year = 2021) 
muni %>% 
  st_drop_geometry() %>% 
  count(sairaanhoitop_name_en)
```

The names look identical so we can join the two datasets and plot a map.

```{r}
libs <- c("ggplot2")
if (check_namespaces(pkgs = libs)) {
muni %>% 
  count(sairaanhoitop_name_en) %>% 
  left_join(xdf, by = c("sairaanhoitop_name_en" = "shp")) %>% 
  ggplot(aes(fill = total_cases)) +
  geom_sf() +
  geom_sf_text(aes(label = paste0(sairaanhoitop_name_en, "\n", total_cases)), 
               color = "white") +
  labs(title = "Number of total COVID-19 cases reported since January 2020", 
       fill = NULL)
} else {
  message("One or more of the following packages is not available: ", 
          paste(libs, collapse = ", "))
}
```


## Zipcode level

You can download data from [Paavo (Open data by postal code area)](https://pxdata.stat.fi/PXWeb/pxweb/en/Postinumeroalueittainen_avoin_tieto/) using `pxweb`-package in a similar manner as in the first example.

```{r zipcode_with_statistics_finland}
libs <- c("ggplot2","pxweb","janitor")
if (check_namespaces(pkgs = libs)) {
library(pxweb)
# lets get all zipcodes and all variables
pxweb_query_list <- 
  list("Postinumeroalue"=c("*"),
       "Tiedot"=c("he_vakiy"),
       "Vuosi"=c("2021"))

px_raw <- 
  pxweb_get(url = "https://statfin.stat.fi/PXWeb/api/v1/en/Postinumeroalueittainen_avoin_tieto/uusin/paavo_pxt_12ey.px",
            query = pxweb_query_list)


px_data <- as_tibble(
  as.data.frame(px_raw, 
                column.name.type = "text", 
                variable.value.type = "text")
  ) %>% setNames(make_clean_names(names(.)))
px_data %>% 
  filter(postal_code_area != "Finland")
} else {
  message("One or more of the following packages is not available: ", 
          paste(libs, collapse = ", "))
}
```

Before we can join the data, we must extract the numerical postal code from `postal_code_area`-variable.

```{r}
libs <- c("ggplot2","pxweb","janitor")
if (check_namespaces(pkgs = libs)) {
px_data$posti_alue <- sub(" .+$", "", px_data$postal_code_area)

# Lets join with spatial data and plot the area of each zipcode
zipcodes19 <- get_zipcodes(year = 2019) 
zipcodes_map <- left_join(zipcodes19, 
                          px_data)
ggplot(zipcodes_map) + 
  geom_sf(aes(fill = inhabitants_total_he), 
          color  = alpha("white", 1/3)) +
  labs(title = "Total number of inhabitants, 2021", 
       fill = NULL)
} else {
  message("One or more of the following packages is not available: ", 
          paste(libs, collapse = ", "))
}
```

