retrieve urban population fraction from World Bank API (#1248)

* retrieve urban population fraction from World Bank API

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

data/urban_percent.csv

@@ -1,30 +0,0 @@
-AT,66
-BA,40
-BE,98
-BG,74
-CH,74
-CZ,73
-DE,75
-DK,88
-EE,68
-ES,80
-FI,84
-FR,80
-GB,83
-GR,78
-HR,59
-HU,71
-IE,63
-IT,69
-LT,67
-LU,90
-LV,67
-NL,90
-NO,80
-PL,61
-PT,63
-RO,55
-RS,56
-SE,86
-SI,50
-SK,54

doc/configtables/licenses-sector.csv

@@ -1,6 +1,5 @@
 description,file/folder,licence,source
 JRC IDEES database,jrc-idees-2015/,CC BY 4.0,https://ec.europa.eu/jrc/en/potencia/jrc-idees
-urban/rural fraction,urban_percent.csv,unknown,unknown
 JRC ENSPRESO biomass potentials,remote,CC BY 4.0,https://data.jrc.ec.europa.eu/dataset/74ed5a04-7d74-4807-9eab-b94774309d9f
 EEA emission statistics,eea/UNFCCC_v23.csv,EEA standard re-use policy,https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16
 Eurostat Energy Balances,eurostat-energy_balances-*/,Eurostat,https://ec.europa.eu/eurostat/web/energy/data/energy-balances

doc/release_notes.rst

@@ -1,3 +1,4 @@
+
 ..
   SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
@@ -10,6 +11,11 @@ Release Notes
 Upcoming Release
 ================
 
+* Retrieve share of urban population from `World Bank API
+  <https://data.worldbank.org/indicator/SP.URB.TOTL.IN.ZS>`__. The data
+  originates from the United Nations Population Division. Previously, a file
+  ``data/urban_percent.csv`` with an undocumented source was used.
+
 * Updated country-specific Energy Availability Factors (EAFs) for nuclear power
   plants based on `IAEA 2021-2023 reported country averages
   <https://pris.iaea.org/PRIS/WorldStatistics/ThreeYrsEnergyAvailabilityFactor.aspx>`__.

rules/build_sector.smk

@@ -6,7 +6,7 @@
 rule build_population_layouts:
     input:
         nuts3_shapes=resources("nuts3_shapes.geojson"),
-        urban_percent="data/urban_percent.csv",
+        urban_percent="data/worldbank/API_SP.URB.TOTL.IN.ZS_DS2_en_csv_v2_3403768.csv",
         cutout=lambda w: "cutouts/"
         + CDIR
         + config_provider("atlite", "default_cutout")(w)

rules/retrieve.smk

@@ -259,6 +259,30 @@ if config["enable"]["retrieve"]:
 
 
 
+if config["enable"]["retrieve"]:
+
+    rule retrieve_worldbank_urban_population:
+        params:
+            zip="data/worldbank/API_SP.URB.TOTL.IN.ZS_DS2_en_csv_v2_3403768.zip",
+        output:
+            gpkg="data/worldbank/API_SP.URB.TOTL.IN.ZS_DS2_en_csv_v2_3403768.csv",
+        run:
+            import os
+            import requests
+
+            response = requests.get(
+                "https://api.worldbank.org/v2/en/indicator/SP.URB.TOTL.IN.ZS?downloadformat=csv",
+                params={"name": "API_SP.URB.TOTL.IN.ZS_DS2_en_csv_v2_3403768.zip"},
+            )
+
+            with open(params["zip"], "wb") as f:
+                f.write(response.content)
+            output_folder = Path(params["zip"]).parent
+            unpack_archive(params["zip"], output_folder)
+            os.remove(params["zip"])
+
+
+
 if config["enable"]["retrieve"]:
 
     # Download directly from naciscdn.org which is a redirect from naturalearth.com

scripts/build_population_layouts.py

@@ -9,6 +9,7 @@ Build mapping between cutout grid cells and population (total, urban, rural).
 import logging
 
 import atlite
+import country_converter as coco
 import geopandas as gpd
 import numpy as np
 import pandas as pd
@@ -17,6 +18,10 @@ from _helpers import configure_logging, set_scenario_config
 
 logger = logging.getLogger(__name__)
 
+cc = coco.CountryConverter()
+
+coco.logging.getLogger().setLevel(coco.logging.CRITICAL)
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -45,20 +50,13 @@ if __name__ == "__main__":
 
     countries = np.sort(nuts3.country.unique())
 
+    urban_fraction = pd.read_csv(snakemake.input.urban_percent, skiprows=4)
+    iso3 = urban_fraction["Country Code"]
+    urban_fraction["iso2"] = cc.convert(names=iso3, src="ISO3", to="ISO2")
     urban_fraction = (
-        pd.read_csv(
+        urban_fraction.query("iso2 in @countries").set_index("iso2")["2019"].div(100)
-            snakemake.input.urban_percent, header=None, index_col=0, names=["fraction"]
-        ).squeeze()
-        / 100.0
     )
 
-    # fill missing Balkans values
-    missing = ["AL", "ME", "MK"]
-    reference = ["RS", "BA"]
-    average = urban_fraction[reference].mean()
-    fill_values = pd.Series({ct: average for ct in missing})
-    urban_fraction = pd.concat([urban_fraction, fill_values])
-
     # population in each grid cell
     pop_cells = pd.Series(I.dot(nuts3["pop"]))