diff --git a/data/urban_percent.csv b/data/urban_percent.csv deleted file mode 100644 index d57e0728..00000000 --- a/data/urban_percent.csv +++ /dev/null @@ -1,30 +0,0 @@ -AT,66 -BA,40 -BE,98 -BG,74 -CH,74 -CZ,73 -DE,75 -DK,88 -EE,68 -ES,80 -FI,84 -FR,80 -GB,83 -GR,78 -HR,59 -HU,71 -IE,63 -IT,69 -LT,67 -LU,90 -LV,67 -NL,90 -NO,80 -PL,61 -PT,63 -RO,55 -RS,56 -SE,86 -SI,50 -SK,54 diff --git a/doc/configtables/licenses-sector.csv b/doc/configtables/licenses-sector.csv index a44e0a5d..8c721260 100644 --- a/doc/configtables/licenses-sector.csv +++ b/doc/configtables/licenses-sector.csv @@ -1,6 +1,5 @@ description,file/folder,licence,source JRC IDEES database,jrc-idees-2015/,CC BY 4.0,https://ec.europa.eu/jrc/en/potencia/jrc-idees -urban/rural fraction,urban_percent.csv,unknown,unknown JRC ENSPRESO biomass potentials,remote,CC BY 4.0,https://data.jrc.ec.europa.eu/dataset/74ed5a04-7d74-4807-9eab-b94774309d9f EEA emission statistics,eea/UNFCCC_v23.csv,EEA standard re-use policy,https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16 Eurostat Energy Balances,eurostat-energy_balances-*/,Eurostat,https://ec.europa.eu/eurostat/web/energy/data/energy-balances diff --git a/doc/release_notes.rst b/doc/release_notes.rst index caf73292..d5fbe781 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -1,3 +1,4 @@ + .. SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors @@ -10,6 +11,11 @@ Release Notes Upcoming Release ================ +* Retrieve share of urban population from `World Bank API + `__. The data + originates from the United Nations Population Division. Previously, a file + ``data/urban_percent.csv`` with an undocumented source was used. + * Updated country-specific Energy Availability Factors (EAFs) for nuclear power plants based on `IAEA 2021-2023 reported country averages `__. diff --git a/rules/build_sector.smk b/rules/build_sector.smk index c1e4d573..f95731b7 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -6,7 +6,7 @@ rule build_population_layouts: input: nuts3_shapes=resources("nuts3_shapes.geojson"), - urban_percent="data/urban_percent.csv", + urban_percent="data/worldbank/API_SP.URB.TOTL.IN.ZS_DS2_en_csv_v2_3403768.csv", cutout=lambda w: "cutouts/" + CDIR + config_provider("atlite", "default_cutout")(w) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index df1f7379..5c4af68e 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -259,6 +259,30 @@ if config["enable"]["retrieve"]: +if config["enable"]["retrieve"]: + + rule retrieve_worldbank_urban_population: + params: + zip="data/worldbank/API_SP.URB.TOTL.IN.ZS_DS2_en_csv_v2_3403768.zip", + output: + gpkg="data/worldbank/API_SP.URB.TOTL.IN.ZS_DS2_en_csv_v2_3403768.csv", + run: + import os + import requests + + response = requests.get( + "https://api.worldbank.org/v2/en/indicator/SP.URB.TOTL.IN.ZS?downloadformat=csv", + params={"name": "API_SP.URB.TOTL.IN.ZS_DS2_en_csv_v2_3403768.zip"}, + ) + + with open(params["zip"], "wb") as f: + f.write(response.content) + output_folder = Path(params["zip"]).parent + unpack_archive(params["zip"], output_folder) + os.remove(params["zip"]) + + + if config["enable"]["retrieve"]: # Download directly from naciscdn.org which is a redirect from naturalearth.com diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index ca664ed0..4cabdc7c 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -9,6 +9,7 @@ Build mapping between cutout grid cells and population (total, urban, rural). import logging import atlite +import country_converter as coco import geopandas as gpd import numpy as np import pandas as pd @@ -17,6 +18,10 @@ from _helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) +cc = coco.CountryConverter() + +coco.logging.getLogger().setLevel(coco.logging.CRITICAL) + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -45,20 +50,13 @@ if __name__ == "__main__": countries = np.sort(nuts3.country.unique()) + urban_fraction = pd.read_csv(snakemake.input.urban_percent, skiprows=4) + iso3 = urban_fraction["Country Code"] + urban_fraction["iso2"] = cc.convert(names=iso3, src="ISO3", to="ISO2") urban_fraction = ( - pd.read_csv( - snakemake.input.urban_percent, header=None, index_col=0, names=["fraction"] - ).squeeze() - / 100.0 + urban_fraction.query("iso2 in @countries").set_index("iso2")["2019"].div(100) ) - # fill missing Balkans values - missing = ["AL", "ME", "MK"] - reference = ["RS", "BA"] - average = urban_fraction[reference].mean() - fill_values = pd.Series({ct: average for ct in missing}) - urban_fraction = pd.concat([urban_fraction, fill_values]) - # population in each grid cell pop_cells = pd.Series(I.dot(nuts3["pop"]))