Update NUTS3 shapes (#1286)

* increase nuts3 shape resolution

* retrieve and use shapes with EPSG:4326

* co-retrieve nuts2 shapes; add retries

* added release note

* changed doc/data_sources

doc/data_sources.rst

@@ -61,13 +61,6 @@ scope to reduce file size, or are not provided through stable URLs elsewhere.
 - **License:** CC-BY 4.0 (`reference <https://www.eea.europa.eu/en/legal-notice#copyright-notice>`__)
 - **Description:** Total GHG emissions and removals in the EU.
 
-``data/bundle/nuts``
-
-- **Source:** GISCO
-- **Link:** https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/
-- **License:** `custom <https://ec.europa.eu/eurostat/web/gisco/geodata/reference-data/administrative-units-statistical-units>`__
-- **Description:** Europe's NUTS administrative regions.
-
 ``data/bundle/emobility``
 
 - **Source:** German Federal Highway Research Institute (BASt)
@@ -127,6 +120,13 @@ Specific retrieval rules
 
 Data in this section is retrieved and extracted in rules specified in ``rules/retrieve.smk``.
 
+``data/nuts``
+
+- **Source:** GISCO
+- **Link:** https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/
+- **License:** `custom <https://ec.europa.eu/eurostat/web/gisco/geodata/reference-data/administrative-units-statistical-units>`__
+- **Description:** Europe's NUTS administrative regions.
+
 ``data/ENSPRESO_BIOMASS.xlsx``
 
 - **Source:** European Commission Joint Research Centre (JRC)

doc/release_notes.rst

@@ -18,6 +18,8 @@ Release Notes
 
 * Updated district heating supply temperatures based on `Euroheat's DHC Market Outlook 2024<https://api.euroheat.org/uploads/Market_Outlook_2024_beeecd62d4.pdf>`__ and `AGFW-Hauptbericht 2022 <https://www.agfw.de/securedl/sdl-eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE3MjU2MjI2MTUsImV4cCI6MTcyNTcxMjYxNSwidXNlciI6MCwiZ3JvdXBzIjpbMCwtMV0sImZpbGUiOiJmaWxlYWRtaW4vdXNlcl91cGxvYWQvWmFobGVuX3VuZF9TdGF0aXN0aWtlbi9IYXVwdGJlcmljaHRfMjAyMi9BR0ZXX0hhdXB0YmVyaWNodF8yMDIyLnBkZiIsInBhZ2UiOjQzNn0.Bhma3PKg9uJnC57Ixi2p9STW5-II9VXPTDXS544M208/AGFW_Hauptbericht_2022.pdf>`__. `min_forward_temperature` and `return_temperature` (not given by Euroheat) are extrapolated based on German values.
 
+* Increased the resolution of NUTS3 and NUTS2 shapes from 1:60M to 1:3M, with data now directly retrieved from GISCO
+
 * Made the overdimensioning factor for heating systems specific for central/decentral heating, defaults to no overdimensionining for central heating and no changes to decentral heating compared to previous version.
 
 * bugfix: The carrier of stores was silently overwritten by their bus_carrier as a side effect when building the co2 constraints

rules/build_electricity.smk

@@ -98,7 +98,7 @@ rule build_shapes:
     input:
         naturalearth=ancient("data/naturalearth/ne_10m_admin_0_countries_deu.shp"),
         eez=ancient("data/eez/World_EEZ_v12_20231025_LR/eez_v12_lowres.gpkg"),
-        nuts3=ancient("data/bundle/NUTS_2013_60M_SH/data/NUTS_RG_60M_2013.shp"),
+        nuts3=ancient("data/nuts/NUTS_RG_03M_2013_4326_LEVL_3.geojson"),
         nuts3pop=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"),
         nuts3gdp=ancient("data/bundle/nama_10r_3gdp.tsv.gz"),
         ch_cantons=ancient("data/ch_cantons.csv"),

rules/build_sector.smk

@@ -405,7 +405,7 @@ rule build_biomass_potentials:
     input:
         enspreso_biomass="data/ENSPRESO_BIOMASS.xlsx",
         eurostat="data/eurostat/Balances-April2023",
-        nuts2="data/bundle/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
+        nuts2="data/nuts/NUTS_RG_03M_2013_4326_LEVL_2.geojson",
         regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
         nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"),
         swiss_cantons=ancient("data/ch_cantons.csv"),

rules/retrieve.smk

@@ -5,6 +5,7 @@
 import requests
 from datetime import datetime, timedelta
 from shutil import move, unpack_archive
+from zipfile import ZipFile
 
 if config["enable"].get("retrieve", "auto") == "auto":
     config["enable"]["retrieve"] = has_internet_access()
@@ -16,12 +17,10 @@ if config["enable"]["retrieve"] is False:
 if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", True):
     datafiles = [
         "je-e-21.03.02.xls",
-        "NUTS_2013_60M_SH/data/NUTS_RG_60M_2013.shp",
         "nama_10r_3popgdp.tsv.gz",
         "nama_10r_3gdp.tsv.gz",
         "corine/g250_clc06_V18_5.tif",
         "eea/UNFCCC_v23.csv",
-        "nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
         "emobility/KFZ__count",
         "emobility/Pkw__count",
         "h2_salt_caverns_GWh_per_sqkm.geojson",
@@ -77,6 +76,32 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle",
             "../scripts/retrieve_eurostat_household_data.py"
 
 
+if config["enable"]["retrieve"]:
+
+    rule retrieve_nuts_shapes:
+        input:
+            shapes=storage(
+                "https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2013-03m.geojson.zip"
+            ),
+        output:
+            shapes_level_3="data/nuts/NUTS_RG_03M_2013_4326_LEVL_3.geojson",
+            shapes_level_2="data/nuts/NUTS_RG_03M_2013_4326_LEVL_2.geojson",
+        params:
+            zip_file="data/nuts/ref-nuts-2013-03m.geojson.zip",
+        run:
+            os.rename(input.shapes, params.zip_file)
+            with ZipFile(params.zip_file, "r") as zip_ref:
+                for level in ["LEVL_3", "LEVL_2"]:
+                    filename = f"NUTS_RG_03M_2013_4326_{level}.geojson"
+                    zip_ref.extract(filename, Path(output.shapes_level_3).parent)
+                    extracted_file = Path(output.shapes_level_3).parent / filename
+                    extracted_file.rename(
+                        getattr(output, f"shapes_level_{level[-1]}")
+                    )
+            os.remove(params.zip_file)
+
+
+
 if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True):
 
     rule retrieve_cutout:

scripts/build_biomass_potentials.py

@@ -193,7 +193,7 @@ def build_nuts2_shapes():
     - consistently name ME, MK
     """
     nuts2 = gpd.GeoDataFrame(
-        gpd.read_file(snakemake.input.nuts2).set_index("id").geometry
+        gpd.read_file(snakemake.input.nuts2).set_index("NUTS_ID").geometry
     )
 
     countries = gpd.read_file(snakemake.input.country_shapes).set_index("name")
@@ -345,7 +345,7 @@ if __name__ == "__main__":
         snakemake = mock_snakemake(
             "build_biomass_potentials",
             simpl="",
-            clusters="38",
+            clusters="39",
             planning_horizons=2050,
         )
 

scripts/build_shapes.py

@@ -150,7 +150,6 @@ def country_cover(country_shapes, eez_shapes=None):
 
 def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp):
     df = gpd.read_file(nuts3)
-    df = df.loc[df["STAT_LEVL_"] == 3]
     df["geometry"] = df["geometry"].map(_simplify_polys)
     df = df.rename(columns={"NUTS_ID": "id"})[["id", "geometry"]].set_index("id")