smk: use storage() only in combination with retrieve rules (#1274)

* smk: use storage() only in combination with retrieve rules

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

doc/release_notes.rst

@@ -17,6 +17,10 @@ Release Notes
 
 * bugfix: The oil generator was incorrectly dropped when the config `oil_refining_emissions` was greater than zero. This was the default behaviour in 0.12.0.
 
+* Uses of Snakemake's ``storage()`` function are integrated into retrieval
+  rules. This simplifies the use of ``mock_snakemake`` and places downloaded
+  data more transparently into the ``data`` directory.
+
 PyPSA-Eur 0.12.0 (30th August 2024)
 ===================================
 

rules/build_sector.smk

@@ -402,10 +402,7 @@ rule build_biomass_potentials:
     params:
         biomass=config_provider("biomass"),
     input:
-        enspreso_biomass=storage(
+        enspreso_biomass="data/ENSPRESO_BIOMASS.xlsx",
-            "https://zenodo.org/records/10356004/files/ENSPRESO_BIOMASS.xlsx",
-            keep_local=True,
-        ),
         eurostat="data/eurostat/Balances-April2023",
         nuts2="data/bundle/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
         regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
@@ -458,10 +455,7 @@ rule build_sequestration_potentials:
             "sector", "regional_co2_sequestration_potential"
         ),
     input:
-        sequestration_potential=storage(
+        sequestration_potential="data/omplete_map_2020_unit_Mt.geojson",
-            "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson",
-            keep_local=True,
-        ),
         regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
         regions_offshore=resources("regions_offshore_elec_s{simpl}_{clusters}.geojson"),
     output:
@@ -503,9 +497,7 @@ rule build_salt_cavern_potentials:
 
 rule build_ammonia_production:
     input:
-        usgs=storage(
+        usgs="data/myb1-2022-nitro-ert.xlsx",
-            "https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/media/files/myb1-2022-nitro-ert.xlsx"
-        ),
     output:
         ammonia_production=resources("ammonia_production.csv"),
     threads: 1
@@ -634,10 +626,7 @@ rule build_industrial_distribution_key:
     input:
         regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
         clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
-        hotmaps=storage(
+        hotmaps="data/Industrial_Database.csv",
-            "https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database/-/raw/master/data/Industrial_Database.csv",
-            keep_local=True,
-        ),
         gem_gspt="data/gem/Global-Steel-Plant-Tracker-April-2024-Standard-Copy-V1.xlsx",
         ammonia="data/ammonia_plants.csv",
         cement_supplement="data/cement-plants-noneu.csv",

rules/retrieve.smk

@@ -191,6 +191,65 @@ if config["enable"]["retrieve"]:
             validate_checksum(output[0], input[0])
 
 
+if config["enable"]["retrieve"]:
+
+    rule retrieve_jrc_enspreso_biomass:
+        input:
+            storage(
+                "https://zenodo.org/records/10356004/files/ENSPRESO_BIOMASS.xlsx",
+                keep_local=True,
+            ),
+        output:
+            "data/ENSPRESO_BIOMASS.xlsx",
+        retries: 1
+        run:
+            move(input[0], output[0])
+
+
+if config["enable"]["retrieve"]:
+
+    rule retrieve_hotmaps_industrial_sites:
+        input:
+            storage(
+                "https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database/-/raw/master/data/Industrial_Database.csv",
+                keep_local=True,
+            ),
+        output:
+            "data/Industrial_Database.csv",
+        retries: 1
+        run:
+            move(input[0], output[0])
+
+
+if config["enable"]["retrieve"]:
+
+    rule retrieve_usgs_ammonia_production:
+        input:
+            storage(
+                "https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/media/files/myb1-2022-nitro-ert.xlsx"
+            ),
+        output:
+            "data/myb1-2022-nitro-ert.xlsx",
+        retries: 1
+        run:
+            move(input[0], output[0])
+
+
+if config["enable"]["retrieve"]:
+
+    rule retrieve_geological_co2_storage_potential:
+        input:
+            storage(
+                "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson",
+                keep_local=True,
+            ),
+        output:
+            "data/complete_map_2020_unit_Mt.geojson",
+        retries: 1
+        run:
+            move(input[0], output[0])
+
+
 if config["enable"]["retrieve"]:
 
     # Downloading Copernicus Global Land Cover for land cover and land use: