add retrieve_cost_data.py script to handle config_provider and move to resources

since dependent on config setting

Snakefile

@@ -22,7 +22,7 @@ if not exists(conf_file) and exists(conf_default_file):
 configfile: "config/config.yaml"
 
 
-COSTS = f"data/costs_{config['costs']['year']}.csv"
+COSTS = f"resources/costs_{config['costs']['year']}.csv"
 ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 4)
 
 run = config["run"]

doc/configtables/sector-opts.csv

@@ -7,5 +7,5 @@ Trigger, Description, Definition, Status
 ``B``,Add biomass,,In active use
 ``I``,Add industry sector,,In active use
 ``A``,Add agriculture sector,,In active use
-``dist``+``n``,Add distribution grid with investment costs of ``n`` times costs in ``data/costs_{cost_year}.csv``,,In active use
+``dist``+``n``,Add distribution grid with investment costs of ``n`` times costs in ``resources/costs_{cost_year}.csv``,,In active use
 ``seq``+``n``,Sets the CO2 sequestration potential to ``n`` Mt CO2 per year,,In active use

doc/costs.rst

@@ -9,7 +9,7 @@ Techno-Economic Assumptions
 
 The database of cost assumptions is retrieved from the repository
 `PyPSA/technology-data <https://github.com/pypsa/technology-data>`_ and then
-saved to a file ``data/costs_{year}.csv``. The ``config/config.yaml`` provides options
+saved to a file ``resources/costs_{year}.csv``. The ``config/config.yaml`` provides options
 to choose a reference year and use a specific version of the repository.
 
 .. literalinclude:: ../config/config.default.yaml
@@ -50,7 +50,7 @@ Modifying Assumptions
 Some cost assumptions (e.g. marginal cost and capital cost) can be directly
 set in the ``config/config.yaml`` (cf. Section  :ref:`costs_cf`  in
 :ref:`config`). To change cost assumptions in more detail, make a copy of
-``data/costs_{year}.csv`` and reference the new cost file in the ``Snakefile``:
+``resources/costs_{year}.csv`` and reference the new cost file in the ``Snakefile``:
 
 .. literalinclude:: ../Snakefile
    :start-at: COSTS

rules/build_sector.smk

@@ -853,10 +853,10 @@ rule prepare_sector_network:
                 "biomass_potentials_s{simpl}_{clusters}_{planning_horizons}.csv"
             )
         ),
-        costs=(
+        costs=lambda w: (
-            "data/costs_{}.csv".format(config_provider("costs", "year"))
+            "resources/costs_{}.csv".format(config_provider("costs", "year"))
-            if config_provider("foresight") == "overnight"
+            if config_provider("foresight")(w) == "overnight"
-            else "data/costs_{planning_horizons}.csv"
+            else "resources/costs_{planning_horizons}.csv"
         ),
         profile_offwind_ac=resources("profile_offwind-ac.nc"),
         profile_offwind_dc=resources("profile_offwind-dc.nc"),

rules/postprocess.smk

@@ -174,10 +174,10 @@ rule make_summary:
             **config["scenario"],
             run=config["run"]["name"],
         ),
-        costs=(
+        costs=lambda w: (
-            "data/costs_{}.csv".format(config_provider("costs", "year"))
+            "resources/costs_{}.csv".format(config_provider("costs", "year"))
-            if config_provider("foresight") == "overnight"
+            if config_provider("foresight")(w) == "overnight"
-            else "data/costs_{}.csv".format(
+            else "resources/costs_{}.csv".format(
                 config_provider("scenario", "planning_horizons", 0)
             )
         ),

rules/retrieve.smk

@@ -83,23 +83,17 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True
 if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", True):
 
     rule retrieve_cost_data:
-        input:
+        params:
-            HTTP.remote(
+            version=lambda w: config_provider("costs", "version")(w),
-                "raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format(
-                    config_provider("costs", "version")
-                )
-                + "costs_{year}.csv",
-                keep_local=True,
-            ),
         output:
-            "data/costs_{year}.csv",
+            resources("costs_{year}.csv"),
         log:
             "logs/retrieve_cost_data_{year}.log",
         resources:
             mem_mb=1000,
         retries: 2
-        run:
+        script:
-            move(input[0], output[0])
+            "../scripts/retrieve_cost_data.py"
 
 
 if config["enable"]["retrieve"] and config["enable"].get(

rules/solve_myopic.smk

@@ -16,7 +16,7 @@ rule add_existing_baseyear:
         busmap_s=resources("busmap_elec_s{simpl}.csv"),
         busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"),
         clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
-        costs=lambda w: "data/costs_{}.csv".format(
+        costs=lambda w: "resources/costs_{}.csv".format(
             config_provider("scenario", "planning_horizons", 0)(w)
         ),
         cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
@@ -71,7 +71,7 @@ rule add_brownfield:
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         network_p=solved_previous_horizon,  #solved network at previous time step
-        costs="data/costs_{planning_horizons}.csv",
+        costs="resources/costs_{planning_horizons}.csv",
         cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
         cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
     output:
@@ -109,7 +109,7 @@ rule solve_sector_network_myopic:
     input:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-        costs="data/costs_{planning_horizons}.csv",
+        costs="resources/costs_{planning_horizons}.csv",
         config=RESULTS + "config.yaml",
     output:
         RESULTS

rules/solve_perfect.smk

@@ -14,7 +14,7 @@ rule add_existing_baseyear:
         busmap_s=resources("busmap_elec_s{simpl}.csv"),
         busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"),
         clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
-        costs="data/costs_{}.csv".format(
+        costs="resources/costs_{}.csv".format(
             config_provider("scenario", "planning_horizons", 0)
         ),
         cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
@@ -95,7 +95,7 @@ rule solve_sector_network_perfect:
     input:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
-        costs="data/costs_2030.csv",
+        costs="resources/costs_2030.csv",
         config=RESULTS + "config.yaml",
     output:
         RESULTS
@@ -127,13 +127,7 @@ rule make_summary_perfect:
         **{
             f"networks_{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}": RESULTS
             + f"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc"
-            for simpl in config_provider("scenario", "simpl")
+        costs="resources/costs_2020.csv",
-            for clusters in config_provider("scenario", "clusters")
-            for opts in config_provider("scenario", "opts")
-            for sector_opts in config_provider("scenario", "sector_opts")
-            for ll in config_provider("scenario", "ll")
-        },
-        costs="data/costs_2020.csv",
     output:
         nodal_costs=RESULTS + "csvs/nodal_costs.csv",
         nodal_capacities=RESULTS + "csvs/nodal_capacities.csv",

scripts/retrieve_cost_data.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2024 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+"""
+Retrieve cost data from ``technology-data``.
+"""
+
+import logging
+from pathlib import Path
+
+from _helpers import configure_logging, progress_retrieve, set_scenario_config
+
+logger = logging.getLogger(__name__)
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake("retrieve_cost_data", year=2030)
+        rootpath = ".."
+    else:
+        rootpath = "."
+    configure_logging(snakemake)
+    set_scenario_config(snakemake)
+
+    version = snakemake.params.version
+    baseurl = f"https://raw.githubusercontent.com/PyPSA/technology-data/{version}/outputs/"
+    filepath = Path(snakemake.output[0])
+    url = baseurl + filepath.name
+
+    print(url)
+
+    to_fn = Path(rootpath) / filepath
+
+    print(to_fn)
+
+    logger.info(f"Downloading technology data from '{url}'.")
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
+    progress_retrieve(url, to_fn, disable=disable_progress)
+
+    logger.info(f"Technology data available at at {to_fn}")