diff --git a/Snakefile b/Snakefile index 7df61162..4acba0c8 100644 --- a/Snakefile +++ b/Snakefile @@ -22,7 +22,7 @@ if not exists(conf_file) and exists(conf_default_file): configfile: "config/config.yaml" -COSTS = f"data/costs_{config['costs']['year']}.csv" +COSTS = f"resources/costs_{config['costs']['year']}.csv" ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 4) run = config["run"] diff --git a/doc/configtables/sector-opts.csv b/doc/configtables/sector-opts.csv index ea39c3b0..fc9e8c10 100644 --- a/doc/configtables/sector-opts.csv +++ b/doc/configtables/sector-opts.csv @@ -7,5 +7,5 @@ Trigger, Description, Definition, Status ``B``,Add biomass,,In active use ``I``,Add industry sector,,In active use ``A``,Add agriculture sector,,In active use -``dist``+``n``,Add distribution grid with investment costs of ``n`` times costs in ``data/costs_{cost_year}.csv``,,In active use +``dist``+``n``,Add distribution grid with investment costs of ``n`` times costs in ``resources/costs_{cost_year}.csv``,,In active use ``seq``+``n``,Sets the CO2 sequestration potential to ``n`` Mt CO2 per year,,In active use diff --git a/doc/costs.rst b/doc/costs.rst index 5ddbb360..9a06fd16 100644 --- a/doc/costs.rst +++ b/doc/costs.rst @@ -9,7 +9,7 @@ Techno-Economic Assumptions The database of cost assumptions is retrieved from the repository `PyPSA/technology-data `_ and then -saved to a file ``data/costs_{year}.csv``. The ``config/config.yaml`` provides options +saved to a file ``resources/costs_{year}.csv``. The ``config/config.yaml`` provides options to choose a reference year and use a specific version of the repository. .. literalinclude:: ../config/config.default.yaml @@ -50,7 +50,7 @@ Modifying Assumptions Some cost assumptions (e.g. marginal cost and capital cost) can be directly set in the ``config/config.yaml`` (cf. Section :ref:`costs_cf` in :ref:`config`). To change cost assumptions in more detail, make a copy of -``data/costs_{year}.csv`` and reference the new cost file in the ``Snakefile``: +``resources/costs_{year}.csv`` and reference the new cost file in the ``Snakefile``: .. literalinclude:: ../Snakefile :start-at: COSTS diff --git a/rules/build_sector.smk b/rules/build_sector.smk index d7fbe638..62b69337 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -853,10 +853,10 @@ rule prepare_sector_network: "biomass_potentials_s{simpl}_{clusters}_{planning_horizons}.csv" ) ), - costs=( - "data/costs_{}.csv".format(config_provider("costs", "year")) - if config_provider("foresight") == "overnight" - else "data/costs_{planning_horizons}.csv" + costs=lambda w: ( + "resources/costs_{}.csv".format(config_provider("costs", "year")) + if config_provider("foresight")(w) == "overnight" + else "resources/costs_{planning_horizons}.csv" ), profile_offwind_ac=resources("profile_offwind-ac.nc"), profile_offwind_dc=resources("profile_offwind-dc.nc"), diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 38e5f7d9..542c8d29 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -174,10 +174,10 @@ rule make_summary: **config["scenario"], run=config["run"]["name"], ), - costs=( - "data/costs_{}.csv".format(config_provider("costs", "year")) - if config_provider("foresight") == "overnight" - else "data/costs_{}.csv".format( + costs=lambda w: ( + "resources/costs_{}.csv".format(config_provider("costs", "year")) + if config_provider("foresight")(w) == "overnight" + else "resources/costs_{}.csv".format( config_provider("scenario", "planning_horizons", 0) ) ), diff --git a/rules/retrieve.smk b/rules/retrieve.smk index c9fd91aa..1b2513fb 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -83,23 +83,17 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", True): rule retrieve_cost_data: - input: - HTTP.remote( - "raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format( - config_provider("costs", "version") - ) - + "costs_{year}.csv", - keep_local=True, - ), + params: + version=lambda w: config_provider("costs", "version")(w), output: - "data/costs_{year}.csv", + resources("costs_{year}.csv"), log: "logs/retrieve_cost_data_{year}.log", resources: mem_mb=1000, retries: 2 - run: - move(input[0], output[0]) + script: + "../scripts/retrieve_cost_data.py" if config["enable"]["retrieve"] and config["enable"].get( diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index a6313cac..bea6b6cc 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -16,7 +16,7 @@ rule add_existing_baseyear: busmap_s=resources("busmap_elec_s{simpl}.csv"), busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), - costs=lambda w: "data/costs_{}.csv".format( + costs=lambda w: "resources/costs_{}.csv".format( config_provider("scenario", "planning_horizons", 0)(w) ), cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), @@ -71,7 +71,7 @@ rule add_brownfield: network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", network_p=solved_previous_horizon, #solved network at previous time step - costs="data/costs_{planning_horizons}.csv", + costs="resources/costs_{planning_horizons}.csv", cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), output: @@ -109,7 +109,7 @@ rule solve_sector_network_myopic: input: network=RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - costs="data/costs_{planning_horizons}.csv", + costs="resources/costs_{planning_horizons}.csv", config=RESULTS + "config.yaml", output: RESULTS diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index ee16a201..d1a5f745 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -14,7 +14,7 @@ rule add_existing_baseyear: busmap_s=resources("busmap_elec_s{simpl}.csv"), busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), - costs="data/costs_{}.csv".format( + costs="resources/costs_{}.csv".format( config_provider("scenario", "planning_horizons", 0) ), cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), @@ -95,7 +95,7 @@ rule solve_sector_network_perfect: input: network=RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", - costs="data/costs_2030.csv", + costs="resources/costs_2030.csv", config=RESULTS + "config.yaml", output: RESULTS @@ -127,13 +127,7 @@ rule make_summary_perfect: **{ f"networks_{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}": RESULTS + f"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc" - for simpl in config_provider("scenario", "simpl") - for clusters in config_provider("scenario", "clusters") - for opts in config_provider("scenario", "opts") - for sector_opts in config_provider("scenario", "sector_opts") - for ll in config_provider("scenario", "ll") - }, - costs="data/costs_2020.csv", + costs="resources/costs_2020.csv", output: nodal_costs=RESULTS + "csvs/nodal_costs.csv", nodal_capacities=RESULTS + "csvs/nodal_capacities.csv", diff --git a/scripts/retrieve_cost_data.py b/scripts/retrieve_cost_data.py new file mode 100644 index 00000000..ceae8bf9 --- /dev/null +++ b/scripts/retrieve_cost_data.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +""" +Retrieve cost data from ``technology-data``. +""" + +import logging +from pathlib import Path + +from _helpers import configure_logging, progress_retrieve, set_scenario_config + +logger = logging.getLogger(__name__) + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("retrieve_cost_data", year=2030) + rootpath = ".." + else: + rootpath = "." + configure_logging(snakemake) + set_scenario_config(snakemake) + + version = snakemake.params.version + baseurl = f"https://raw.githubusercontent.com/PyPSA/technology-data/{version}/outputs/" + filepath = Path(snakemake.output[0]) + url = baseurl + filepath.name + + print(url) + + to_fn = Path(rootpath) / filepath + + print(to_fn) + + logger.info(f"Downloading technology data from '{url}'.") + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + progress_retrieve(url, to_fn, disable=disable_progress) + + logger.info(f"Technology data available at at {to_fn}")