From 03e3278f89f9ada1177e7202907289129ef71872 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 31 Jul 2023 12:23:03 +0200 Subject: [PATCH 1/5] prepare_sector: add biomass transport costs for networks wo biomass network --- rules/build_sector.smk | 52 +++++++++++++------------------ scripts/prepare_sector_network.py | 27 +++++++++++++--- 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 4fe561a8..170a25e8 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -295,35 +295,25 @@ rule build_biomass_potentials: "../scripts/build_biomass_potentials.py" -if config["sector"]["biomass_transport"]: - - rule build_biomass_transport_costs: - input: - transport_cost_data=HTTP.remote( - "publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", - keep_local=True, - ), - output: - biomass_transport_costs=RESOURCES + "biomass_transport_costs.csv", - threads: 1 - resources: - mem_mb=1000, - log: - LOGS + "build_biomass_transport_costs.log", - benchmark: - BENCHMARKS + "build_biomass_transport_costs" - conda: - "../envs/environment.yaml" - script: - "../scripts/build_biomass_transport_costs.py" - - build_biomass_transport_costs_output = rules.build_biomass_transport_costs.output - - -if not config["sector"]["biomass_transport"]: - # this is effecively an `else` statement which is however not liked by snakefmt - - build_biomass_transport_costs_output = {} +rule build_biomass_transport_costs: + input: + transport_cost_data=HTTP.remote( + "publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", + keep_local=True, + ), + output: + biomass_transport_costs=RESOURCES + "biomass_transport_costs.csv", + threads: 1 + resources: + mem_mb=1000, + log: + LOGS + "build_biomass_transport_costs.log", + benchmark: + BENCHMARKS + "build_biomass_transport_costs" + conda: + "../envs/environment.yaml" + script: + "../scripts/build_biomass_transport_costs.py" if config["sector"]["regional_co2_sequestration_potential"]["enable"]: @@ -721,7 +711,7 @@ rule prepare_sector_network: RDIR=RDIR, input: **build_retro_cost_output, - **build_biomass_transport_costs_output, + **rules.build_biomass_transport_costs.output, **gas_infrastructure, **build_sequestration_potentials_output, network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", @@ -782,7 +772,7 @@ rule prepare_sector_network: + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", threads: 1 resources: - mem_mb=2000, + mem_mb=20000, log: LOGS + "prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index e0afbcd8..2fc7d2f9 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -2155,12 +2155,12 @@ def add_biomass(n, costs): p_nom_extendable=True, ) - if options["biomass_transport"]: - transport_costs = pd.read_csv( - snakemake.input.biomass_transport_costs, - index_col=0, - ).squeeze() + transport_costs = pd.read_csv( + snakemake.input.biomass_transport_costs, + index_col=0, + ).squeeze() + if options["biomass_transport"]: # add biomass transport biomass_transport = create_network_topology( n, "biomass transport ", bidirectional=False @@ -2185,6 +2185,23 @@ def add_biomass(n, costs): carrier="solid biomass transport", ) + elif options["biomass_spatial"]: + # add artificial biomass generators at nodes which include transport costs + bus_transport_costs = spatial.biomass.nodes.to_series().apply( + lambda x: transport_costs[x[:2]] + ) + average_distance = 200 # km #TODO: validate this assumption + + n.madd( + "Generator", + spatial.biomass.nodes, + bus=spatial.biomass.nodes, + carrier="solid biomass", + p_nom=10000, + marginal_cost=costs.at["solid biomass", "fuel"] + + bus_transport_costs * average_distance, + ) + # AC buses with district heating urban_central = n.buses.index[n.buses.carrier == "urban central heat"] if not urban_central.empty and options["chp"]: From f205ff9536bbf61f704d7a3b95e92e8711b25159 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 2 Aug 2023 14:02:25 +0200 Subject: [PATCH 2/5] prepare_sectors: make biomass transport costs conditional again --- rules/build_sector.smk | 51 +++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 170a25e8..356abdc5 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -295,25 +295,34 @@ rule build_biomass_potentials: "../scripts/build_biomass_potentials.py" -rule build_biomass_transport_costs: - input: - transport_cost_data=HTTP.remote( - "publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", - keep_local=True, - ), - output: - biomass_transport_costs=RESOURCES + "biomass_transport_costs.csv", - threads: 1 - resources: - mem_mb=1000, - log: - LOGS + "build_biomass_transport_costs.log", - benchmark: - BENCHMARKS + "build_biomass_transport_costs" - conda: - "../envs/environment.yaml" - script: - "../scripts/build_biomass_transport_costs.py" +if config["sector"]["biomass_transport"] or config["sector"]["biomass_spatial"]: + + rule build_biomass_transport_costs: + input: + transport_cost_data=HTTP.remote( + "publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", + keep_local=True, + ), + output: + biomass_transport_costs=RESOURCES + "biomass_transport_costs.csv", + threads: 1 + resources: + mem_mb=1000, + log: + LOGS + "build_biomass_transport_costs.log", + benchmark: + BENCHMARKS + "build_biomass_transport_costs" + conda: + "../envs/environment.yaml" + script: + "../scripts/build_biomass_transport_costs.py" + + build_biomass_transport_costs_output = rules.build_biomass_transport_costs.output + + +if not (config["sector"]["biomass_transport"] or config["sector"]["biomass_spatial"]): + # this is effecively an `else` statement which is however not liked by snakefmt + build_biomass_transport_costs_output = {} if config["sector"]["regional_co2_sequestration_potential"]["enable"]: @@ -711,7 +720,7 @@ rule prepare_sector_network: RDIR=RDIR, input: **build_retro_cost_output, - **rules.build_biomass_transport_costs.output, + **build_biomass_transport_costs_output, **gas_infrastructure, **build_sequestration_potentials_output, network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", @@ -772,7 +781,7 @@ rule prepare_sector_network: + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", threads: 1 resources: - mem_mb=20000, + mem_mb=2000, log: LOGS + "prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", From 5b057575df0bd4f0eb8bec58f343b98a22ce8b09 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 2 Aug 2023 14:39:20 +0200 Subject: [PATCH 3/5] prepare_sector_network: make use of biomass transport file optional --- scripts/prepare_sector_network.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 2fc7d2f9..0c1faacc 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -2155,13 +2155,12 @@ def add_biomass(n, costs): p_nom_extendable=True, ) - transport_costs = pd.read_csv( - snakemake.input.biomass_transport_costs, - index_col=0, - ).squeeze() - if options["biomass_transport"]: # add biomass transport + transport_costs = pd.read_csv( + snakemake.input.biomass_transport_costs, index_col=0 + ) + transport_costs = transport_costs.squeeze() biomass_transport = create_network_topology( n, "biomass transport ", bidirectional=False ) @@ -2187,6 +2186,10 @@ def add_biomass(n, costs): elif options["biomass_spatial"]: # add artificial biomass generators at nodes which include transport costs + transport_costs = pd.read_csv( + snakemake.input.biomass_transport_costs, index_col=0 + ) + transport_costs = transport_costs.squeeze() bus_transport_costs = spatial.biomass.nodes.to_series().apply( lambda x: transport_costs[x[:2]] ) From 1bf6ea15323986bedeb89ade712fb5d256d7ba37 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 2 Aug 2023 14:31:08 +0200 Subject: [PATCH 4/5] only copy active configuration to results folder use copy config for all solve rules remove copying of environment --- rules/postprocess.smk | 18 +----------------- rules/solve_electricity.smk | 1 + rules/solve_myopic.smk | 2 +- rules/solve_overnight.smk | 4 +--- scripts/copy_config.py | 14 +------------- 5 files changed, 5 insertions(+), 34 deletions(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 105318f3..46f9a935 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -39,7 +39,7 @@ rule copy_config: params: RDIR=RDIR, output: - RESULTS + "config/config.yaml", + RESULTS + "config.yaml", threads: 1 resources: mem_mb=1000, @@ -51,22 +51,6 @@ rule copy_config: "../scripts/copy_config.py" -rule copy_conda_env: - output: - RESULTS + "config/environment.yaml", - threads: 1 - resources: - mem_mb=500, - log: - LOGS + "copy_conda_env.log", - benchmark: - BENCHMARKS + "copy_conda_env" - conda: - "../envs/environment.yaml" - shell: - "conda env export -f {output} --no-builds" - - rule make_summary: params: foresight=config["foresight"], diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index f73a5b0c..6943cc55 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -13,6 +13,7 @@ rule solve_network: ), input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + config=RESULTS + "config.yaml", output: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", log: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index ba2787a1..6e84e5f4 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -92,7 +92,7 @@ rule solve_sector_network_myopic: network=RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", costs="data/costs_{planning_horizons}.csv", - config=RESULTS + "config/config.yaml", + config=RESULTS + "config.yaml", output: RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index d0313f30..a01c1195 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -14,9 +14,7 @@ rule solve_sector_network: input: network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - costs="data/costs_{}.csv".format(config["costs"]["year"]), - config=RESULTS + "config/config.yaml", - #env=RDIR + 'config/environment.yaml', + config=RESULTS + "config.yaml", output: RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/scripts/copy_config.py b/scripts/copy_config.py index 79d2e32b..a549d893 100644 --- a/scripts/copy_config.py +++ b/scripts/copy_config.py @@ -11,25 +11,13 @@ from shutil import copy import yaml -files = { - "config/config.yaml": "config.yaml", - "Snakefile": "Snakefile", - "scripts/solve_network.py": "solve_network.py", - "scripts/prepare_sector_network.py": "prepare_sector_network.py", -} - if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake snakemake = mock_snakemake("copy_config") - basepath = Path(f"results/{snakemake.params.RDIR}config/") - - for f, name in files.items(): - copy(f, basepath / name) - - with open(basepath / "config.snakemake.yaml", "w") as yaml_file: + with open(snakemake.output[0], "w") as yaml_file: yaml.dump( snakemake.config, yaml_file, From 7fa577c3a802f412f213668598a8814a15ab48bc Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Thu, 3 Aug 2023 18:21:36 +0200 Subject: [PATCH 5/5] simplify network: increase mem requirement --- rules/build_electricity.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 6964a87d..41f457a0 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -371,7 +371,7 @@ rule simplify_network: BENCHMARKS + "simplify_network/elec_s{simpl}" threads: 1 resources: - mem_mb=4000, + mem_mb=12000, conda: "../envs/environment.yaml" script: