diff --git a/Snakefile b/Snakefile index 17823ac5..ee8979fb 100644 --- a/Snakefile +++ b/Snakefile @@ -42,7 +42,7 @@ wildcard_constraints: opts="[-+a-zA-Z0-9\.]*", sector_opts="[-+a-zA-Z0-9\.\s]*", - +include: "rules/common.smk" include: "rules/collect.smk" include: "rules/retrieve.smk" include: "rules/build_electricity.smk" diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 37cb29ab..7d4cfdb8 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -11,10 +11,12 @@ rule build_population_layouts: pop_layout_total=RESOURCES + "pop_layout_total.nc", pop_layout_urban=RESOURCES + "pop_layout_urban.nc", pop_layout_rural=RESOURCES + "pop_layout_rural.nc", + log: + LOGS + "build_population_layouts.log" resources: mem_mb=20000, benchmark: - "benchmarks/build_population_layouts" + BENCHMARKS + "build_population_layouts" threads: 8 conda: "../envs/environment.yaml" script: @@ -30,10 +32,12 @@ rule build_clustered_population_layouts: cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", output: clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", + log: + LOGS + "build_clustered_population_layouts_{simpl}_{clusters}.log" resources: mem_mb=10000, benchmark: - "benchmarks/build_clustered_population_layouts/s{simpl}_{clusters}" + BENCHMARKS + "build_clustered_population_layouts/s{simpl}_{clusters}" conda: "../envs/environment.yaml" script: "../scripts/build_clustered_population_layouts.py" @@ -50,8 +54,10 @@ rule build_simplified_population_layouts: clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}.csv", resources: mem_mb=10000, + log: + LOGS + "build_simplified_population_layouts_{simpl}" benchmark: - "benchmarks/build_clustered_population_layouts/s{simpl}" + BENCHMARKS + "build_simplified_population_layouts/s{simpl}" conda: "../envs/environment.yaml" script: "../scripts/build_clustered_population_layouts.py" @@ -66,6 +72,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: cleaned_gas_network=RESOURCES + "gas_network.csv", resources: mem_mb=4000, + log: + LOGS + "build_gas_network.log" conda: "../envs/environment.yaml" script: "../scripts/build_gas_network.py" @@ -89,10 +97,13 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: + "gas_input_locations_s{simpl}_{clusters}_simplified.csv", resources: mem_mb=2000, + log: + LOGS + "build_gas_input_locations_s{simpl}_{clusters}.log" conda: "../envs/environment.yaml" script: "../scripts/build_gas_input_locations.py" + rule cluster_gas_network: input: cleaned_gas_network=RESOURCES + "gas_network.csv", @@ -104,6 +115,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: clustered_gas_network=RESOURCES + "gas_network_elec_s{simpl}_{clusters}.csv", resources: mem_mb=4000, + log: + LOGS + "cluster_gas_network_s{simpl}_{clusters}.log" conda: "../envs/environment.yaml" script: "../scripts/cluster_gas_network.py" @@ -130,8 +143,10 @@ rule build_heat_demands: resources: mem_mb=20000, threads: 8 + log: + LOGS + "build_heat_demands_{scope}_{simpl}_{clusters}.loc" benchmark: - "benchmarks/build_heat_demands/{scope}_s{simpl}_{clusters}" + BENCHMARKS + "build_heat_demands/{scope}_s{simpl}_{clusters}" conda: "../envs/environment.yaml" script: "../scripts/build_heat_demand.py" @@ -148,8 +163,10 @@ rule build_temperature_profiles: resources: mem_mb=20000, threads: 8 + log: + LOGS + "build_temperature_profiles_{scope}_{simpl}_{clusters}.log" benchmark: - "benchmarks/build_temperature_profiles/{scope}_s{simpl}_{clusters}" + BENCHMARKS + "build_temperature_profiles/{scope}_s{simpl}_{clusters}" conda: "../envs/environment.yaml" script: "../scripts/build_temperature_profiles.py" @@ -172,8 +189,10 @@ rule build_cop_profiles: cop_air_urban=RESOURCES + "cop_air_urban_elec_s{simpl}_{clusters}.nc", resources: mem_mb=20000, + log: + LOGS + "build_cop_profiles_s{simpl}_{clusters}.log" benchmark: - "benchmarks/build_cop_profiles/s{simpl}_{clusters}" + BENCHMARKS + "build_cop_profiles/s{simpl}_{clusters}" conda: "../envs/environment.yaml" script: "../scripts/build_cop_profiles.py" @@ -189,19 +208,15 @@ rule build_solar_thermal_profiles: resources: mem_mb=20000, threads: 16 + log: + LOGS + "build_solar_thermal_profiles_{scope}_s{simpl}_{clusters}.log" benchmark: - "benchmarks/build_solar_thermal_profiles/{scope}_s{simpl}_{clusters}" + BENCHMARKS + "build_solar_thermal_profiles/{scope}_s{simpl}_{clusters}" conda: "../envs/environment.yaml" script: "../scripts/build_solar_thermal_profiles.py" -def input_eurostat(w): - # 2016 includes BA, 2017 does not - report_year = config["energy"]["eurostat_report_year"] - return f"data/eurostat-energy_balances-june_{report_year}_edition" - - rule build_energy_totals: input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", @@ -217,8 +232,10 @@ rule build_energy_totals: threads: 16 resources: mem_mb=10000, + log: + LOGS + "build_energy_totals.log" benchmark: - "benchmarks/build_energy_totals" + BENCHMARKS + "build_energy_totals" conda: "../envs/environment.yaml" script: "../scripts/build_energy_totals.py" @@ -243,8 +260,10 @@ rule build_biomass_potentials: threads: 1 resources: mem_mb=1000, + log: + LOGS + "build_biomass_potentials_s{simpl}_{clusters}.log" benchmark: - "benchmarks/build_biomass_potentials_s{simpl}_{clusters}" + BENCHMARKS + "build_biomass_potentials_s{simpl}_{clusters}" conda: "../envs/environment.yaml" script: "../scripts/build_biomass_potentials.py" @@ -263,8 +282,10 @@ if config["sector"]["biomass_transport"]: threads: 1 resources: mem_mb=1000, + log: + LOGS + "build_biomass_transport_costs.log" benchmark: - "benchmarks/build_biomass_transport_costs" + BENCHMARKS + "build_biomass_transport_costs" conda: "../envs/environment.yaml" script: "../scripts/build_biomass_transport_costs.py" @@ -296,8 +317,10 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: threads: 1 resources: mem_mb=4000, + log: + LOGS + "build_sequestration_potentials_s{simpl}_{clusters}.log" benchmark: - "benchmarks/build_sequestration_potentials_s{simpl}_{clusters}" + BENCHMARKS + "build_sequestration_potentials_s{simpl}_{clusters}" conda: "../envs/environment.yaml" script: "../scripts/build_sequestration_potentials.py" @@ -320,8 +343,10 @@ rule build_salt_cavern_potentials: threads: 1 resources: mem_mb=2000, + log: + LOGS + "build_salt_cavern_potentials_s{simpl}_{clusters}.log" benchmark: - "benchmarks/build_salt_cavern_potentials_s{simpl}_{clusters}" + BENCHMARKS + "build_salt_cavern_potentials_s{simpl}_{clusters}" conda: "../envs/environment.yaml" script: "../scripts/build_salt_cavern_potentials.py" @@ -335,8 +360,10 @@ rule build_ammonia_production: threads: 1 resources: mem_mb=1000, + log: + LOGS + "build_ammonia_production.log" benchmark: - "benchmarks/build_ammonia_production" + BENCHMARKS + "build_ammonia_production" conda: "../envs/environment.yaml" script: "../scripts/build_ammonia_production.py" @@ -351,8 +378,10 @@ rule build_industry_sector_ratios: threads: 1 resources: mem_mb=1000, + log: + LOGS + "build_industry_sector_ratios.log" benchmark: - "benchmarks/build_industry_sector_ratios" + BENCHMARKS + "build_industry_sector_ratios" conda: "../envs/environment.yaml" script: "../scripts/build_industry_sector_ratios.py" @@ -369,8 +398,10 @@ rule build_industrial_production_per_country: threads: 8 resources: mem_mb=1000, + log: + LOGS + "build_industrial_production_per_country.log" benchmark: - "benchmarks/build_industrial_production_per_country" + BENCHMARKS + "build_industrial_production_per_country" conda: "../envs/environment.yaml" script: "../scripts/build_industrial_production_per_country.py" @@ -386,8 +417,10 @@ rule build_industrial_production_per_country_tomorrow: threads: 1 resources: mem_mb=1000, + log: + LOGS + "build_industrial_production_per_country_tomorrow_{planning_horizons}.log" benchmark: - "benchmarks/build_industrial_production_per_country_tomorrow_{planning_horizons}" + BENCHMARKS + "build_industrial_production_per_country_tomorrow_{planning_horizons}" conda: "../envs/environment.yaml" script: "../scripts/build_industrial_production_per_country_tomorrow.py" @@ -404,8 +437,10 @@ rule build_industrial_distribution_key: threads: 1 resources: mem_mb=1000, + log: + LOGS + "build_industrial_distribution_key_s{simpl}_{clusters}.log" benchmark: - "benchmarks/build_industrial_distribution_key/s{simpl}_{clusters}" + BENCHMARKS + "build_industrial_distribution_key/s{simpl}_{clusters}" conda: "../envs/environment.yaml" script: "../scripts/build_industrial_distribution_key.py" @@ -423,8 +458,10 @@ rule build_industrial_production_per_node: threads: 1 resources: mem_mb=1000, + log: + LOGS + "build_industrial_production_per_node_s{simpl}_{clusters}_{planning_horizons}.log" benchmark: - "benchmarks/build_industrial_production_per_node/s{simpl}_{clusters}_{planning_horizons}" + BENCHMARKS + "build_industrial_production_per_node/s{simpl}_{clusters}_{planning_horizons}" conda: "../envs/environment.yaml" script: "../scripts/build_industrial_production_per_node.py" @@ -443,8 +480,10 @@ rule build_industrial_energy_demand_per_node: threads: 1 resources: mem_mb=1000, + log: + LOGS + "build_industrial_energy_demand_per_node_s{simpl}_{clusters}_{planning_horizons}.log" benchmark: - "benchmarks/build_industrial_energy_demand_per_node/s{simpl}_{clusters}_{planning_horizons}" + BENCHMARKS + "build_industrial_energy_demand_per_node/s{simpl}_{clusters}_{planning_horizons}" conda: "../envs/environment.yaml" script: "../scripts/build_industrial_energy_demand_per_node.py" @@ -462,8 +501,10 @@ rule build_industrial_energy_demand_per_country_today: threads: 8 resources: mem_mb=1000, + log: + LOGS + "build_industrial_energy_demand_per_country_today.log" benchmark: - "benchmarks/build_industrial_energy_demand_per_country_today" + BENCHMARKS + "build_industrial_energy_demand_per_country_today" conda: "../envs/environment.yaml" script: "../scripts/build_industrial_energy_demand_per_country_today.py" @@ -481,8 +522,10 @@ rule build_industrial_energy_demand_per_node_today: threads: 1 resources: mem_mb=1000, + log: + LOGS + "build_industrial_energy_demand_per_node_today_s{simpl}_{clusters}.log" benchmark: - "benchmarks/build_industrial_energy_demand_per_node_today/s{simpl}_{clusters}" + BENCHMARKS + "build_industrial_energy_demand_per_node_today/s{simpl}_{clusters}" conda: "../envs/environment.yaml" script: "../scripts/build_industrial_energy_demand_per_node_today.py" @@ -507,8 +550,10 @@ if config["sector"]["retrofitting"]["retro_endogen"]: floor_area=RESOURCES + "floor_area_elec_s{simpl}_{clusters}.csv", resources: mem_mb=1000, + log: + LOGS + "build_retro_cost_s{simpl}_{clusters}.log" benchmark: - "benchmarks/build_retro_cost/s{simpl}_{clusters}" + BENCHMARKS + "build_retro_cost/s{simpl}_{clusters}" conda: "../envs/environment.yaml" script: "../scripts/build_retro_cost.py" @@ -530,6 +575,8 @@ rule build_population_weighted_energy_totals: threads: 1 resources: mem_mb=2000, + log: + LOGS + "build_population_weighted_energy_totals_s{simpl}_{clusters}.log" conda: "../envs/environment.yaml" script: "../scripts/build_population_weighted_energy_totals.py" @@ -546,6 +593,8 @@ rule build_shipping_demand: threads: 1 resources: mem_mb=2000, + log: + LOGS + "build_shipping_demand_s{simpl}_{clusters}.log" conda: "../envs/environment.yaml" script: "../scripts/build_shipping_demand.py" @@ -568,6 +617,8 @@ rule build_transport_demand: threads: 1 resources: mem_mb=2000, + log: + LOGS + "build_transport_demand_s{simpl}_{clusters}.log" conda: "../envs/environment.yaml" script: "../scripts/build_transport_demand.py" @@ -641,10 +692,11 @@ rule prepare_sector_network: threads: 1 resources: mem_mb=2000, + log: + LOGS + "prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" benchmark: ( - RESULTS - + "benchmarks/prepare_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + BENCHMARKS + "prepare_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" script: diff --git a/rules/common.smk b/rules/common.smk new file mode 100644 index 00000000..552556a9 --- /dev/null +++ b/rules/common.smk @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +def memory(w): + factor = 3.0 + for o in w.opts.split("-"): + m = re.match(r"^(\d+)h$", o, re.IGNORECASE) + if m is not None: + factor /= int(m.group(1)) + break + for o in w.opts.split("-"): + m = re.match(r"^(\d+)seg$", o, re.IGNORECASE) + if m is not None: + factor *= int(m.group(1)) / 8760 + break + if w.clusters.endswith("m"): + return int(factor * (18000 + 180 * int(w.clusters[:-1]))) + elif w.clusters == "all": + return int(factor * (18000 + 180 * 4000)) + else: + return int(factor * (10000 + 195 * int(w.clusters))) + + +def input_eurostat(w): + # 2016 includes BA, 2017 does not + report_year = config["energy"]["eurostat_report_year"] + return f"data/eurostat-energy_balances-june_{report_year}_edition" + + +def solved_previous_horizon(wildcards): + planning_horizons = config["scenario"]["planning_horizons"] + i = planning_horizons.index(int(wildcards.planning_horizons)) + planning_horizon_p = str(planning_horizons[i - 1]) + return ( + RESULTS + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" + + planning_horizon_p + + ".nc" + ) \ No newline at end of file diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 52647029..a061728c 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -24,8 +24,7 @@ rule plot_network: mem_mb=10000, benchmark: ( - RESULTS - + "benchmarks/plot_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + BENCHMARKS + "plot_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -42,7 +41,7 @@ rule copy_config: resources: mem_mb=1000, benchmark: - RESULTS + "benchmarks/copy_config" + BENCHMARKS + "copy_config" conda: "../envs/environment.yaml" script: @@ -55,8 +54,10 @@ rule copy_conda_env: threads: 1 resources: mem_mb=500, + log: + LOGS + "copy_conda_env.log" benchmark: - RESULTS + "benchmarks/copy_conda_env" + BENCHMARKS + "copy_conda_env" conda: "../envs/environment.yaml" shell: @@ -100,8 +101,10 @@ rule make_summary: threads: 2 resources: mem_mb=10000, + log: + LOGS + "make_summary.log" benchmark: - RESULTS + "benchmarks/make_summary" + BENCHMARKS + "make_summary" conda: "../envs/environment.yaml" script: @@ -124,8 +127,10 @@ rule plot_summary: threads: 2 resources: mem_mb=10000, + log: + LOGS + "plot_summary.log" benchmark: - RESULTS + "benchmarks/plot_summary" + BENCHMARKS + "plot_summary" conda: "../envs/environment.yaml" script: diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 3fa85b10..a6f525e6 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -26,6 +26,8 @@ if config["enable"].get("retrieve_databundle", True): resources: mem_mb=1000, retries: 2 + conda: + "../envs/environment.yaml" script: "../scripts/retrieve_databundle.py" @@ -63,7 +65,7 @@ if config["enable"].get("retrieve_cost_data", True): output: "data/costs_{year}.csv", log: - "logs/retrieve_cost_data_{year}.log", + LOGS + "retrieve_cost_data_{year}.log", resources: mem_mb=1000, retries: 2 @@ -82,6 +84,8 @@ if config["enable"].get("retrieve_natura_raster", True): ), output: RESOURCES + "natura.tiff", + log: + LOGS + "retrieve_natura_raster.log" resources: mem_mb=5000, retries: 2 @@ -108,8 +112,10 @@ if config["enable"].get("retrieve_sector_databundle", True): output: *datafiles, log: - "logs/retrieve_sector_databundle.log", + LOGS + "retrieve_sector_databundle.log", retries: 2 + conda: + "../envs/environment.yaml" script: "../scripts/retrieve_sector_databundle.py" @@ -125,7 +131,11 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule retrieve_gas_infrastructure_data: output: expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles), + log: + LOGS + "retrieve_gas_infrastructure_data.log", retries: 2 + conda: + "../envs/environment.yaml" script: "../scripts/retrieve_gas_infrastructure_data.py" @@ -139,6 +149,8 @@ rule retrieve_load_data: ), output: "data/load_raw.csv", + log: + LOGS + "retrieve_load_data.log", resources: mem_mb=5000, retries: 2 @@ -155,6 +167,8 @@ rule retrieve_ship_raster: ), output: "data/shipdensity_global.zip", + log: + LOGS + "retrieve_ship_raster.log" resources: mem_mb=5000, retries: 2 diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index e32cfb8d..bda5a0a4 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -3,26 +3,6 @@ # SPDX-License-Identifier: MIT -def memory(w): - factor = 3.0 - for o in w.opts.split("-"): - m = re.match(r"^(\d+)h$", o, re.IGNORECASE) - if m is not None: - factor /= int(m.group(1)) - break - for o in w.opts.split("-"): - m = re.match(r"^(\d+)seg$", o, re.IGNORECASE) - if m is not None: - factor *= int(m.group(1)) / 8760 - break - if w.clusters.endswith("m"): - return int(factor * (18000 + 180 * int(w.clusters[:-1]))) - elif w.clusters == "all": - return int(factor * (18000 + 180 * 4000)) - else: - return int(factor * (10000 + 195 * int(w.clusters))) - - rule solve_network: input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index ab4937bc..a0dafee1 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -28,10 +28,11 @@ rule add_existing_baseyear: threads: 1 resources: mem_mb=2000, + log: + LOGS + "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" benchmark: ( - RESULTS - + "benchmarks/add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + BENCHMARKS + "add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -39,18 +40,6 @@ rule add_existing_baseyear: "../scripts/add_existing_baseyear.py" -def solved_previous_horizon(wildcards): - planning_horizons = config["scenario"]["planning_horizons"] - i = planning_horizons.index(int(wildcards.planning_horizons)) - planning_horizon_p = str(planning_horizons[i - 1]) - return ( - RESULTS - + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" - + planning_horizon_p - + ".nc" - ) - - rule add_brownfield: input: overrides="data/override_component_attrs", @@ -66,10 +55,11 @@ rule add_brownfield: threads: 4 resources: mem_mb=10000, + log: + LOGS + "add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" benchmark: ( - RESULTS - + "benchmarks/add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + BENCHMARKS + "add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -104,8 +94,7 @@ rule solve_sector_network_myopic: mem_mb=config["solving"]["mem"], benchmark: ( - RESULTS - + "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + BENCHMARKS + "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index c37e09dd..89d4f69e 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -29,7 +29,7 @@ rule solve_sector_network: benchmark: ( RESULTS - + "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + + BENCHMARKS + "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml"