add logfiles to all rules

This commit is contained in:
Fabian Neumann 2023-03-08 19:22:00 +01:00
parent 67f54b98ed
commit bf895de7cc
8 changed files with 158 additions and 78 deletions

View File

@ -42,7 +42,7 @@ wildcard_constraints:
opts="[-+a-zA-Z0-9\.]*",
sector_opts="[-+a-zA-Z0-9\.\s]*",
include: "rules/common.smk"
include: "rules/collect.smk"
include: "rules/retrieve.smk"
include: "rules/build_electricity.smk"

View File

@ -11,10 +11,12 @@ rule build_population_layouts:
pop_layout_total=RESOURCES + "pop_layout_total.nc",
pop_layout_urban=RESOURCES + "pop_layout_urban.nc",
pop_layout_rural=RESOURCES + "pop_layout_rural.nc",
log:
LOGS + "build_population_layouts.log"
resources:
mem_mb=20000,
benchmark:
"benchmarks/build_population_layouts"
BENCHMARKS + "build_population_layouts"
threads: 8
conda: "../envs/environment.yaml"
script:
@ -30,10 +32,12 @@ rule build_clustered_population_layouts:
cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc",
output:
clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
log:
LOGS + "build_clustered_population_layouts_{simpl}_{clusters}.log"
resources:
mem_mb=10000,
benchmark:
"benchmarks/build_clustered_population_layouts/s{simpl}_{clusters}"
BENCHMARKS + "build_clustered_population_layouts/s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_clustered_population_layouts.py"
@ -50,8 +54,10 @@ rule build_simplified_population_layouts:
clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}.csv",
resources:
mem_mb=10000,
log:
LOGS + "build_simplified_population_layouts_{simpl}"
benchmark:
"benchmarks/build_clustered_population_layouts/s{simpl}"
BENCHMARKS + "build_simplified_population_layouts/s{simpl}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_clustered_population_layouts.py"
@ -66,6 +72,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
cleaned_gas_network=RESOURCES + "gas_network.csv",
resources:
mem_mb=4000,
log:
LOGS + "build_gas_network.log"
conda: "../envs/environment.yaml"
script:
"../scripts/build_gas_network.py"
@ -89,10 +97,13 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
+ "gas_input_locations_s{simpl}_{clusters}_simplified.csv",
resources:
mem_mb=2000,
log:
LOGS + "build_gas_input_locations_s{simpl}_{clusters}.log"
conda: "../envs/environment.yaml"
script:
"../scripts/build_gas_input_locations.py"
rule cluster_gas_network:
input:
cleaned_gas_network=RESOURCES + "gas_network.csv",
@ -104,6 +115,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
clustered_gas_network=RESOURCES + "gas_network_elec_s{simpl}_{clusters}.csv",
resources:
mem_mb=4000,
log:
LOGS + "cluster_gas_network_s{simpl}_{clusters}.log"
conda: "../envs/environment.yaml"
script:
"../scripts/cluster_gas_network.py"
@ -130,8 +143,10 @@ rule build_heat_demands:
resources:
mem_mb=20000,
threads: 8
log:
LOGS + "build_heat_demands_{scope}_{simpl}_{clusters}.loc"
benchmark:
"benchmarks/build_heat_demands/{scope}_s{simpl}_{clusters}"
BENCHMARKS + "build_heat_demands/{scope}_s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_heat_demand.py"
@ -148,8 +163,10 @@ rule build_temperature_profiles:
resources:
mem_mb=20000,
threads: 8
log:
LOGS + "build_temperature_profiles_{scope}_{simpl}_{clusters}.log"
benchmark:
"benchmarks/build_temperature_profiles/{scope}_s{simpl}_{clusters}"
BENCHMARKS + "build_temperature_profiles/{scope}_s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_temperature_profiles.py"
@ -172,8 +189,10 @@ rule build_cop_profiles:
cop_air_urban=RESOURCES + "cop_air_urban_elec_s{simpl}_{clusters}.nc",
resources:
mem_mb=20000,
log:
LOGS + "build_cop_profiles_s{simpl}_{clusters}.log"
benchmark:
"benchmarks/build_cop_profiles/s{simpl}_{clusters}"
BENCHMARKS + "build_cop_profiles/s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_cop_profiles.py"
@ -189,19 +208,15 @@ rule build_solar_thermal_profiles:
resources:
mem_mb=20000,
threads: 16
log:
LOGS + "build_solar_thermal_profiles_{scope}_s{simpl}_{clusters}.log"
benchmark:
"benchmarks/build_solar_thermal_profiles/{scope}_s{simpl}_{clusters}"
BENCHMARKS + "build_solar_thermal_profiles/{scope}_s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_solar_thermal_profiles.py"
def input_eurostat(w):
# 2016 includes BA, 2017 does not
report_year = config["energy"]["eurostat_report_year"]
return f"data/eurostat-energy_balances-june_{report_year}_edition"
rule build_energy_totals:
input:
nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
@ -217,8 +232,10 @@ rule build_energy_totals:
threads: 16
resources:
mem_mb=10000,
log:
LOGS + "build_energy_totals.log"
benchmark:
"benchmarks/build_energy_totals"
BENCHMARKS + "build_energy_totals"
conda: "../envs/environment.yaml"
script:
"../scripts/build_energy_totals.py"
@ -243,8 +260,10 @@ rule build_biomass_potentials:
threads: 1
resources:
mem_mb=1000,
log:
LOGS + "build_biomass_potentials_s{simpl}_{clusters}.log"
benchmark:
"benchmarks/build_biomass_potentials_s{simpl}_{clusters}"
BENCHMARKS + "build_biomass_potentials_s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_biomass_potentials.py"
@ -263,8 +282,10 @@ if config["sector"]["biomass_transport"]:
threads: 1
resources:
mem_mb=1000,
log:
LOGS + "build_biomass_transport_costs.log"
benchmark:
"benchmarks/build_biomass_transport_costs"
BENCHMARKS + "build_biomass_transport_costs"
conda: "../envs/environment.yaml"
script:
"../scripts/build_biomass_transport_costs.py"
@ -296,8 +317,10 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]:
threads: 1
resources:
mem_mb=4000,
log:
LOGS + "build_sequestration_potentials_s{simpl}_{clusters}.log"
benchmark:
"benchmarks/build_sequestration_potentials_s{simpl}_{clusters}"
BENCHMARKS + "build_sequestration_potentials_s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_sequestration_potentials.py"
@ -320,8 +343,10 @@ rule build_salt_cavern_potentials:
threads: 1
resources:
mem_mb=2000,
log:
LOGS + "build_salt_cavern_potentials_s{simpl}_{clusters}.log"
benchmark:
"benchmarks/build_salt_cavern_potentials_s{simpl}_{clusters}"
BENCHMARKS + "build_salt_cavern_potentials_s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_salt_cavern_potentials.py"
@ -335,8 +360,10 @@ rule build_ammonia_production:
threads: 1
resources:
mem_mb=1000,
log:
LOGS + "build_ammonia_production.log"
benchmark:
"benchmarks/build_ammonia_production"
BENCHMARKS + "build_ammonia_production"
conda: "../envs/environment.yaml"
script:
"../scripts/build_ammonia_production.py"
@ -351,8 +378,10 @@ rule build_industry_sector_ratios:
threads: 1
resources:
mem_mb=1000,
log:
LOGS + "build_industry_sector_ratios.log"
benchmark:
"benchmarks/build_industry_sector_ratios"
BENCHMARKS + "build_industry_sector_ratios"
conda: "../envs/environment.yaml"
script:
"../scripts/build_industry_sector_ratios.py"
@ -369,8 +398,10 @@ rule build_industrial_production_per_country:
threads: 8
resources:
mem_mb=1000,
log:
LOGS + "build_industrial_production_per_country.log"
benchmark:
"benchmarks/build_industrial_production_per_country"
BENCHMARKS + "build_industrial_production_per_country"
conda: "../envs/environment.yaml"
script:
"../scripts/build_industrial_production_per_country.py"
@ -386,8 +417,10 @@ rule build_industrial_production_per_country_tomorrow:
threads: 1
resources:
mem_mb=1000,
log:
LOGS + "build_industrial_production_per_country_tomorrow_{planning_horizons}.log"
benchmark:
"benchmarks/build_industrial_production_per_country_tomorrow_{planning_horizons}"
BENCHMARKS + "build_industrial_production_per_country_tomorrow_{planning_horizons}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_industrial_production_per_country_tomorrow.py"
@ -404,8 +437,10 @@ rule build_industrial_distribution_key:
threads: 1
resources:
mem_mb=1000,
log:
LOGS + "build_industrial_distribution_key_s{simpl}_{clusters}.log"
benchmark:
"benchmarks/build_industrial_distribution_key/s{simpl}_{clusters}"
BENCHMARKS + "build_industrial_distribution_key/s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_industrial_distribution_key.py"
@ -423,8 +458,10 @@ rule build_industrial_production_per_node:
threads: 1
resources:
mem_mb=1000,
log:
LOGS + "build_industrial_production_per_node_s{simpl}_{clusters}_{planning_horizons}.log"
benchmark:
"benchmarks/build_industrial_production_per_node/s{simpl}_{clusters}_{planning_horizons}"
BENCHMARKS + "build_industrial_production_per_node/s{simpl}_{clusters}_{planning_horizons}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_industrial_production_per_node.py"
@ -443,8 +480,10 @@ rule build_industrial_energy_demand_per_node:
threads: 1
resources:
mem_mb=1000,
log:
LOGS + "build_industrial_energy_demand_per_node_s{simpl}_{clusters}_{planning_horizons}.log"
benchmark:
"benchmarks/build_industrial_energy_demand_per_node/s{simpl}_{clusters}_{planning_horizons}"
BENCHMARKS + "build_industrial_energy_demand_per_node/s{simpl}_{clusters}_{planning_horizons}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_industrial_energy_demand_per_node.py"
@ -462,8 +501,10 @@ rule build_industrial_energy_demand_per_country_today:
threads: 8
resources:
mem_mb=1000,
log:
LOGS + "build_industrial_energy_demand_per_country_today.log"
benchmark:
"benchmarks/build_industrial_energy_demand_per_country_today"
BENCHMARKS + "build_industrial_energy_demand_per_country_today"
conda: "../envs/environment.yaml"
script:
"../scripts/build_industrial_energy_demand_per_country_today.py"
@ -481,8 +522,10 @@ rule build_industrial_energy_demand_per_node_today:
threads: 1
resources:
mem_mb=1000,
log:
LOGS + "build_industrial_energy_demand_per_node_today_s{simpl}_{clusters}.log"
benchmark:
"benchmarks/build_industrial_energy_demand_per_node_today/s{simpl}_{clusters}"
BENCHMARKS + "build_industrial_energy_demand_per_node_today/s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_industrial_energy_demand_per_node_today.py"
@ -507,8 +550,10 @@ if config["sector"]["retrofitting"]["retro_endogen"]:
floor_area=RESOURCES + "floor_area_elec_s{simpl}_{clusters}.csv",
resources:
mem_mb=1000,
log:
LOGS + "build_retro_cost_s{simpl}_{clusters}.log"
benchmark:
"benchmarks/build_retro_cost/s{simpl}_{clusters}"
BENCHMARKS + "build_retro_cost/s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"../scripts/build_retro_cost.py"
@ -530,6 +575,8 @@ rule build_population_weighted_energy_totals:
threads: 1
resources:
mem_mb=2000,
log:
LOGS + "build_population_weighted_energy_totals_s{simpl}_{clusters}.log"
conda: "../envs/environment.yaml"
script:
"../scripts/build_population_weighted_energy_totals.py"
@ -546,6 +593,8 @@ rule build_shipping_demand:
threads: 1
resources:
mem_mb=2000,
log:
LOGS + "build_shipping_demand_s{simpl}_{clusters}.log"
conda: "../envs/environment.yaml"
script:
"../scripts/build_shipping_demand.py"
@ -568,6 +617,8 @@ rule build_transport_demand:
threads: 1
resources:
mem_mb=2000,
log:
LOGS + "build_transport_demand_s{simpl}_{clusters}.log"
conda: "../envs/environment.yaml"
script:
"../scripts/build_transport_demand.py"
@ -641,10 +692,11 @@ rule prepare_sector_network:
threads: 1
resources:
mem_mb=2000,
log:
LOGS + "prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
benchmark:
(
RESULTS
+ "benchmarks/prepare_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
BENCHMARKS + "prepare_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda: "../envs/environment.yaml"
script:

40
rules/common.smk Normal file
View File

@ -0,0 +1,40 @@
# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
#
# SPDX-License-Identifier: MIT
def memory(w):
factor = 3.0
for o in w.opts.split("-"):
m = re.match(r"^(\d+)h$", o, re.IGNORECASE)
if m is not None:
factor /= int(m.group(1))
break
for o in w.opts.split("-"):
m = re.match(r"^(\d+)seg$", o, re.IGNORECASE)
if m is not None:
factor *= int(m.group(1)) / 8760
break
if w.clusters.endswith("m"):
return int(factor * (18000 + 180 * int(w.clusters[:-1])))
elif w.clusters == "all":
return int(factor * (18000 + 180 * 4000))
else:
return int(factor * (10000 + 195 * int(w.clusters)))
def input_eurostat(w):
# 2016 includes BA, 2017 does not
report_year = config["energy"]["eurostat_report_year"]
return f"data/eurostat-energy_balances-june_{report_year}_edition"
def solved_previous_horizon(wildcards):
planning_horizons = config["scenario"]["planning_horizons"]
i = planning_horizons.index(int(wildcards.planning_horizons))
planning_horizon_p = str(planning_horizons[i - 1])
return (
RESULTS
+ "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_"
+ planning_horizon_p
+ ".nc"
)

View File

@ -24,8 +24,7 @@ rule plot_network:
mem_mb=10000,
benchmark:
(
RESULTS
+ "benchmarks/plot_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
BENCHMARKS + "plot_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda:
"../envs/environment.yaml"
@ -42,7 +41,7 @@ rule copy_config:
resources:
mem_mb=1000,
benchmark:
RESULTS + "benchmarks/copy_config"
BENCHMARKS + "copy_config"
conda:
"../envs/environment.yaml"
script:
@ -55,8 +54,10 @@ rule copy_conda_env:
threads: 1
resources:
mem_mb=500,
log:
LOGS + "copy_conda_env.log"
benchmark:
RESULTS + "benchmarks/copy_conda_env"
BENCHMARKS + "copy_conda_env"
conda:
"../envs/environment.yaml"
shell:
@ -100,8 +101,10 @@ rule make_summary:
threads: 2
resources:
mem_mb=10000,
log:
LOGS + "make_summary.log"
benchmark:
RESULTS + "benchmarks/make_summary"
BENCHMARKS + "make_summary"
conda:
"../envs/environment.yaml"
script:
@ -124,8 +127,10 @@ rule plot_summary:
threads: 2
resources:
mem_mb=10000,
log:
LOGS + "plot_summary.log"
benchmark:
RESULTS + "benchmarks/plot_summary"
BENCHMARKS + "plot_summary"
conda:
"../envs/environment.yaml"
script:

View File

@ -26,6 +26,8 @@ if config["enable"].get("retrieve_databundle", True):
resources:
mem_mb=1000,
retries: 2
conda:
"../envs/environment.yaml"
script:
"../scripts/retrieve_databundle.py"
@ -63,7 +65,7 @@ if config["enable"].get("retrieve_cost_data", True):
output:
"data/costs_{year}.csv",
log:
"logs/retrieve_cost_data_{year}.log",
LOGS + "retrieve_cost_data_{year}.log",
resources:
mem_mb=1000,
retries: 2
@ -82,6 +84,8 @@ if config["enable"].get("retrieve_natura_raster", True):
),
output:
RESOURCES + "natura.tiff",
log:
LOGS + "retrieve_natura_raster.log"
resources:
mem_mb=5000,
retries: 2
@ -108,8 +112,10 @@ if config["enable"].get("retrieve_sector_databundle", True):
output:
*datafiles,
log:
"logs/retrieve_sector_databundle.log",
LOGS + "retrieve_sector_databundle.log",
retries: 2
conda:
"../envs/environment.yaml"
script:
"../scripts/retrieve_sector_databundle.py"
@ -125,7 +131,11 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
rule retrieve_gas_infrastructure_data:
output:
expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles),
log:
LOGS + "retrieve_gas_infrastructure_data.log",
retries: 2
conda:
"../envs/environment.yaml"
script:
"../scripts/retrieve_gas_infrastructure_data.py"
@ -139,6 +149,8 @@ rule retrieve_load_data:
),
output:
"data/load_raw.csv",
log:
LOGS + "retrieve_load_data.log",
resources:
mem_mb=5000,
retries: 2
@ -155,6 +167,8 @@ rule retrieve_ship_raster:
),
output:
"data/shipdensity_global.zip",
log:
LOGS + "retrieve_ship_raster.log"
resources:
mem_mb=5000,
retries: 2

View File

@ -3,26 +3,6 @@
# SPDX-License-Identifier: MIT
def memory(w):
factor = 3.0
for o in w.opts.split("-"):
m = re.match(r"^(\d+)h$", o, re.IGNORECASE)
if m is not None:
factor /= int(m.group(1))
break
for o in w.opts.split("-"):
m = re.match(r"^(\d+)seg$", o, re.IGNORECASE)
if m is not None:
factor *= int(m.group(1)) / 8760
break
if w.clusters.endswith("m"):
return int(factor * (18000 + 180 * int(w.clusters[:-1])))
elif w.clusters == "all":
return int(factor * (18000 + 180 * 4000))
else:
return int(factor * (10000 + 195 * int(w.clusters)))
rule solve_network:
input:
RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",

View File

@ -28,10 +28,11 @@ rule add_existing_baseyear:
threads: 1
resources:
mem_mb=2000,
log:
LOGS + "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
benchmark:
(
RESULTS
+ "benchmarks/add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
BENCHMARKS + "add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda:
"../envs/environment.yaml"
@ -39,18 +40,6 @@ rule add_existing_baseyear:
"../scripts/add_existing_baseyear.py"
def solved_previous_horizon(wildcards):
planning_horizons = config["scenario"]["planning_horizons"]
i = planning_horizons.index(int(wildcards.planning_horizons))
planning_horizon_p = str(planning_horizons[i - 1])
return (
RESULTS
+ "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_"
+ planning_horizon_p
+ ".nc"
)
rule add_brownfield:
input:
overrides="data/override_component_attrs",
@ -66,10 +55,11 @@ rule add_brownfield:
threads: 4
resources:
mem_mb=10000,
log:
LOGS + "add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
benchmark:
(
RESULTS
+ "benchmarks/add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
BENCHMARKS + "add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda:
"../envs/environment.yaml"
@ -104,8 +94,7 @@ rule solve_sector_network_myopic:
mem_mb=config["solving"]["mem"],
benchmark:
(
RESULTS
+ "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
BENCHMARKS + "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda:
"../envs/environment.yaml"

View File

@ -29,7 +29,7 @@ rule solve_sector_network:
benchmark:
(
RESULTS
+ "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+ BENCHMARKS + "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda:
"../envs/environment.yaml"