bjoern/pypsa-eur
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add conda directive to all rules for use with --use-conda

Browse Source
This commit is contained in:
Fabian Neumann 2023-03-08 17:44:25 +01:00
parent 6bae022d44
commit 8eb37bc263
9 changed files with 127 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Snakefile
View File

@ -9,6 +9,11 @@ from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider
HTTP = HTTPRemoteProvider()
from snakemake.utils import min_version
min_version("6.15")
if not exists("config.yaml"):
copyfile("config.default.yaml", "config.yaml")

48
rules/build_electricity.smk
View File

@ -12,8 +12,9 @@ if config["enable"].get("prepare_links_p_nom", False):
threads: 1
resources:
mem_mb=1500,
conda: "../envs/environment.yaml"
script:
"scripts/prepare_links_p_nom.py"
"../scripts/prepare_links_p_nom.py"
rule build_load_data:
@ -25,8 +26,9 @@ rule build_load_data:
LOGS + "build_load_data.log",
resources:
mem_mb=5000,
conda: "../envs/environment.yaml"
script:
"scripts/build_load_data.py"
"../scripts/build_load_data.py"
rule build_powerplants:
@ -40,8 +42,9 @@ rule build_powerplants:
threads: 1
resources:
mem_mb=5000,
conda: "../envs/environment.yaml"
script:
"scripts/build_powerplants.py"
"../scripts/build_powerplants.py"
rule base_network:
@ -66,8 +69,9 @@ rule base_network:
threads: 1
resources:
mem_mb=1500,
conda: "../envs/environment.yaml"
script:
"scripts/base_network.py"
"../scripts/base_network.py"
rule build_shapes:
@ -89,8 +93,9 @@ rule build_shapes:
threads: 1
resources:
mem_mb=1500,
conda: "../envs/environment.yaml"
script:
"scripts/build_shapes.py"
"../scripts/build_shapes.py"
rule build_bus_regions:
@ -106,8 +111,9 @@ rule build_bus_regions:
threads: 1
resources:
mem_mb=1000,
conda: "../envs/environment.yaml"
script:
"scripts/build_bus_regions.py"
"../scripts/build_bus_regions.py"
if config["enable"].get("build_cutout", False):
@ -125,8 +131,9 @@ if config["enable"].get("build_cutout", False):
threads: ATLITE_NPROCESSES
resources:
mem_mb=ATLITE_NPROCESSES * 1000,
conda: "../envs/environment.yaml"
script:
"scripts/build_cutout.py"
"../scripts/build_cutout.py"
if config["enable"].get("build_natura_raster", False):
@ -141,8 +148,9 @@ if config["enable"].get("build_natura_raster", False):
mem_mb=5000,
log:
LOGS + "build_natura_raster.log",
conda: "../envs/environment.yaml"
script:
"scripts/build_natura_raster.py"
"../scripts/build_natura_raster.py"
rule build_ship_raster:
@ -163,8 +171,9 @@ rule build_ship_raster:
mem_mb=5000,
benchmark:
BENCHMARKS + "build_ship_raster"
conda: "../envs/environment.yaml"
script:
"scripts/build_ship_raster.py"
"../scripts/build_ship_raster.py"
rule build_renewable_profiles:
@ -208,8 +217,9 @@ rule build_renewable_profiles:
mem_mb=ATLITE_NPROCESSES * 5000,
wildcard_constraints:
technology="(?!hydro).*", # Any technology other than hydro
conda: "../envs/environment.yaml"
script:
"scripts/build_renewable_profiles.py"
"../scripts/build_renewable_profiles.py"
rule build_hydro_profile:
@ -223,8 +233,9 @@ rule build_hydro_profile:
LOGS + "build_hydro_profile.log",
resources:
mem_mb=5000,
conda: "../envs/environment.yaml"
script:
"scripts/build_hydro_profile.py"
"../scripts/build_hydro_profile.py"
rule add_electricity:
@ -256,8 +267,9 @@ rule add_electricity:
threads: 1
resources:
mem_mb=5000,
conda: "../envs/environment.yaml"
script:
"scripts/add_electricity.py"
"../scripts/add_electricity.py"
rule simplify_network:
@ -279,8 +291,9 @@ rule simplify_network:
threads: 1
resources:
mem_mb=4000,
conda: "../envs/environment.yaml"
script:
"scripts/simplify_network.py"
"../scripts/simplify_network.py"
rule cluster_network:
@ -308,8 +321,9 @@ rule cluster_network:
threads: 1
resources:
mem_mb=6000,
conda: "../envs/environment.yaml"
script:
"scripts/cluster_network.py"
"../scripts/cluster_network.py"
rule add_extra_components:
@ -325,8 +339,9 @@ rule add_extra_components:
threads: 1
resources:
mem_mb=3000,
conda: "../envs/environment.yaml"
script:
"scripts/add_extra_components.py"
"../scripts/add_extra_components.py"
rule prepare_network:
@ -342,5 +357,6 @@ rule prepare_network:
threads: 1
resources:
mem_mb=4000,
conda: "../envs/environment.yaml"
script:
"scripts/prepare_network.py"
"../scripts/prepare_network.py"

87
rules/build_sector.smk
View File

@ -16,8 +16,9 @@ rule build_population_layouts:
benchmark:
"benchmarks/build_population_layouts"
threads: 8
conda: "../envs/environment.yaml"
script:
"scripts/build_population_layouts.py"
"../scripts/build_population_layouts.py"
rule build_clustered_population_layouts:
@ -33,8 +34,9 @@ rule build_clustered_population_layouts:
mem_mb=10000,
benchmark:
"benchmarks/build_clustered_population_layouts/s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"scripts/build_clustered_population_layouts.py"
"../scripts/build_clustered_population_layouts.py"
rule build_simplified_population_layouts:
@ -50,8 +52,9 @@ rule build_simplified_population_layouts:
mem_mb=10000,
benchmark:
"benchmarks/build_clustered_population_layouts/s{simpl}"
conda: "../envs/environment.yaml"
script:
"scripts/build_clustered_population_layouts.py"
"../scripts/build_clustered_population_layouts.py"
if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
@ -63,8 +66,9 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
cleaned_gas_network=RESOURCES + "gas_network.csv",
resources:
mem_mb=4000,
conda: "../envs/environment.yaml"
script:
"scripts/build_gas_network.py"
"../scripts/build_gas_network.py"
rule build_gas_input_locations:
input:
@ -85,8 +89,9 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
+ "gas_input_locations_s{simpl}_{clusters}_simplified.csv",
resources:
mem_mb=2000,
conda: "../envs/environment.yaml"
script:
"scripts/build_gas_input_locations.py"
"../scripts/build_gas_input_locations.py"
rule cluster_gas_network:
input:
@ -99,8 +104,9 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
clustered_gas_network=RESOURCES + "gas_network_elec_s{simpl}_{clusters}.csv",
resources:
mem_mb=4000,
conda: "../envs/environment.yaml"
script:
"scripts/cluster_gas_network.py"
"../scripts/cluster_gas_network.py"
gas_infrastructure = {
**rules.cluster_gas_network.output,
@ -126,8 +132,9 @@ rule build_heat_demands:
threads: 8
benchmark:
"benchmarks/build_heat_demands/{scope}_s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"scripts/build_heat_demand.py"
"../scripts/build_heat_demand.py"
rule build_temperature_profiles:
@ -143,8 +150,9 @@ rule build_temperature_profiles:
threads: 8
benchmark:
"benchmarks/build_temperature_profiles/{scope}_s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"scripts/build_temperature_profiles.py"
"../scripts/build_temperature_profiles.py"
rule build_cop_profiles:
@ -166,8 +174,9 @@ rule build_cop_profiles:
mem_mb=20000,
benchmark:
"benchmarks/build_cop_profiles/s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"scripts/build_cop_profiles.py"
"../scripts/build_cop_profiles.py"
rule build_solar_thermal_profiles:
@ -182,8 +191,9 @@ rule build_solar_thermal_profiles:
threads: 16
benchmark:
"benchmarks/build_solar_thermal_profiles/{scope}_s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"scripts/build_solar_thermal_profiles.py"
"../scripts/build_solar_thermal_profiles.py"
def input_eurostat(w):
@ -209,8 +219,9 @@ rule build_energy_totals:
mem_mb=10000,
benchmark:
"benchmarks/build_energy_totals"
conda: "../envs/environment.yaml"
script:
"scripts/build_energy_totals.py"
"../scripts/build_energy_totals.py"
rule build_biomass_potentials:
@ -234,8 +245,9 @@ rule build_biomass_potentials:
mem_mb=1000,
benchmark:
"benchmarks/build_biomass_potentials_s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"scripts/build_biomass_potentials.py"
"../scripts/build_biomass_potentials.py"
if config["sector"]["biomass_transport"]:
@ -253,8 +265,9 @@ if config["sector"]["biomass_transport"]:
mem_mb=1000,
benchmark:
"benchmarks/build_biomass_transport_costs"
conda: "../envs/environment.yaml"
script:
"scripts/build_biomass_transport_costs.py"
"../scripts/build_biomass_transport_costs.py"
build_biomass_transport_costs_output = rules.build_biomass_transport_costs.output
@ -285,8 +298,9 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]:
mem_mb=4000,
benchmark:
"benchmarks/build_sequestration_potentials_s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"scripts/build_sequestration_potentials.py"
"../scripts/build_sequestration_potentials.py"
build_sequestration_potentials_output = rules.build_sequestration_potentials.output
@ -308,8 +322,9 @@ rule build_salt_cavern_potentials:
mem_mb=2000,
benchmark:
"benchmarks/build_salt_cavern_potentials_s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"scripts/build_salt_cavern_potentials.py"
"../scripts/build_salt_cavern_potentials.py"
rule build_ammonia_production:
@ -322,8 +337,9 @@ rule build_ammonia_production:
mem_mb=1000,
benchmark:
"benchmarks/build_ammonia_production"
conda: "../envs/environment.yaml"
script:
"scripts/build_ammonia_production.py"
"../scripts/build_ammonia_production.py"
rule build_industry_sector_ratios:
@ -337,8 +353,9 @@ rule build_industry_sector_ratios:
mem_mb=1000,
benchmark:
"benchmarks/build_industry_sector_ratios"
conda: "../envs/environment.yaml"
script:
"scripts/build_industry_sector_ratios.py"
"../scripts/build_industry_sector_ratios.py"
rule build_industrial_production_per_country:
@ -354,8 +371,9 @@ rule build_industrial_production_per_country:
mem_mb=1000,
benchmark:
"benchmarks/build_industrial_production_per_country"
conda: "../envs/environment.yaml"
script:
"scripts/build_industrial_production_per_country.py"
"../scripts/build_industrial_production_per_country.py"
rule build_industrial_production_per_country_tomorrow:
@ -370,8 +388,9 @@ rule build_industrial_production_per_country_tomorrow:
mem_mb=1000,
benchmark:
"benchmarks/build_industrial_production_per_country_tomorrow_{planning_horizons}"
conda: "../envs/environment.yaml"
script:
"scripts/build_industrial_production_per_country_tomorrow.py"
"../scripts/build_industrial_production_per_country_tomorrow.py"
rule build_industrial_distribution_key:
@ -387,8 +406,9 @@ rule build_industrial_distribution_key:
mem_mb=1000,
benchmark:
"benchmarks/build_industrial_distribution_key/s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"scripts/build_industrial_distribution_key.py"
"../scripts/build_industrial_distribution_key.py"
rule build_industrial_production_per_node:
@ -405,8 +425,9 @@ rule build_industrial_production_per_node:
mem_mb=1000,
benchmark:
"benchmarks/build_industrial_production_per_node/s{simpl}_{clusters}_{planning_horizons}"
conda: "../envs/environment.yaml"
script:
"scripts/build_industrial_production_per_node.py"
"../scripts/build_industrial_production_per_node.py"
rule build_industrial_energy_demand_per_node:
@ -424,8 +445,9 @@ rule build_industrial_energy_demand_per_node:
mem_mb=1000,
benchmark:
"benchmarks/build_industrial_energy_demand_per_node/s{simpl}_{clusters}_{planning_horizons}"
conda: "../envs/environment.yaml"
script:
"scripts/build_industrial_energy_demand_per_node.py"
"../scripts/build_industrial_energy_demand_per_node.py"
rule build_industrial_energy_demand_per_country_today:
@ -442,8 +464,9 @@ rule build_industrial_energy_demand_per_country_today:
mem_mb=1000,
benchmark:
"benchmarks/build_industrial_energy_demand_per_country_today"
conda: "../envs/environment.yaml"
script:
"scripts/build_industrial_energy_demand_per_country_today.py"
"../scripts/build_industrial_energy_demand_per_country_today.py"
rule build_industrial_energy_demand_per_node_today:
@ -460,8 +483,9 @@ rule build_industrial_energy_demand_per_node_today:
mem_mb=1000,
benchmark:
"benchmarks/build_industrial_energy_demand_per_node_today/s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"scripts/build_industrial_energy_demand_per_node_today.py"
"../scripts/build_industrial_energy_demand_per_node_today.py"
if config["sector"]["retrofitting"]["retro_endogen"]:
@ -485,8 +509,9 @@ if config["sector"]["retrofitting"]["retro_endogen"]:
mem_mb=1000,
benchmark:
"benchmarks/build_retro_cost/s{simpl}_{clusters}"
conda: "../envs/environment.yaml"
script:
"scripts/build_retro_cost.py"
"../scripts/build_retro_cost.py"
build_retro_cost_output = rules.build_retro_cost.output
@ -505,8 +530,9 @@ rule build_population_weighted_energy_totals:
threads: 1
resources:
mem_mb=2000,
conda: "../envs/environment.yaml"
script:
"scripts/build_population_weighted_energy_totals.py"
"../scripts/build_population_weighted_energy_totals.py"
rule build_shipping_demand:
@ -520,8 +546,9 @@ rule build_shipping_demand:
threads: 1
resources:
mem_mb=2000,
conda: "../envs/environment.yaml"
script:
"scripts/build_shipping_demand.py"
"../scripts/build_shipping_demand.py"
rule build_transport_demand:
@ -541,8 +568,9 @@ rule build_transport_demand:
threads: 1
resources:
mem_mb=2000,
conda: "../envs/environment.yaml"
script:
"scripts/build_transport_demand.py"
"../scripts/build_transport_demand.py"
rule prepare_sector_network:
@ -618,5 +646,6 @@ rule prepare_sector_network:
RESULTS
+ "benchmarks/prepare_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda: "../envs/environment.yaml"
script:
"scripts/prepare_sector_network.py"
"../scripts/prepare_sector_network.py"

12
rules/collect.smk
View File

@ -4,6 +4,13 @@
localrules: all, cluster_all_networks, extra_components_all_networks, prepare_all_networks, prepare_sector_networks, solve_all_elec_networks, solve_all_networks, plot_all_networks
rule all:
input:
RESULTS + "graphs/costs.pdf",
default_target: True
rule cluster_all_networks:
input:
expand(RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", **config["scenario"]),
@ -33,11 +40,6 @@ rule prepare_sector_networks:
),
rule all:
input:
RESULTS + "graphs/costs.pdf",
rule solve_all_elec_networks:
input:
expand(

13
rules/postprocess.smk
View File

@ -23,8 +23,9 @@ rule plot_network:
RESULTS
+ "benchmarks/plot_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda: "../envs/environment.yaml"
script:
"scripts/plot_network.py"
"../scripts/plot_network.py"
rule copy_config:
@ -37,8 +38,9 @@ rule copy_config:
mem_mb=1000,
benchmark:
RESULTS + "benchmarks/copy_config"
conda: "../envs/environment.yaml"
script:
"scripts/copy_config.py"
"../scripts/copy_config.py"
rule copy_conda_env:
@ -49,6 +51,7 @@ rule copy_conda_env:
mem_mb=500,
benchmark:
RESULTS + "benchmarks/copy_conda_env"
conda: "../envs/environment.yaml"
shell:
"conda env export -f {output} --no-builds"
@ -92,8 +95,9 @@ rule make_summary:
mem_mb=10000,
benchmark:
RESULTS + "benchmarks/make_summary"
conda: "../envs/environment.yaml"
script:
"scripts/make_summary.py"
"../scripts/make_summary.py"
rule plot_summary:
@ -114,5 +118,6 @@ rule plot_summary:
mem_mb=10000,
benchmark:
RESULTS + "benchmarks/plot_summary"
conda: "../envs/environment.yaml"
script:
"scripts/plot_summary.py"
"../scripts/plot_summary.py"

6
rules/retrieve.smk
View File

@ -28,7 +28,7 @@ if config["enable"].get("retrieve_databundle", True):
resources:
mem_mb=1000,
script:
"scripts/retrieve_databundle.py"
"../scripts/retrieve_databundle.py"
if config["enable"].get("retrieve_cutout", True):
@ -110,7 +110,7 @@ if config["enable"].get("retrieve_sector_databundle", True):
log:
"logs/retrieve_sector_databundle.log",
script:
"scripts/retrieve_sector_databundle.py"
"../scripts/retrieve_sector_databundle.py"
if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
@ -126,7 +126,7 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
output:
expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles),
script:
"scripts/retrieve_gas_infrastructure_data.py"
"../scripts/retrieve_gas_infrastructure_data.py"
rule retrieve_load_data:

6
rules/solve_electricity.smk
View File

@ -42,8 +42,9 @@ rule solve_network:
mem_mb=memory,
shadow:
"minimal"
conda: "../envs/environment.yaml"
script:
"scripts/solve_network.py"
"../scripts/solve_network.py"
rule solve_operations_network:
@ -71,5 +72,6 @@ rule solve_operations_network:
mem_mb=(lambda w: 5000 + 372 * int(w.clusters)),
shadow:
"minimal"
conda: "../envs/environment.yaml"
script:
"scripts/solve_operations_network.py"
"../scripts/solve_operations_network.py"

7
rules/solve_myopic.smk
View File

@ -32,9 +32,11 @@ rule add_existing_baseyear:
RESULTS
+ "benchmarks/add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda: "../envs/environment.yaml"
script:
"scripts/add_existing_baseyear.py"
def solved_previous_horizon(wildcards):
planning_horizons = config["scenario"]["planning_horizons"]
i = planning_horizons.index(int(wildcards.planning_horizons))
@ -46,6 +48,7 @@ def solved_previous_horizon(wildcards):
+ ".nc"
)
rule add_brownfield:
input:
overrides="data/override_component_attrs",
@ -66,11 +69,14 @@ rule add_brownfield:
RESULTS
+ "benchmarks/add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda: "../envs/environment.yaml"
script:
"scripts/add_brownfield.py"
ruleorder: add_existing_baseyear > add_brownfield
rule solve_sector_network_myopic:
input:
overrides="data/override_component_attrs",
@ -98,5 +104,6 @@ rule solve_sector_network_myopic:
RESULTS
+ "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda: "../envs/environment.yaml"
script:
"scripts/solve_sector_network.py"

3
rules/solve_overnight.smk
View File

@ -30,5 +30,6 @@ rule solve_sector_network:
RESULTS
+ "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda: "../envs/environment.yaml"
script:
"scripts/solve_sector_network.py"
"../scripts/solve_sector_network.py"