diff --git a/Snakefile b/Snakefile index 3f959bb9..66c504eb 100644 --- a/Snakefile +++ b/Snakefile @@ -9,6 +9,11 @@ from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider HTTP = HTTPRemoteProvider() +from snakemake.utils import min_version + +min_version("6.15") + + if not exists("config.yaml"): copyfile("config.default.yaml", "config.yaml") diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index dfb16ddf..5cefc2fc 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -12,8 +12,9 @@ if config["enable"].get("prepare_links_p_nom", False): threads: 1 resources: mem_mb=1500, + conda: "../envs/environment.yaml" script: - "scripts/prepare_links_p_nom.py" + "../scripts/prepare_links_p_nom.py" rule build_load_data: @@ -25,8 +26,9 @@ rule build_load_data: LOGS + "build_load_data.log", resources: mem_mb=5000, + conda: "../envs/environment.yaml" script: - "scripts/build_load_data.py" + "../scripts/build_load_data.py" rule build_powerplants: @@ -40,8 +42,9 @@ rule build_powerplants: threads: 1 resources: mem_mb=5000, + conda: "../envs/environment.yaml" script: - "scripts/build_powerplants.py" + "../scripts/build_powerplants.py" rule base_network: @@ -66,8 +69,9 @@ rule base_network: threads: 1 resources: mem_mb=1500, + conda: "../envs/environment.yaml" script: - "scripts/base_network.py" + "../scripts/base_network.py" rule build_shapes: @@ -89,8 +93,9 @@ rule build_shapes: threads: 1 resources: mem_mb=1500, + conda: "../envs/environment.yaml" script: - "scripts/build_shapes.py" + "../scripts/build_shapes.py" rule build_bus_regions: @@ -106,8 +111,9 @@ rule build_bus_regions: threads: 1 resources: mem_mb=1000, + conda: "../envs/environment.yaml" script: - "scripts/build_bus_regions.py" + "../scripts/build_bus_regions.py" if config["enable"].get("build_cutout", False): @@ -125,8 +131,9 @@ if config["enable"].get("build_cutout", False): threads: ATLITE_NPROCESSES resources: mem_mb=ATLITE_NPROCESSES * 1000, + conda: "../envs/environment.yaml" script: - "scripts/build_cutout.py" + "../scripts/build_cutout.py" if config["enable"].get("build_natura_raster", False): @@ -141,8 +148,9 @@ if config["enable"].get("build_natura_raster", False): mem_mb=5000, log: LOGS + "build_natura_raster.log", + conda: "../envs/environment.yaml" script: - "scripts/build_natura_raster.py" + "../scripts/build_natura_raster.py" rule build_ship_raster: @@ -163,8 +171,9 @@ rule build_ship_raster: mem_mb=5000, benchmark: BENCHMARKS + "build_ship_raster" + conda: "../envs/environment.yaml" script: - "scripts/build_ship_raster.py" + "../scripts/build_ship_raster.py" rule build_renewable_profiles: @@ -208,8 +217,9 @@ rule build_renewable_profiles: mem_mb=ATLITE_NPROCESSES * 5000, wildcard_constraints: technology="(?!hydro).*", # Any technology other than hydro + conda: "../envs/environment.yaml" script: - "scripts/build_renewable_profiles.py" + "../scripts/build_renewable_profiles.py" rule build_hydro_profile: @@ -223,8 +233,9 @@ rule build_hydro_profile: LOGS + "build_hydro_profile.log", resources: mem_mb=5000, + conda: "../envs/environment.yaml" script: - "scripts/build_hydro_profile.py" + "../scripts/build_hydro_profile.py" rule add_electricity: @@ -256,8 +267,9 @@ rule add_electricity: threads: 1 resources: mem_mb=5000, + conda: "../envs/environment.yaml" script: - "scripts/add_electricity.py" + "../scripts/add_electricity.py" rule simplify_network: @@ -279,8 +291,9 @@ rule simplify_network: threads: 1 resources: mem_mb=4000, + conda: "../envs/environment.yaml" script: - "scripts/simplify_network.py" + "../scripts/simplify_network.py" rule cluster_network: @@ -308,8 +321,9 @@ rule cluster_network: threads: 1 resources: mem_mb=6000, + conda: "../envs/environment.yaml" script: - "scripts/cluster_network.py" + "../scripts/cluster_network.py" rule add_extra_components: @@ -325,8 +339,9 @@ rule add_extra_components: threads: 1 resources: mem_mb=3000, + conda: "../envs/environment.yaml" script: - "scripts/add_extra_components.py" + "../scripts/add_extra_components.py" rule prepare_network: @@ -342,5 +357,6 @@ rule prepare_network: threads: 1 resources: mem_mb=4000, + conda: "../envs/environment.yaml" script: - "scripts/prepare_network.py" \ No newline at end of file + "../scripts/prepare_network.py" \ No newline at end of file diff --git a/rules/build_sector.smk b/rules/build_sector.smk index e4272171..30a083e5 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -16,8 +16,9 @@ rule build_population_layouts: benchmark: "benchmarks/build_population_layouts" threads: 8 + conda: "../envs/environment.yaml" script: - "scripts/build_population_layouts.py" + "../scripts/build_population_layouts.py" rule build_clustered_population_layouts: @@ -33,8 +34,9 @@ rule build_clustered_population_layouts: mem_mb=10000, benchmark: "benchmarks/build_clustered_population_layouts/s{simpl}_{clusters}" + conda: "../envs/environment.yaml" script: - "scripts/build_clustered_population_layouts.py" + "../scripts/build_clustered_population_layouts.py" rule build_simplified_population_layouts: @@ -50,8 +52,9 @@ rule build_simplified_population_layouts: mem_mb=10000, benchmark: "benchmarks/build_clustered_population_layouts/s{simpl}" + conda: "../envs/environment.yaml" script: - "scripts/build_clustered_population_layouts.py" + "../scripts/build_clustered_population_layouts.py" if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: @@ -63,8 +66,9 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: cleaned_gas_network=RESOURCES + "gas_network.csv", resources: mem_mb=4000, + conda: "../envs/environment.yaml" script: - "scripts/build_gas_network.py" + "../scripts/build_gas_network.py" rule build_gas_input_locations: input: @@ -85,8 +89,9 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: + "gas_input_locations_s{simpl}_{clusters}_simplified.csv", resources: mem_mb=2000, + conda: "../envs/environment.yaml" script: - "scripts/build_gas_input_locations.py" + "../scripts/build_gas_input_locations.py" rule cluster_gas_network: input: @@ -99,8 +104,9 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: clustered_gas_network=RESOURCES + "gas_network_elec_s{simpl}_{clusters}.csv", resources: mem_mb=4000, + conda: "../envs/environment.yaml" script: - "scripts/cluster_gas_network.py" + "../scripts/cluster_gas_network.py" gas_infrastructure = { **rules.cluster_gas_network.output, @@ -126,8 +132,9 @@ rule build_heat_demands: threads: 8 benchmark: "benchmarks/build_heat_demands/{scope}_s{simpl}_{clusters}" + conda: "../envs/environment.yaml" script: - "scripts/build_heat_demand.py" + "../scripts/build_heat_demand.py" rule build_temperature_profiles: @@ -143,8 +150,9 @@ rule build_temperature_profiles: threads: 8 benchmark: "benchmarks/build_temperature_profiles/{scope}_s{simpl}_{clusters}" + conda: "../envs/environment.yaml" script: - "scripts/build_temperature_profiles.py" + "../scripts/build_temperature_profiles.py" rule build_cop_profiles: @@ -166,8 +174,9 @@ rule build_cop_profiles: mem_mb=20000, benchmark: "benchmarks/build_cop_profiles/s{simpl}_{clusters}" + conda: "../envs/environment.yaml" script: - "scripts/build_cop_profiles.py" + "../scripts/build_cop_profiles.py" rule build_solar_thermal_profiles: @@ -182,8 +191,9 @@ rule build_solar_thermal_profiles: threads: 16 benchmark: "benchmarks/build_solar_thermal_profiles/{scope}_s{simpl}_{clusters}" + conda: "../envs/environment.yaml" script: - "scripts/build_solar_thermal_profiles.py" + "../scripts/build_solar_thermal_profiles.py" def input_eurostat(w): @@ -209,8 +219,9 @@ rule build_energy_totals: mem_mb=10000, benchmark: "benchmarks/build_energy_totals" + conda: "../envs/environment.yaml" script: - "scripts/build_energy_totals.py" + "../scripts/build_energy_totals.py" rule build_biomass_potentials: @@ -234,8 +245,9 @@ rule build_biomass_potentials: mem_mb=1000, benchmark: "benchmarks/build_biomass_potentials_s{simpl}_{clusters}" + conda: "../envs/environment.yaml" script: - "scripts/build_biomass_potentials.py" + "../scripts/build_biomass_potentials.py" if config["sector"]["biomass_transport"]: @@ -253,8 +265,9 @@ if config["sector"]["biomass_transport"]: mem_mb=1000, benchmark: "benchmarks/build_biomass_transport_costs" + conda: "../envs/environment.yaml" script: - "scripts/build_biomass_transport_costs.py" + "../scripts/build_biomass_transport_costs.py" build_biomass_transport_costs_output = rules.build_biomass_transport_costs.output @@ -285,8 +298,9 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: mem_mb=4000, benchmark: "benchmarks/build_sequestration_potentials_s{simpl}_{clusters}" + conda: "../envs/environment.yaml" script: - "scripts/build_sequestration_potentials.py" + "../scripts/build_sequestration_potentials.py" build_sequestration_potentials_output = rules.build_sequestration_potentials.output @@ -308,8 +322,9 @@ rule build_salt_cavern_potentials: mem_mb=2000, benchmark: "benchmarks/build_salt_cavern_potentials_s{simpl}_{clusters}" + conda: "../envs/environment.yaml" script: - "scripts/build_salt_cavern_potentials.py" + "../scripts/build_salt_cavern_potentials.py" rule build_ammonia_production: @@ -322,8 +337,9 @@ rule build_ammonia_production: mem_mb=1000, benchmark: "benchmarks/build_ammonia_production" + conda: "../envs/environment.yaml" script: - "scripts/build_ammonia_production.py" + "../scripts/build_ammonia_production.py" rule build_industry_sector_ratios: @@ -337,8 +353,9 @@ rule build_industry_sector_ratios: mem_mb=1000, benchmark: "benchmarks/build_industry_sector_ratios" + conda: "../envs/environment.yaml" script: - "scripts/build_industry_sector_ratios.py" + "../scripts/build_industry_sector_ratios.py" rule build_industrial_production_per_country: @@ -354,8 +371,9 @@ rule build_industrial_production_per_country: mem_mb=1000, benchmark: "benchmarks/build_industrial_production_per_country" + conda: "../envs/environment.yaml" script: - "scripts/build_industrial_production_per_country.py" + "../scripts/build_industrial_production_per_country.py" rule build_industrial_production_per_country_tomorrow: @@ -370,8 +388,9 @@ rule build_industrial_production_per_country_tomorrow: mem_mb=1000, benchmark: "benchmarks/build_industrial_production_per_country_tomorrow_{planning_horizons}" + conda: "../envs/environment.yaml" script: - "scripts/build_industrial_production_per_country_tomorrow.py" + "../scripts/build_industrial_production_per_country_tomorrow.py" rule build_industrial_distribution_key: @@ -387,8 +406,9 @@ rule build_industrial_distribution_key: mem_mb=1000, benchmark: "benchmarks/build_industrial_distribution_key/s{simpl}_{clusters}" + conda: "../envs/environment.yaml" script: - "scripts/build_industrial_distribution_key.py" + "../scripts/build_industrial_distribution_key.py" rule build_industrial_production_per_node: @@ -405,8 +425,9 @@ rule build_industrial_production_per_node: mem_mb=1000, benchmark: "benchmarks/build_industrial_production_per_node/s{simpl}_{clusters}_{planning_horizons}" + conda: "../envs/environment.yaml" script: - "scripts/build_industrial_production_per_node.py" + "../scripts/build_industrial_production_per_node.py" rule build_industrial_energy_demand_per_node: @@ -424,8 +445,9 @@ rule build_industrial_energy_demand_per_node: mem_mb=1000, benchmark: "benchmarks/build_industrial_energy_demand_per_node/s{simpl}_{clusters}_{planning_horizons}" + conda: "../envs/environment.yaml" script: - "scripts/build_industrial_energy_demand_per_node.py" + "../scripts/build_industrial_energy_demand_per_node.py" rule build_industrial_energy_demand_per_country_today: @@ -442,8 +464,9 @@ rule build_industrial_energy_demand_per_country_today: mem_mb=1000, benchmark: "benchmarks/build_industrial_energy_demand_per_country_today" + conda: "../envs/environment.yaml" script: - "scripts/build_industrial_energy_demand_per_country_today.py" + "../scripts/build_industrial_energy_demand_per_country_today.py" rule build_industrial_energy_demand_per_node_today: @@ -460,8 +483,9 @@ rule build_industrial_energy_demand_per_node_today: mem_mb=1000, benchmark: "benchmarks/build_industrial_energy_demand_per_node_today/s{simpl}_{clusters}" + conda: "../envs/environment.yaml" script: - "scripts/build_industrial_energy_demand_per_node_today.py" + "../scripts/build_industrial_energy_demand_per_node_today.py" if config["sector"]["retrofitting"]["retro_endogen"]: @@ -485,8 +509,9 @@ if config["sector"]["retrofitting"]["retro_endogen"]: mem_mb=1000, benchmark: "benchmarks/build_retro_cost/s{simpl}_{clusters}" + conda: "../envs/environment.yaml" script: - "scripts/build_retro_cost.py" + "../scripts/build_retro_cost.py" build_retro_cost_output = rules.build_retro_cost.output @@ -505,8 +530,9 @@ rule build_population_weighted_energy_totals: threads: 1 resources: mem_mb=2000, + conda: "../envs/environment.yaml" script: - "scripts/build_population_weighted_energy_totals.py" + "../scripts/build_population_weighted_energy_totals.py" rule build_shipping_demand: @@ -520,8 +546,9 @@ rule build_shipping_demand: threads: 1 resources: mem_mb=2000, + conda: "../envs/environment.yaml" script: - "scripts/build_shipping_demand.py" + "../scripts/build_shipping_demand.py" rule build_transport_demand: @@ -541,8 +568,9 @@ rule build_transport_demand: threads: 1 resources: mem_mb=2000, + conda: "../envs/environment.yaml" script: - "scripts/build_transport_demand.py" + "../scripts/build_transport_demand.py" rule prepare_sector_network: @@ -618,5 +646,6 @@ rule prepare_sector_network: RESULTS + "benchmarks/prepare_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) + conda: "../envs/environment.yaml" script: - "scripts/prepare_sector_network.py" \ No newline at end of file + "../scripts/prepare_sector_network.py" \ No newline at end of file diff --git a/rules/collect.smk b/rules/collect.smk index 471a23a1..250fa893 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -4,6 +4,13 @@ localrules: all, cluster_all_networks, extra_components_all_networks, prepare_all_networks, prepare_sector_networks, solve_all_elec_networks, solve_all_networks, plot_all_networks + +rule all: + input: + RESULTS + "graphs/costs.pdf", + default_target: True + + rule cluster_all_networks: input: expand(RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", **config["scenario"]), @@ -33,11 +40,6 @@ rule prepare_sector_networks: ), -rule all: - input: - RESULTS + "graphs/costs.pdf", - - rule solve_all_elec_networks: input: expand( diff --git a/rules/postprocess.smk b/rules/postprocess.smk index d17ad229..a322a72e 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -23,8 +23,9 @@ rule plot_network: RESULTS + "benchmarks/plot_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) + conda: "../envs/environment.yaml" script: - "scripts/plot_network.py" + "../scripts/plot_network.py" rule copy_config: @@ -37,8 +38,9 @@ rule copy_config: mem_mb=1000, benchmark: RESULTS + "benchmarks/copy_config" + conda: "../envs/environment.yaml" script: - "scripts/copy_config.py" + "../scripts/copy_config.py" rule copy_conda_env: @@ -49,6 +51,7 @@ rule copy_conda_env: mem_mb=500, benchmark: RESULTS + "benchmarks/copy_conda_env" + conda: "../envs/environment.yaml" shell: "conda env export -f {output} --no-builds" @@ -92,8 +95,9 @@ rule make_summary: mem_mb=10000, benchmark: RESULTS + "benchmarks/make_summary" + conda: "../envs/environment.yaml" script: - "scripts/make_summary.py" + "../scripts/make_summary.py" rule plot_summary: @@ -114,5 +118,6 @@ rule plot_summary: mem_mb=10000, benchmark: RESULTS + "benchmarks/plot_summary" + conda: "../envs/environment.yaml" script: - "scripts/plot_summary.py" \ No newline at end of file + "../scripts/plot_summary.py" \ No newline at end of file diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 2790f61f..cd5ab42f 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -28,7 +28,7 @@ if config["enable"].get("retrieve_databundle", True): resources: mem_mb=1000, script: - "scripts/retrieve_databundle.py" + "../scripts/retrieve_databundle.py" if config["enable"].get("retrieve_cutout", True): @@ -110,7 +110,7 @@ if config["enable"].get("retrieve_sector_databundle", True): log: "logs/retrieve_sector_databundle.log", script: - "scripts/retrieve_sector_databundle.py" + "../scripts/retrieve_sector_databundle.py" if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: @@ -126,7 +126,7 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: output: expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles), script: - "scripts/retrieve_gas_infrastructure_data.py" + "../scripts/retrieve_gas_infrastructure_data.py" rule retrieve_load_data: diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 4cc6496f..794d45a2 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -42,8 +42,9 @@ rule solve_network: mem_mb=memory, shadow: "minimal" + conda: "../envs/environment.yaml" script: - "scripts/solve_network.py" + "../scripts/solve_network.py" rule solve_operations_network: @@ -71,5 +72,6 @@ rule solve_operations_network: mem_mb=(lambda w: 5000 + 372 * int(w.clusters)), shadow: "minimal" + conda: "../envs/environment.yaml" script: - "scripts/solve_operations_network.py" \ No newline at end of file + "../scripts/solve_operations_network.py" \ No newline at end of file diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 6de5e3ea..7d49b5eb 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -32,9 +32,11 @@ rule add_existing_baseyear: RESULTS + "benchmarks/add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) + conda: "../envs/environment.yaml" script: "scripts/add_existing_baseyear.py" + def solved_previous_horizon(wildcards): planning_horizons = config["scenario"]["planning_horizons"] i = planning_horizons.index(int(wildcards.planning_horizons)) @@ -46,6 +48,7 @@ def solved_previous_horizon(wildcards): + ".nc" ) + rule add_brownfield: input: overrides="data/override_component_attrs", @@ -66,11 +69,14 @@ rule add_brownfield: RESULTS + "benchmarks/add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) + conda: "../envs/environment.yaml" script: "scripts/add_brownfield.py" + ruleorder: add_existing_baseyear > add_brownfield + rule solve_sector_network_myopic: input: overrides="data/override_component_attrs", @@ -98,5 +104,6 @@ rule solve_sector_network_myopic: RESULTS + "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) + conda: "../envs/environment.yaml" script: "scripts/solve_sector_network.py" \ No newline at end of file diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 8b70d2da..7637e75c 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -30,5 +30,6 @@ rule solve_sector_network: RESULTS + "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) + conda: "../envs/environment.yaml" script: - "scripts/solve_sector_network.py" \ No newline at end of file + "../scripts/solve_sector_network.py" \ No newline at end of file