From 3bdf3b6f5e65699f0e9a1df8feae34a3b6aa7b54 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 6 Mar 2023 19:09:45 +0100 Subject: [PATCH] use mock_snakemake from pypsa-eur --- scripts/add_brownfield.py | 2 +- scripts/add_existing_baseyear.py | 2 +- scripts/build_ammonia_production.py | 2 +- scripts/build_biomass_potentials.py | 2 +- scripts/build_clustered_population_layouts.py | 2 +- scripts/build_cop_profiles.py | 2 +- scripts/build_energy_totals.py | 2 +- scripts/build_gas_input_locations.py | 2 +- scripts/build_gas_network.py | 2 +- scripts/build_heat_demand.py | 2 +- scripts/build_industrial_distribution_key.py | 2 +- ...ustrial_energy_demand_per_country_today.py | 2 +- ...build_industrial_energy_demand_per_node.py | 2 +- ...industrial_energy_demand_per_node_today.py | 2 +- ...build_industrial_production_per_country.py | 2 +- ...ustrial_production_per_country_tomorrow.py | 2 +- .../build_industrial_production_per_node.py | 2 +- scripts/build_industry_sector_ratios.py | 2 +- scripts/build_population_layouts.py | 2 +- ...build_population_weighted_energy_totals.py | 2 +- scripts/build_retro_cost.py | 2 +- scripts/build_salt_cavern_potentials.py | 2 +- scripts/build_sequestration_potentials.py | 2 +- scripts/build_shipping_demand.py | 2 +- scripts/build_solar_thermal_profiles.py | 2 +- scripts/build_temperature_profiles.py | 2 +- scripts/build_transport_demand.py | 2 +- scripts/cluster_gas_network.py | 2 +- scripts/copy_config.py | 2 +- scripts/helper.py | 68 ------------------- scripts/make_summary.py | 2 +- scripts/plot_network.py | 2 +- scripts/plot_summary.py | 2 +- scripts/prepare_sector_network.py | 2 +- scripts/retrieve_gas_infrastructure_data.py | 2 +- scripts/solve_sector_network.py | 2 +- 36 files changed, 35 insertions(+), 103 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index e1ecda0c..ae840044 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -125,7 +125,7 @@ def add_brownfield(n, n_p, year): # %% if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "add_brownfield", diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 4b1a4fa8..09e72faf 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -606,7 +606,7 @@ def add_heating_capacities_installed_before_baseyear( # %% if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "add_existing_baseyear", diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index c0ed7cf3..6e911235 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -35,7 +35,7 @@ country_to_alpha2 = { if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("build_ammonia_production") diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index c0b059bc..43360d34 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -206,7 +206,7 @@ def convert_nuts2_to_regions(bio_nuts2, regions): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("build_biomass_potentials", simpl="", clusters="5") diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index 9703c96f..9fb70d40 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -14,7 +14,7 @@ import xarray as xr if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_clustered_population_layouts", diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index 85d8da30..44b8052e 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -27,7 +27,7 @@ def coefficient_of_performance(delta_T, source="air"): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_cop_profiles", diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 07d7baac..68eb60a5 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -751,7 +751,7 @@ def build_transport_data(countries, population, idees): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("build_energy_totals") diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index 1da9fff1..86d477db 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -80,7 +80,7 @@ def build_gas_input_locations(lng_fn, entry_fn, prod_fn, countries): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_gas_input_locations", diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py index a427d4eb..c13cca3f 100644 --- a/scripts/build_gas_network.py +++ b/scripts/build_gas_network.py @@ -144,7 +144,7 @@ def prepare_dataset( if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("build_gas_network") diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index c00054c3..a626dc28 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -16,7 +16,7 @@ from dask.distributed import Client, LocalCluster if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_heat_demands", diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 436f6989..db2628b2 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -138,7 +138,7 @@ def build_nodal_distribution_key(hotmaps, regions): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_industrial_distribution_key", diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 00f41f03..1f1cb474 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -196,7 +196,7 @@ def industrial_energy_demand(countries): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today") diff --git a/scripts/build_industrial_energy_demand_per_node.py b/scripts/build_industrial_energy_demand_per_node.py index 7b2a3d9b..95d0782c 100644 --- a/scripts/build_industrial_energy_demand_per_node.py +++ b/scripts/build_industrial_energy_demand_per_node.py @@ -11,7 +11,7 @@ import pandas as pd if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_industrial_energy_demand_per_node", diff --git a/scripts/build_industrial_energy_demand_per_node_today.py b/scripts/build_industrial_energy_demand_per_node_today.py index 902bca20..e715bc53 100644 --- a/scripts/build_industrial_energy_demand_per_node_today.py +++ b/scripts/build_industrial_energy_demand_per_node_today.py @@ -69,7 +69,7 @@ def build_nodal_industrial_energy_demand(): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_industrial_energy_demand_per_node_today", diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 99a8b750..e57583b9 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -294,7 +294,7 @@ def separate_basic_chemicals(demand): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("build_industrial_production_per_country") diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py index 350f3c4d..e554e159 100644 --- a/scripts/build_industrial_production_per_country_tomorrow.py +++ b/scripts/build_industrial_production_per_country_tomorrow.py @@ -12,7 +12,7 @@ from prepare_sector_network import get if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow") diff --git a/scripts/build_industrial_production_per_node.py b/scripts/build_industrial_production_per_node.py index ea56ee2b..681c809c 100644 --- a/scripts/build_industrial_production_per_node.py +++ b/scripts/build_industrial_production_per_node.py @@ -66,7 +66,7 @@ def build_nodal_industrial_production(): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_industrial_production_per_node", diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index 89e35057..3259b885 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -1462,7 +1462,7 @@ def other_industrial_sectors(): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("build_industry_sector_ratios") diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index 4eb590a6..1325d20d 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -21,7 +21,7 @@ import xarray as xr if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("build_population_layouts") diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index 252ced2d..1d29a08f 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -11,7 +11,7 @@ import pandas as pd if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_population_weighted_energy_totals", diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index cef2ef1f..b5dcbc94 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -1032,7 +1032,7 @@ def sample_dE_costs_area( # %% --- MAIN -------------------------------------------------------------- if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_retro_cost", diff --git a/scripts/build_salt_cavern_potentials.py b/scripts/build_salt_cavern_potentials.py index a496e4d2..082c688f 100644 --- a/scripts/build_salt_cavern_potentials.py +++ b/scripts/build_salt_cavern_potentials.py @@ -77,7 +77,7 @@ def salt_cavern_potential_by_region(caverns, regions): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_salt_cavern_potentials", simpl="", clusters="37" diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index c75314d4..63aa9dda 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -30,7 +30,7 @@ def allocate_sequestration_potential( if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_sequestration_potentials", simpl="", clusters="181" diff --git a/scripts/build_shipping_demand.py b/scripts/build_shipping_demand.py index c8c0dc96..8000c66c 100644 --- a/scripts/build_shipping_demand.py +++ b/scripts/build_shipping_demand.py @@ -14,7 +14,7 @@ import pandas as pd if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_shipping_demand_per_node", diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index 98b4b468..1cec57f1 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -16,7 +16,7 @@ from dask.distributed import Client, LocalCluster if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_solar_thermal_profiles", diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index c12fc7f2..f0f08cea 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -16,7 +16,7 @@ from dask.distributed import Client, LocalCluster if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_temperature_profiles", diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index b2d308e3..6d743899 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -159,7 +159,7 @@ def bev_dsm_profile(snapshots, nodes, options): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_transport_demand", diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index 7ae1a9a2..7c181391 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -108,7 +108,7 @@ def aggregate_parallel_pipes(df): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("cluster_gas_network", simpl="", clusters="37") diff --git a/scripts/copy_config.py b/scripts/copy_config.py index 792e9007..0ff60e86 100644 --- a/scripts/copy_config.py +++ b/scripts/copy_config.py @@ -17,7 +17,7 @@ files = { if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("copy_config") diff --git a/scripts/helper.py b/scripts/helper.py index 2c041d1c..c699bfef 100644 --- a/scripts/helper.py +++ b/scripts/helper.py @@ -55,74 +55,6 @@ def override_component_attrs(directory): return attrs -# from pypsa-eur/_helpers.py -def mock_snakemake(rulename, **wildcards): - """ - This function is expected to be executed from the 'scripts'-directory of ' - the snakemake project. It returns a snakemake.script.Snakemake object, - based on the Snakefile. - - If a rule has wildcards, you have to specify them in **wildcards. - - Parameters - ---------- - rulename: str - name of the rule for which the snakemake object should be generated - **wildcards: - keyword arguments fixing the wildcards. Only necessary if wildcards are - needed. - """ - import os - - import snakemake as sm - from packaging.version import Version, parse - from pypsa.descriptors import Dict - from snakemake.script import Snakemake - - script_dir = Path(__file__).parent.resolve() - assert ( - Path.cwd().resolve() == script_dir - ), f"mock_snakemake has to be run from the repository scripts directory {script_dir}" - os.chdir(script_dir.parent) - for p in sm.SNAKEFILE_CHOICES: - if os.path.exists(p): - snakefile = p - break - kwargs = dict(rerun_triggers=[]) if parse(sm.__version__) > Version("7.7.0") else {} - workflow = sm.Workflow(snakefile, overwrite_configfiles=[], **kwargs) - workflow.include(snakefile) - workflow.global_resources = {} - rule = workflow.get_rule(rulename) - dag = sm.dag.DAG(workflow, rules=[rule]) - wc = Dict(wildcards) - job = sm.jobs.Job(rule, dag, wc) - - def make_accessable(*ios): - for io in ios: - for i in range(len(io)): - io[i] = os.path.abspath(io[i]) - - make_accessable(job.input, job.output, job.log) - snakemake = Snakemake( - job.input, - job.output, - job.params, - job.wildcards, - job.threads, - job.resources, - job.log, - job.dag.workflow.config, - job.rule.name, - None, - ) - # create log and output dir if not existent - for path in list(snakemake.log) + list(snakemake.output): - Path(path).parent.mkdir(parents=True, exist_ok=True) - - os.chdir(script_dir) - return snakemake - - # from pypsa-eur/_helpers.py def progress_retrieve(url, file): import urllib diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 1a076728..fad76225 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -673,7 +673,7 @@ def to_csv(df): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("make_summary") diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 02c93953..db5a54ee 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -911,7 +911,7 @@ def plot_series(network, carrier="AC", name="test"): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "plot_network", diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index fe07066a..9af1e09e 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -541,7 +541,7 @@ def plot_carbon_budget_distribution(input_eurostat): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("plot_summary") diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 22ed59a7..134e7672 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3227,7 +3227,7 @@ def set_temporal_aggregation(n, opts, solver_name): # %% if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "prepare_sector_network", diff --git a/scripts/retrieve_gas_infrastructure_data.py b/scripts/retrieve_gas_infrastructure_data.py index 69aa1867..3978b49c 100644 --- a/scripts/retrieve_gas_infrastructure_data.py +++ b/scripts/retrieve_gas_infrastructure_data.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("retrieve_gas_network_data") rootpath = ".." diff --git a/scripts/solve_sector_network.py b/scripts/solve_sector_network.py index 1a7ea352..9239e573 100644 --- a/scripts/solve_sector_network.py +++ b/scripts/solve_sector_network.py @@ -318,7 +318,7 @@ def solve_network(n, config, opts="", **kwargs): # %% if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( "solve_network_myopic",