diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 3b77c437..b0727b17 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -12,7 +12,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import update_config_with_sector_opts +from _helpers import update_config_with_sector_opts, configure_logging, set_scenario_config from add_existing_baseyear import add_build_year_to_new_assets from pypsa.clustering.spatial import normed_or_uniform @@ -210,7 +210,8 @@ if __name__ == "__main__": planning_horizons=2030, ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + configure_logging(snakemake) + set_scenario_config(snakemake) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index c0d37a5b..25e35edd 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -15,7 +15,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import update_config_with_sector_opts +from _helpers import update_config_with_sector_opts, set_scenario_config, configure_logging from add_electricity import sanitize_carriers from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs @@ -552,7 +552,8 @@ if __name__ == "__main__": planning_horizons=2020, ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + configure_logging(snakemake) + set_scenario_config(snakemake) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index 1bcdf9ae..47907cc9 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -9,6 +9,8 @@ Build historical annual ammonia production per country in ktonNH3/a. import country_converter as coco import pandas as pd +from _helpers import set_scenario_config + cc = coco.CountryConverter() @@ -18,6 +20,8 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_ammonia_production") + set_scenario_config(snakemake) + ammonia = pd.read_excel( snakemake.input.usgs, sheet_name="T12", diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 6b5cb147..2fe6922c 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -16,6 +16,8 @@ import pandas as pd logger = logging.getLogger(__name__) AVAILABLE_BIOMASS_YEARS = [2010, 2020, 2030, 2040, 2050] +from _helpers import configure_logging, set_scenario_config + def build_nuts_population_data(year=2013): pop = pd.read_csv( @@ -220,6 +222,9 @@ if __name__ == "__main__": clusters="5", planning_horizons=2050, ) + + configure_logging(snakemake) + set_scenario_config(snakemake) overnight = snakemake.config["foresight"] == "overnight" params = snakemake.params.biomass diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index f1d386bd..4217baff 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -12,6 +12,8 @@ import geopandas as gpd import pandas as pd import xarray as xr +from _helpers import set_scenario_config + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -22,6 +24,8 @@ if __name__ == "__main__": clusters=48, ) + set_scenario_config(snakemake) + cutout = atlite.Cutout(snakemake.input.cutout) clustered_regions = ( diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index 4b1d952e..ee68dd0a 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -15,6 +15,8 @@ https://doi.org/10.1039/C2EE22653G. import xarray as xr +from _helpers import set_scenario_config + def coefficient_of_performance(delta_T, source="air"): if source == "air": @@ -35,6 +37,8 @@ if __name__ == "__main__": clusters=48, ) + set_scenario_config(snakemake) + for area in ["total", "urban", "rural"]: for source in ["air", "soil"]: source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"]) diff --git a/scripts/build_daily_heat_demand.py b/scripts/build_daily_heat_demand.py index e334b1b3..03ba8c2e 100644 --- a/scripts/build_daily_heat_demand.py +++ b/scripts/build_daily_heat_demand.py @@ -12,6 +12,7 @@ import numpy as np import pandas as pd import xarray as xr from dask.distributed import Client, LocalCluster +from _helpers import set_scenario_config if __name__ == "__main__": if "snakemake" not in globals(): @@ -23,6 +24,7 @@ if __name__ == "__main__": simpl="", clusters=48, ) + set_scenario_config(snakemake) nprocesses = int(snakemake.threads) cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) diff --git a/scripts/build_district_heat_share.py b/scripts/build_district_heat_share.py index 86c42631..121e4663 100644 --- a/scripts/build_district_heat_share.py +++ b/scripts/build_district_heat_share.py @@ -10,6 +10,7 @@ import logging import pandas as pd from prepare_sector_network import get +from _helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) @@ -24,6 +25,8 @@ if __name__ == "__main__": clusters=48, planning_horizons="2050", ) + configure_logging(snakemake) + set_scenario_config(snakemake) investment_year = int(snakemake.wildcards.planning_horizons[-4:]) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index c67bb49d..f22ddc25 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -14,7 +14,7 @@ import country_converter as coco import geopandas as gpd import numpy as np import pandas as pd -from _helpers import mute_print +from _helpers import mute_print, configure_logging, set_scenario_config from tqdm import tqdm cc = coco.CountryConverter() @@ -743,7 +743,8 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_energy_totals") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + configure_logging(snakemake) + set_scenario_config(snakemake) params = snakemake.params.energy diff --git a/scripts/build_existing_heating_distribution.py b/scripts/build_existing_heating_distribution.py index 78518597..eb2361c2 100644 --- a/scripts/build_existing_heating_distribution.py +++ b/scripts/build_existing_heating_distribution.py @@ -9,6 +9,7 @@ horizon. import country_converter as coco import numpy as np import pandas as pd +from _helpers import set_scenario_config cc = coco.CountryConverter() @@ -126,5 +127,6 @@ if __name__ == "__main__": clusters=48, planning_horizons=2050, ) + set_scenario_config(snakemake) build_existing_heating() diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index 081f74b9..5aba46e5 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -12,6 +12,7 @@ import logging import geopandas as gpd import pandas as pd from cluster_gas_network import load_bus_regions +from _helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) @@ -134,7 +135,8 @@ if __name__ == "__main__": clusters="128", ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + configure_logging(snakemake) + set_scenario_config(snakemake) regions = load_bus_regions( snakemake.input.regions_onshore, snakemake.input.regions_offshore diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py index 13cd75ba..0febd43d 100644 --- a/scripts/build_gas_network.py +++ b/scripts/build_gas_network.py @@ -13,6 +13,7 @@ import geopandas as gpd import pandas as pd from pypsa.geo import haversine_pts from shapely.geometry import Point +from _helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) @@ -143,7 +144,8 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_gas_network") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + configure_logging(snakemake) + set_scenario_config(snakemake) gas_network = load_dataset(snakemake.input.gas_network) diff --git a/scripts/build_hourly_heat_demand.py b/scripts/build_hourly_heat_demand.py index c972da89..a3110e1f 100644 --- a/scripts/build_hourly_heat_demand.py +++ b/scripts/build_hourly_heat_demand.py @@ -10,7 +10,7 @@ from itertools import product import pandas as pd import xarray as xr -from _helpers import generate_periodic_profiles +from _helpers import generate_periodic_profiles, set_scenario_config if __name__ == "__main__": if "snakemake" not in globals(): @@ -22,6 +22,7 @@ if __name__ == "__main__": simpl="", clusters=48, ) + set_scenario_config(snakemake) snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 9b234e29..29b7538e 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -14,6 +14,8 @@ import country_converter as coco import geopandas as gpd import pandas as pd +from _helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) cc = coco.CountryConverter() @@ -148,8 +150,8 @@ if __name__ == "__main__": simpl="", clusters=128, ) - - logging.basicConfig(level=snakemake.config["logging"]["level"]) + configure_logging(snakemake) + set_scenario_config(snakemake) countries = snakemake.params.countries diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index d1c672f1..99342b9e 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -12,6 +12,7 @@ from functools import partial import country_converter as coco import pandas as pd from tqdm import tqdm +from _helpers import set_scenario_config cc = coco.CountryConverter() @@ -175,6 +176,7 @@ if __name__ == "__main__": from _helpers import mock_snakemake snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today") + set_scenario_config(snakemake) params = snakemake.params.industry year = params.get("reference_year", 2015) diff --git a/scripts/build_industrial_energy_demand_per_node.py b/scripts/build_industrial_energy_demand_per_node.py index 55c10c5d..fcc0abd9 100644 --- a/scripts/build_industrial_energy_demand_per_node.py +++ b/scripts/build_industrial_energy_demand_per_node.py @@ -8,6 +8,8 @@ Build industrial energy demand per model region. import pandas as pd +from _helpers import set_scenario_config + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -18,6 +20,7 @@ if __name__ == "__main__": clusters=48, planning_horizons=2030, ) + set_scenario_config(snakemake) # import EU ratios df as csv fn = snakemake.input.industry_sector_ratios diff --git a/scripts/build_industrial_energy_demand_per_node_today.py b/scripts/build_industrial_energy_demand_per_node_today.py index d845e704..1b7aba83 100644 --- a/scripts/build_industrial_energy_demand_per_node_today.py +++ b/scripts/build_industrial_energy_demand_per_node_today.py @@ -11,6 +11,8 @@ from itertools import product import numpy as np import pandas as pd +from _helpers import set_scenario_config + # map JRC/our sectors to hotmaps sector, where mapping exist sector_mapping = { "Electric arc": "Iron and steel", @@ -75,5 +77,6 @@ if __name__ == "__main__": simpl="", clusters=48, ) + set_scenario_config(snakemake) build_nodal_industrial_energy_demand() diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 0aea4f15..d9ec9afd 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -13,7 +13,7 @@ from functools import partial import country_converter as coco import numpy as np import pandas as pd -from _helpers import mute_print +from _helpers import mute_print, set_scenario_config, configure_logging from tqdm import tqdm logger = logging.getLogger(__name__) @@ -274,8 +274,8 @@ if __name__ == "__main__": from _helpers import mock_snakemake snakemake = mock_snakemake("build_industrial_production_per_country") - - logging.basicConfig(level=snakemake.config["logging"]["level"]) + configure_logging(snakemake) + set_scenario_config(snakemake) countries = snakemake.params.countries diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py index ffed5195..835a2687 100644 --- a/scripts/build_industrial_production_per_country_tomorrow.py +++ b/scripts/build_industrial_production_per_country_tomorrow.py @@ -9,11 +9,14 @@ Build future industrial production per country. import pandas as pd from prepare_sector_network import get +from _helpers import set_scenario_config + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow") + set_scenario_config(snakemake) params = snakemake.params.industry diff --git a/scripts/build_industrial_production_per_node.py b/scripts/build_industrial_production_per_node.py index 7b69948a..c84590b7 100644 --- a/scripts/build_industrial_production_per_node.py +++ b/scripts/build_industrial_production_per_node.py @@ -7,6 +7,7 @@ Build industrial production per model region. """ from itertools import product +from _helpers import set_scenario_config import pandas as pd @@ -72,5 +73,6 @@ if __name__ == "__main__": simpl="", clusters=48, ) + set_scenario_config(snakemake) build_nodal_industrial_production() diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index 45705002..6014feee 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -7,7 +7,7 @@ Build specific energy consumption by carrier and industries. """ import pandas as pd -from _helpers import mute_print +from _helpers import mute_print, set_scenario_config # GWh/ktoe OR MWh/toe toe_to_MWh = 11.630 @@ -1464,6 +1464,7 @@ if __name__ == "__main__": from _helpers import mock_snakemake snakemake = mock_snakemake("build_industry_sector_ratios") + set_scenario_config(snakemake) # TODO make params option year = 2015 diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index cb63c27e..db548140 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -14,6 +14,8 @@ import numpy as np import pandas as pd import xarray as xr +from _helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) if __name__ == "__main__": @@ -22,7 +24,8 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_population_layouts") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + configure_logging(snakemake) + set_scenario_config(snakemake) cutout = atlite.Cutout(snakemake.input.cutout) diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index 879e3b9b..6ed32086 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -8,6 +8,8 @@ Distribute country-level energy demands by population. import pandas as pd +from _helpers import set_scenario_config + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -17,6 +19,7 @@ if __name__ == "__main__": simpl="", clusters=48, ) + set_scenario_config(snakemake) pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index 60d74afa..b6153a2d 100755 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -68,6 +68,7 @@ The script has the following structure: """ import pandas as pd import xarray as xr +from _helpers import set_scenario_config # (i) --- FIXED PARAMETER / STANDARD VALUES ----------------------------------- @@ -1053,6 +1054,7 @@ if __name__ == "__main__": ll="v1.0", sector_opts="Co2L0-168H-T-H-B-I-solar3-dist1", ) + set_scenario_config(snakemake) # ******** config ********************************************************* diff --git a/scripts/build_salt_cavern_potentials.py b/scripts/build_salt_cavern_potentials.py index ed039772..08071b22 100644 --- a/scripts/build_salt_cavern_potentials.py +++ b/scripts/build_salt_cavern_potentials.py @@ -24,6 +24,7 @@ onshore (>50km from sea), offshore (Figure 7). import geopandas as gpd import pandas as pd +from _helpers import set_scenario_config def concat_gdf(gdf_list, crs="EPSG:4326"): @@ -77,6 +78,8 @@ if __name__ == "__main__": "build_salt_cavern_potentials", simpl="", clusters="37" ) + set_scenario_config(snakemake) + fn_onshore = snakemake.input.regions_onshore fn_offshore = snakemake.input.regions_offshore diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index f6ad3526..9e7678bb 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -11,6 +11,8 @@ database_en>`_. import geopandas as gpd import pandas as pd +from _helpers import set_scenario_config + def area(gdf): """ @@ -39,6 +41,8 @@ if __name__ == "__main__": "build_sequestration_potentials", simpl="", clusters="181" ) + set_scenario_config(snakemake) + cf = snakemake.params.sequestration_potential gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) diff --git a/scripts/build_shipping_demand.py b/scripts/build_shipping_demand.py index 8000c66c..0690e33d 100644 --- a/scripts/build_shipping_demand.py +++ b/scripts/build_shipping_demand.py @@ -11,6 +11,7 @@ import json import geopandas as gpd import pandas as pd +from _helpers import set_scenario_config if __name__ == "__main__": if "snakemake" not in globals(): @@ -21,6 +22,7 @@ if __name__ == "__main__": simpl="", clusters=48, ) + set_scenario_config(snakemake) scope = gpd.read_file(snakemake.input.scope).geometry[0] regions = gpd.read_file(snakemake.input.regions).set_index("name") diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index ee6ed881..12826420 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -13,6 +13,8 @@ import pandas as pd import xarray as xr from dask.distributed import Client, LocalCluster +from _helpers import set_scenario_config + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -22,6 +24,7 @@ if __name__ == "__main__": simpl="", clusters=48, ) + set_scenario_config(snakemake) nprocesses = int(snakemake.threads) cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index 02fa4a71..878ebc03 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -12,6 +12,7 @@ import numpy as np import pandas as pd import xarray as xr from dask.distributed import Client, LocalCluster +from _helpers import set_scenario_config if __name__ == "__main__": if "snakemake" not in globals(): @@ -22,6 +23,7 @@ if __name__ == "__main__": simpl="", clusters=48, ) + set_scenario_config(snakemake) nprocesses = int(snakemake.threads) cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index 33c8faae..671357dd 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -13,7 +13,7 @@ import logging import numpy as np import pandas as pd import xarray as xr -from _helpers import configure_logging, generate_periodic_profiles +from _helpers import configure_logging, generate_periodic_profiles, set_scenario_config logger = logging.getLogger(__name__) @@ -171,6 +171,7 @@ if __name__ == "__main__": clusters=48, ) configure_logging(snakemake) + set_scenario_config(snakemake) pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index e709d772..567f4458 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -12,6 +12,7 @@ import geopandas as gpd import pandas as pd from pypsa.geo import haversine_pts from shapely import wkt +from _helpers import set_scenario_config, configure_logging logger = logging.getLogger(__name__) @@ -105,8 +106,8 @@ if __name__ == "__main__": from _helpers import mock_snakemake snakemake = mock_snakemake("cluster_gas_network", simpl="", clusters="37") - - logging.basicConfig(level=snakemake.config["logging"]["level"]) + configure_logging(snakemake) + set_scenario_config(snakemake) fn = snakemake.input.cleaned_gas_network df = pd.read_csv(fn, index_col=0) diff --git a/scripts/determine_availability_matrix_MD_UA.py b/scripts/determine_availability_matrix_MD_UA.py index efe9a712..84076383 100644 --- a/scripts/determine_availability_matrix_MD_UA.py +++ b/scripts/determine_availability_matrix_MD_UA.py @@ -15,7 +15,7 @@ import fiona import geopandas as gpd import matplotlib.pyplot as plt import numpy as np -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from atlite.gis import shape_availability from rasterio.plot import show @@ -38,6 +38,7 @@ if __name__ == "__main__": "determine_availability_matrix_MD_UA", technology="solar" ) configure_logging(snakemake) + set_scenario_config(snakemake) nprocesses = None # snakemake.config["atlite"].get("nprocesses") noprogress = not snakemake.config["atlite"].get("show_progress", True) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 76d8099c..82ce2328 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -14,6 +14,7 @@ import numpy as np import pandas as pd import pypsa from prepare_sector_network import prepare_costs +from _helpers import set_scenario_config, configure_logging idx = pd.IndexSlice logger = logging.getLogger(__name__) @@ -673,7 +674,8 @@ if __name__ == "__main__": snakemake = mock_snakemake("make_summary") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + configure_logging(snakemake) + set_scenario_config(snakemake) networks_dict = { (cluster, ll, opt + sector_opt, planning_horizon): "results/" diff --git a/scripts/make_summary_perfect.py b/scripts/make_summary_perfect.py index 064db454..7c2e8d30 100644 --- a/scripts/make_summary_perfect.py +++ b/scripts/make_summary_perfect.py @@ -19,6 +19,7 @@ from make_summary import assign_carriers, assign_locations from prepare_sector_network import prepare_costs from pypsa.descriptors import get_active_assets from six import iteritems +from _helpers import set_scenario_config idx = pd.IndexSlice @@ -722,6 +723,7 @@ if __name__ == "__main__": from _helpers import mock_snakemake snakemake = mock_snakemake("make_summary_perfect") + set_scenario_config(snakemake) run = snakemake.config["run"]["name"] if run != "": diff --git a/scripts/plot_gas_network.py b/scripts/plot_gas_network.py index e2953604..26186d51 100644 --- a/scripts/plot_gas_network.py +++ b/scripts/plot_gas_network.py @@ -13,7 +13,7 @@ import geopandas as gpd import matplotlib.pyplot as plt import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from plot_power_network import assign_location, load_projection from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches @@ -237,6 +237,7 @@ if __name__ == "__main__": ) configure_logging(snakemake) + set_scenario_config(snakemake) n = pypsa.Network(snakemake.input.network) diff --git a/scripts/plot_hydrogen_network.py b/scripts/plot_hydrogen_network.py index 95741170..4cb58557 100644 --- a/scripts/plot_hydrogen_network.py +++ b/scripts/plot_hydrogen_network.py @@ -13,7 +13,7 @@ import geopandas as gpd import matplotlib.pyplot as plt import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from plot_power_network import assign_location, load_projection from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches @@ -254,6 +254,7 @@ if __name__ == "__main__": ) configure_logging(snakemake) + set_scenario_config(snakemake) n = pypsa.Network(snakemake.input.network) diff --git a/scripts/plot_power_network.py b/scripts/plot_power_network.py index 0e13e497..6db53bcc 100644 --- a/scripts/plot_power_network.py +++ b/scripts/plot_power_network.py @@ -14,7 +14,7 @@ import geopandas as gpd import matplotlib.pyplot as plt import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from plot_summary import preferred_order, rename_techs from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches @@ -257,6 +257,7 @@ if __name__ == "__main__": ) configure_logging(snakemake) + set_scenario_config(snakemake) n = pypsa.Network(snakemake.input.network) diff --git a/scripts/plot_power_network_clustered.py b/scripts/plot_power_network_clustered.py index 8217ac2e..43746039 100644 --- a/scripts/plot_power_network_clustered.py +++ b/scripts/plot_power_network_clustered.py @@ -6,13 +6,13 @@ Plot clustered electricity transmission network. """ -import cartopy.crs as ccrs import geopandas as gpd import matplotlib.pyplot as plt import pypsa from matplotlib.lines import Line2D from plot_power_network import load_projection from pypsa.plot import add_legend_lines +from _helpers import set_scenario_config if __name__ == "__main__": if "snakemake" not in globals(): @@ -23,6 +23,7 @@ if __name__ == "__main__": clusters=128, configfiles=["../../config/config.test.yaml"], ) + set_scenario_config(snakemake) lw_factor = 2e3 diff --git a/scripts/plot_power_network_perfect.py b/scripts/plot_power_network_perfect.py index ff576d33..f7506a00 100644 --- a/scripts/plot_power_network_perfect.py +++ b/scripts/plot_power_network_perfect.py @@ -13,7 +13,7 @@ import geopandas as gpd import matplotlib.pyplot as plt import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from plot_power_network import assign_location, load_projection, rename_techs_tyndp from plot_summary import preferred_order from pypsa.plot import add_legend_circles, add_legend_lines @@ -184,6 +184,7 @@ if __name__ == "__main__": ) configure_logging(snakemake) + set_scenario_config(snakemake) n = pypsa.Network(snakemake.input.network) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index cfb32441..da077348 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -12,6 +12,7 @@ import matplotlib.gridspec as gridspec import matplotlib.pyplot as plt import pandas as pd from prepare_sector_network import co2_emissions_year +from _helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) plt.style.use("ggplot") @@ -572,7 +573,8 @@ if __name__ == "__main__": snakemake = mock_snakemake("plot_summary") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + configure_logging(snakemake) + set_scenario_config(snakemake) n_header = 4 diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index cf013577..de9932fd 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -12,7 +12,7 @@ import re import numpy as np import pandas as pd import pypsa -from _helpers import update_config_with_sector_opts +from _helpers import update_config_with_sector_opts, set_scenario_config, configure_logging from add_existing_baseyear import add_build_year_to_new_assets from pypsa.descriptors import expand_series from pypsa.io import import_components_from_dataframe @@ -514,6 +514,8 @@ if __name__ == "__main__": ll="v1.5", sector_opts="1p7-4380H-T-H-B-I-A-dist1", ) + configure_logging(snakemake) + set_scenario_config(snakemake) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) # parameters ----------------------------------------------------------- diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index b1161c19..bf9ed58f 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -18,7 +18,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import set_scenario_config, update_config_with_sector_opts +from _helpers import configure_logging, set_scenario_config, update_config_with_sector_opts from add_electricity import calculate_annuity, sanitize_carriers, sanitize_locations from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2 from networkx.algorithms import complement @@ -3572,7 +3572,7 @@ if __name__ == "__main__": planning_horizons="2030", ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + configure_logging(snakemake) set_scenario_config(snakemake) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 25894063..b6a4d378 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -36,7 +36,7 @@ import logging import tarfile from pathlib import Path -from _helpers import configure_logging, progress_retrieve, validate_checksum +from _helpers import configure_logging, progress_retrieve, validate_checksum, set_scenario_config logger = logging.getLogger(__name__) @@ -49,9 +49,8 @@ if __name__ == "__main__": rootpath = ".." else: rootpath = "." - configure_logging( - snakemake - ) # TODO Make logging compatible with progressbar (see PR #102) + configure_logging(snakemake) + set_scenario_config(snakemake) if snakemake.config["tutorial"]: url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz" diff --git a/scripts/retrieve_electricity_demand.py b/scripts/retrieve_electricity_demand.py index a8a44b68..94077fdf 100644 --- a/scripts/retrieve_electricity_demand.py +++ b/scripts/retrieve_electricity_demand.py @@ -12,7 +12,7 @@ import pandas as pd logger = logging.getLogger(__name__) -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config if __name__ == "__main__": if "snakemake" not in globals(): @@ -23,6 +23,7 @@ if __name__ == "__main__": else: rootpath = "." configure_logging(snakemake) + set_scenario_config(snakemake) url = "https://data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv" diff --git a/scripts/retrieve_gas_infrastructure_data.py b/scripts/retrieve_gas_infrastructure_data.py index d984b9fe..7cc1c270 100644 --- a/scripts/retrieve_gas_infrastructure_data.py +++ b/scripts/retrieve_gas_infrastructure_data.py @@ -11,7 +11,7 @@ import logging import zipfile from pathlib import Path -from _helpers import progress_retrieve, validate_checksum +from _helpers import progress_retrieve, validate_checksum, set_scenario_config, configure_logging logger = logging.getLogger(__name__) @@ -24,6 +24,8 @@ if __name__ == "__main__": rootpath = ".." else: rootpath = "." + configure_logging(snakemake) + set_scenario_config(snakemake) url = "https://zenodo.org/record/4767098/files/IGGIELGN.zip" diff --git a/scripts/retrieve_irena.py b/scripts/retrieve_irena.py index 7b123475..dbd11129 100644 --- a/scripts/retrieve_irena.py +++ b/scripts/retrieve_irena.py @@ -26,7 +26,7 @@ This rule downloads the existing capacities from `IRENASTAT