diff --git a/Snakefile b/Snakefile index 98d5f2d9..22dc9981 100644 --- a/Snakefile +++ b/Snakefile @@ -29,6 +29,7 @@ rule solve_all_elec_networks: if config['enable'].get('prepare_links_p_nom', False): rule prepare_links_p_nom: output: 'data/links_p_nom.csv' + log: 'logs/prepare_links_p_nom.log' threads: 1 resources: mem=500 # group: 'nonfeedin_preparation' @@ -48,6 +49,7 @@ if not config.get('tutorial', False): if config['enable'].get('retrieve_databundle', True): rule retrieve_databundle: output: expand('data/bundle/{file}', file=datafiles) + log: "logs/retrieve_databundle.log" script: 'scripts/retrieve_databundle.py' rule build_powerplants: @@ -55,6 +57,7 @@ rule build_powerplants: base_network="networks/base.nc", custom_powerplants="data/custom_powerplants.csv" output: "resources/powerplants.csv" + log: "logs/build_powerplants.log" threads: 1 resources: mem=500 # group: 'nonfeedin_preparation' @@ -74,6 +77,7 @@ rule base_network: offshore_shapes='resources/offshore_shapes.geojson', europe_shape='resources/europe_shape.geojson' output: "networks/base.nc" + log: "logs/base_network.log" benchmark: "benchmarks/base_network" threads: 1 resources: mem=500 @@ -94,6 +98,7 @@ rule build_shapes: offshore_shapes='resources/offshore_shapes.geojson', europe_shape='resources/europe_shape.geojson', nuts3_shapes='resources/nuts3_shapes.geojson' + log: "logs/build_shapes.log" threads: 1 resources: mem=500 # group: 'nonfeedin_preparation' @@ -107,6 +112,7 @@ rule build_bus_regions: output: regions_onshore="resources/regions_onshore.geojson", regions_offshore="resources/regions_offshore.geojson" + log: "logs/build_bus_regions.log" resources: mem=1000 # group: 'nonfeedin_preparation' script: "scripts/build_bus_regions.py" @@ -114,6 +120,7 @@ rule build_bus_regions: if config['enable'].get('build_cutout', False): rule build_cutout: output: directory("cutouts/{cutout}") + log: "logs/build_cutout.log" resources: mem=config['atlite'].get('nprocesses', 4) * 1000 threads: config['atlite'].get('nprocesses', 4) benchmark: "benchmarks/build_cutout_{cutout}" @@ -122,6 +129,7 @@ if config['enable'].get('build_cutout', False): else: rule retrieve_cutout: output: directory(expand("cutouts/{cutouts}", **config['atlite'])), + log: "logs/retrieve_cutout.log" script: 'scripts/retrieve_cutout.py' @@ -131,10 +139,12 @@ if config['enable'].get('build_natura_raster', False): natura="data/bundle/natura/Natura2000_end2015.shp", cutouts=expand("cutouts/{cutouts}", **config['atlite']) output: "resources/natura.tiff" + log: "logs/build_natura_raster.log" script: "scripts/build_natura_raster.py" else: rule retrieve_natura_raster: output: "resources/natura.tiff" + log: "logs/retrieve_natura_raster.log" script: 'scripts/retrieve_natura_raster.py' rule build_renewable_profiles: @@ -152,6 +162,7 @@ rule build_renewable_profiles: else "resources/regions_offshore.geojson"), cutout=lambda wildcards: "cutouts/" + config["renewable"][wildcards.technology]['cutout'] output: profile="resources/profile_{technology}.nc", + log: "logs/build_renewable_profile_{technology}.log" resources: mem=config['atlite'].get('nprocesses', 2) * 5000 threads: config['atlite'].get('nprocesses', 2) benchmark: "benchmarks/build_renewable_profiles_{technology}" @@ -165,6 +176,7 @@ if 'hydro' in config['renewable'].keys(): eia_hydro_generation='data/bundle/EIA_hydro_generation_2000_2014.csv', cutout="cutouts/" + config["renewable"]['hydro']['cutout'] output: 'resources/profile_hydro.nc' + log: "logs/build_hydro_profile.log" resources: mem=5000 # group: 'feedin_preparation' script: 'scripts/build_hydro_profile.py' @@ -182,6 +194,7 @@ rule add_electricity: **{'profile_' + t: "resources/profile_" + t + ".nc" for t in config['renewable']} output: "networks/elec.nc" + log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" threads: 1 resources: mem=3000 @@ -199,6 +212,7 @@ rule simplify_network: regions_onshore="resources/regions_onshore_{network}_s{simpl}.geojson", regions_offshore="resources/regions_offshore_{network}_s{simpl}.geojson", clustermaps='resources/clustermaps_{network}_s{simpl}.h5' + log: "logs/simplify_network/{network}_s{simpl}.log" benchmark: "benchmarks/simplify_network/{network}_s{simpl}" threads: 1 resources: mem=4000 @@ -217,6 +231,7 @@ rule cluster_network: regions_onshore="resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson", regions_offshore="resources/regions_offshore_{network}_s{simpl}_{clusters}.geojson", clustermaps='resources/clustermaps_{network}_s{simpl}_{clusters}.h5' + log: "logs/cluster_network/{network}_s{simpl}_{clusters}.log" benchmark: "benchmarks/cluster_network/{network}_s{simpl}_{clusters}" threads: 1 resources: mem=3000 @@ -229,6 +244,7 @@ rule add_extra_components: network='networks/{network}_s{simpl}_{clusters}.nc', tech_costs=COSTS, output: 'networks/{network}_s{simpl}_{clusters}_ec.nc' + log: "logs/add_extra_components/{network}_s{simpl}_{clusters}.log" benchmark: "benchmarks/add_extra_components/{network}_s{simpl}_{clusters}_ec" threads: 1 resources: mem=3000 @@ -239,6 +255,7 @@ rule add_extra_components: rule prepare_network: input: 'networks/{network}_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS output: 'networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' + log: "logs/prepare_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" threads: 1 resources: mem=1000 # benchmark: "benchmarks/prepare_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}" @@ -262,9 +279,9 @@ rule solve_network: output: "results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" shadow: "shallow" log: - solver=normpath("logs/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), - python="logs/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", - memory="logs/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log" + solver=normpath("logs/solve_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), + python="logs/solve_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", + memory="logs/solve_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log" benchmark: "benchmarks/solve_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 resources: mem=memory @@ -303,6 +320,7 @@ rule plot_network: output: only_map="results/plots/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", ext="results/plots/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_ext.{ext}" + log: "logs/plot_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_{ext}.log" script: "scripts/plot_network.py" def input_make_summary(w): @@ -323,11 +341,13 @@ def input_make_summary(w): rule make_summary: input: input_make_summary output: directory("results/summaries/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}") + log: "logs/make_summary/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log", script: "scripts/make_summary.py" rule plot_summary: input: "results/summaries/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}" output: "results/plots/summary_{summary}_{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}" + log: "logs/plot_summary/{summary}_{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log" script: "scripts/plot_summary.py" def input_plot_p_nom_max(wildcards): @@ -337,6 +357,7 @@ def input_plot_p_nom_max(wildcards): rule plot_p_nom_max: input: input_plot_p_nom_max output: "results/plots/{network}_s{simpl}_cum_p_nom_max_{clusters}_{technology}_{country}.{ext}" + log: "logs/plot_p_nom_max/{network}_s{simpl}_{clusters}_{technology}_{country}_{ext}.log" script: "scripts/plot_p_nom_max.py" rule build_country_flh: @@ -360,6 +381,7 @@ rule build_country_flh: uncorrected="resources/country_flh_uncorrected_{technology}.csv", plot="resources/country_flh_{technology}.pdf", exclusion=directory("resources/country_exclusion_{technology}") + log: "logs/build_country_flh_{technology}.log" resources: mem=10000 benchmark: "benchmarks/build_country_flh_{technology}" # group: 'feedin_preparation' diff --git a/config.default.yaml b/config.default.yaml index d90f0d84..5be7c91f 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -1,6 +1,9 @@ version: 0.1 tutorial: false -logging_level: INFO + +logging: + level: INFO + format: '%(levelname)s:%(name)s:%(message)s' summary_dir: results diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 061ba745..04803058 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -1,6 +1,8 @@ version: 0.1 tutorial: true -logging_level: INFO +logging: + level: INFO + format: '%(levelname)s:%(name)s:%(message)s' summary_dir: results diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv index 63db0ef4..c2577729 100644 --- a/doc/configtables/toplevel.csv +++ b/doc/configtables/toplevel.csv @@ -1,7 +1,9 @@ ,Unit,Values,Description version,--,0.1,"Version of PyPSA-Eur" tutorial,bool,"{true, false}","Switch to retrieve the tutorial data set instead of the full data set." -logging_level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only" +logging,,, +-- level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only" +-- format,--,"e.g. ``%(levelname)s:%(name)s:%(message)s``","Custom format for log messages. See `LogRecord `_ which should be included in the energy system model." focus_weights,--,"Keys should be two-digit country codes (e.g. DE) and values should range between 0 and 1","Ratio of total clusters for particular countries. the remaining weight is distributed according to mean load. An example: ``focus_weights: DE: 0.6 FR: 0.2``." diff --git a/doc/configuration.rst b/doc/configuration.rst index f196238d..c13cf03c 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -13,7 +13,7 @@ Top-level configuration .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 1-6,14 + :lines: 1-8,17 .. csv-table:: :header-rows: 1 @@ -37,7 +37,7 @@ facilitate running multiple scenarios through a single command snakemake solve_all_elec_networks -For each wildcard, a **list of values** is provided. The rule ``solve_all_elec_networks`` will trigger the rules for creating ``results/networks/elec_s{simpl}_{clusters}_l{ll}_{opts}.nc`` for **all combinations** of the provided wildcard values as defined by Python's `itertools.product(...) `_ function that snakemake's `expand(...) function `_ uses. +For each wildcard, a **list of values** is provided. The rule ``solve_all_elec_networks`` will trigger the rules for creating ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc`` for **all combinations** of the provided wildcard values as defined by Python's `itertools.product(...) `_ function that snakemake's `expand(...) function `_ uses. An exemplary dependency graph (starting from the simplification rules) then looks like this: @@ -45,7 +45,7 @@ An exemplary dependency graph (starting from the simplification rules) then look .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 7-12 + :lines: 10-15 .. csv-table:: :header-rows: 1 @@ -61,7 +61,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 16-19 + :lines: 19-22 .. csv-table:: :header-rows: 1 @@ -75,7 +75,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 26-42 + :lines: 30-47 .. csv-table:: :header-rows: 1 @@ -92,7 +92,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 50-63 + :lines: 55-68 .. csv-table:: :header-rows: 1 @@ -109,7 +109,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 65-82 + :lines: 70-87 .. csv-table:: :header-rows: 1 @@ -121,7 +121,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 65,83-95 + :lines: 70,88-100 .. csv-table:: :header-rows: 1 @@ -133,7 +133,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 65,96-109 + :lines: 70,101-114 .. csv-table:: :header-rows: 1 @@ -145,7 +145,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 65,110-129 + :lines: 70,115-134 .. csv-table:: :header-rows: 1 @@ -157,7 +157,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 65,130-136 + :lines: 70,135-141 .. csv-table:: :header-rows: 1 @@ -171,7 +171,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 138-145 + :lines: 133-150 .. csv-table:: :header-rows: 1 @@ -185,7 +185,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 147-150 + :lines: 152-155 .. csv-table:: :header-rows: 1 @@ -199,7 +199,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 152-155 + :lines: 157-160 .. csv-table:: :header-rows: 1 @@ -213,7 +213,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 157-158 + :lines: 162-163 .. csv-table:: :header-rows: 1 @@ -227,7 +227,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 160-172 + :lines: 165-177 .. csv-table:: :header-rows: 1 @@ -249,7 +249,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 174-182 + :lines: 179-187 .. csv-table:: :header-rows: 1 @@ -261,7 +261,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 174,183-199 + :lines: 179,188-204 .. csv-table:: :header-rows: 1 @@ -275,7 +275,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 201-335 + :lines: 206-340 .. csv-table:: :header-rows: 1 diff --git a/doc/tutorial.rst b/doc/tutorial.rst index b503fdd7..4f543ed8 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -38,47 +38,47 @@ The model can be adapted to only include selected countries (e.g. Germany) inste .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 14 + :lines: 15 Likewise, the example's temporal scope can be restricted (e.g. to a single month). .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 16-19 + :lines: 17-20 It is also possible to allow less or more carbon-dioxide emissions. Here, we limit the emissions of Germany 100 Megatonnes per year. .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 28 + :lines: 30 PyPSA-Eur also includes a database of existing conventional powerplants. We can select which types of powerplants we like to be included with fixed capacities: .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 40 + :lines: 43 To accurately model the temporal and spatial availability of renewables such as wind and solar energy, we rely on historical weather data. It is advisable to adapt the required range of coordinates to the selection of countries. .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 42-50 + :lines: 45-53 We can also decide which weather data source should be used to calculate potentials and capacity factor time-series for each carrier. For example, we may want to use the ERA-5 dataset for solar and not the default SARAH-2 dataset. .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 52,95-96 + :lines: 55,98-99 Finally, it is possible to pick a solver. For instance, this tutorial uses the open-source solvers CBC and Ipopt and does not rely on the commercial solvers Gurobi or CPLEX (for which free academic licenses are available). .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 151,160-161 + :lines: 154,163-164 .. note:: @@ -110,8 +110,8 @@ orders ``snakemake`` to run the script ``solve_network`` that produces the solve .. code:: rule solve_network: - input: "networks/{network}_s{simpl}_{clusters}_l{ll}_{opts}.nc" - output: "results/networks/{network}_s{simpl}_{clusters}_l{ll}_{opts}.nc" + input: "networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" [...] script: "scripts/solve_network.py" diff --git a/environment.docs.yaml b/environment.docs.yaml index 7cec5e9e..78fa7032 100644 --- a/environment.docs.yaml +++ b/environment.docs.yaml @@ -17,7 +17,7 @@ dependencies: - memory_profiler - yaml - pytables - - powerplantmatching>=0.4.2 + - powerplantmatching>=0.4.3 # Second order dependencies which should really be deps of atlite - xarray diff --git a/environment.yaml b/environment.yaml index 09e9ca84..a8dc8c20 100644 --- a/environment.yaml +++ b/environment.yaml @@ -18,7 +18,7 @@ dependencies: - memory_profiler - yaml - pytables - - powerplantmatching>=0.4.2 + - powerplantmatching>=0.4.3 # Second order dependencies which should really be deps of atlite - xarray diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 4146dab4..faf5c609 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1,17 +1,48 @@ import pandas as pd -from six import iterkeys, itervalues -import urllib -from progressbar import ProgressBar -import pypsa +def configure_logging(snakemake, skip_handlers=False): + """ + Configure the basic behaviour for the logging module. -from add_electricity import load_costs, update_transmission_costs + Note: Must only be called once from the __main__ section of a script. + + The setup includes printing log messages to STDERR and to a log file defined + by either (in priority order): snakemake.log.python, snakemake.log[0] or "logs/{rulename}.log". + Additional keywords from logging.basicConfig are accepted via the snakemake configuration + file under snakemake.config.logging. + + Parameters + ---------- + snakemake : snakemake object + Your snakemake object containing a snakemake.config and snakemake.log. + skip_handlers : True | False (default) + Do (not) skip the default handlers created for redirecting output to STDERR and file. + """ + + import logging + + kwargs = snakemake.config.get('logging', dict()) + kwargs.setdefault("level", "INFO") + + if skip_handlers is False: + kwargs.update( + {'handlers': [ + # Prefer the 'python' log, otherwise take the first log for each + # Snakemake rule + logging.FileHandler(snakemake.log.get('python', snakemake.log[0] if snakemake.log else f"logs/{snakemake.rule}.log")), + logging.StreamHandler() + ] + }) + logging.basicConfig(**kwargs) def pdbcast(v, h): return pd.DataFrame(v.values.reshape((-1, 1)) * h.values, index=v.index, columns=h.index) def load_network(fn, tech_costs, config, combine_hydro_ps=True): + import pypsa + from add_electricity import update_transmission_costs, load_costs + opts = config['plotting'] n = pypsa.Network(fn) @@ -73,6 +104,8 @@ def aggregate_p_curtailed(n): ]) def aggregate_costs(n, flatten=False, opts=None, existing_only=False): + from six import iterkeys, itervalues + components = dict(Link=("p_nom", "p0"), Generator=("p_nom", "p"), StorageUnit=("p_nom", "p"), @@ -107,6 +140,9 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False): return costs def progress_retrieve(url, file): + import urllib + from progressbar import ProgressBar + pbar = ProgressBar(0, 100) def dlProgress(count, blockSize, totalSize): diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index a78ab1cb..31ae15a8 100644 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -91,6 +91,9 @@ from vresutils.load import timeseries_opsd from vresutils import transfer as vtransfer import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + import pandas as pd import numpy as np import xarray as xr @@ -99,7 +102,6 @@ import pypsa import powerplantmatching as ppm idx = pd.IndexSlice -logger = logging.getLogger(__name__) def normed(s): return s/s.sum() @@ -522,7 +524,7 @@ if __name__ == "__main__": for t in snakemake.config['renewable']}) ) - logging.basicConfig(level=snakemake.config['logging_level']) + configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) Nyears = n.snapshot_weightings.sum()/8760. diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 41454edd..5c238a4c 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -45,16 +45,16 @@ The rule :mod:`add_extra_components` attaches additional extendable components t - ``Stores`` of carrier 'H2' and/or 'battery' in combination with ``Links``. If this option is chosen, the script adds extra buses with corresponding carrier where energy ``Stores`` are attached and which are connected to the corresponding power buses via two links, one each for charging and discharging. This leads to three investment variables for the energy capacity, charging and discharging capacity of the storage unit. """ - import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + import pandas as pd import pypsa from add_electricity import (load_costs, normed, add_nice_carrier_names, _add_missing_carriers_from_costs) idx = pd.IndexSlice -logger = logging.getLogger(__name__) - def attach_storageunits(n, costs): elec_opts = snakemake.config['electricity'] @@ -150,7 +150,7 @@ if __name__ == "__main__": Dict(network='networks/elec_s_5.nc', tech_costs='data/costs.csv')) - logging.basicConfig(level=snakemake.config['logging_level']) + configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) Nyears = n.snapshot_weightings.sum()/8760. diff --git a/scripts/base_network.py b/scripts/base_network.py index ea3610a4..547c1384 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -58,6 +58,10 @@ Description """ +import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + import yaml import pandas as pd import geopandas as gpd @@ -72,9 +76,6 @@ import shapely, shapely.prepared, shapely.wkt import networkx as nx -import logging -logger = logging.getLogger(__name__) - import pypsa def _get_oid(df): @@ -569,7 +570,7 @@ if __name__ == "__main__": output = ['networks/base.nc'] ) - logging.basicConfig(level=snakemake.config['logging_level']) + configure_logging(snakemake) n = base_network() n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 2b95c070..6a2a075c 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -36,6 +36,11 @@ Description ----------- """ + +import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + from vresutils.graph import voronoi_partition_pts import os @@ -43,12 +48,10 @@ import os import pandas as pd import geopandas as gpd - import pypsa -import logging if __name__ == "__main__": - logging.basicConfig(level=snakemake.config["logging_level"]) + configure_logging(snakemake) countries = snakemake.config['countries'] diff --git a/scripts/build_country_flh.py b/scripts/build_country_flh.py index 5aae088e..a2c22c92 100644 --- a/scripts/build_country_flh.py +++ b/scripts/build_country_flh.py @@ -57,6 +57,10 @@ Description """ +import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + import os import atlite import numpy as np @@ -72,8 +76,6 @@ from vresutils import landuse as vlanduse from vresutils.array import spdiag import progressbar as pgb -import logging -logger = logging.getLogger(__name__) from build_renewable_profiles import init_globals, calculate_potential @@ -175,7 +177,8 @@ if __name__ == '__main__': snakemake.config["renewable"][snakemake.wildcards.technology]['cutout']) pgb.streams.wrap_stderr() - logging.basicConfig(level=snakemake.config['logging_level']) + + configure_logging(snakemake) config = snakemake.config['renewable'][snakemake.wildcards.technology] diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 02a69df4..1dec0196 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -86,13 +86,16 @@ Description ----------- """ -import os -import atlite + import logging logger = logging.getLogger(__name__) +from _helpers import configure_logging + +import os +import atlite if __name__ == "__main__": - logging.basicConfig(level=snakemake.config['logging_level']) + configure_logging(snakemake) cutout_params = snakemake.config['atlite']['cutouts'][snakemake.wildcards.cutout] for p in ('xs', 'ys', 'years', 'months'): diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 485d3eff..a8fac2d8 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -54,16 +54,18 @@ Description :mod:`build_renewable_profiles` """ +import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + import os import atlite import geopandas as gpd from vresutils import hydro as vhydro -import logging - if __name__ == "__main__": - logging.basicConfig(level=snakemake.config['logging_level']) - + configure_logging(snakemake) + config = snakemake.config['renewable']['hydro'] cutout = atlite.Cutout(config['cutout'], cutout_dir=os.path.dirname(snakemake.input.cutout)) diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py index 99d78c4a..5677b5f5 100644 --- a/scripts/build_natura_raster.py +++ b/scripts/build_natura_raster.py @@ -35,6 +35,10 @@ Description """ +import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + import numpy as np import atlite import geokit as gk @@ -47,6 +51,8 @@ def determine_cutout_xXyY(cutout_name): return [x - dx/2., X + dx/2., y - dy/2., Y + dy/2.] if __name__ == "__main__": + configure_logging(snakemake) + cutout_names = np.unique([res['cutout'] for res in snakemake.config['renewable'].values()]) xs, Xs, ys, Ys = zip(*(determine_cutout_xXyY(cutout) for cutout in cutout_names)) xXyY = min(xs), max(Xs), min(ys), max(Ys) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 6d52a440..9bc6462e 100644 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -68,15 +68,15 @@ The configuration options ``electricity: powerplants_filter`` and ``electricity: """ import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + from scipy.spatial import cKDTree as KDTree import pypsa import powerplantmatching as pm import pandas as pd -logger = logging.getLogger(__name__) - - def add_custom_powerplants(ppl): custom_ppl_query = snakemake.config['electricity']['custom_powerplants'] if not custom_ppl_query: @@ -88,6 +88,7 @@ def add_custom_powerplants(ppl): if __name__ == "__main__": + if 'snakemake' not in globals(): from vresutils.snakemake import MockSnakemake, Dict @@ -97,7 +98,7 @@ if __name__ == "__main__": output=['resources/powerplants.csv'] ) - logging.basicConfig(level=snakemake.config['logging_level']) + configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) countries = n.buses.country.unique() diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index dc1252ae..857870ef 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -150,6 +150,9 @@ node (`p_nom_max`): ``simple`` and ``conservative``: reached. """ +import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging import matplotlib.pyplot as plt @@ -170,8 +173,6 @@ from vresutils import landuse as vlanduse from vresutils.array import spdiag import progressbar as pgb -import logging -logger = logging.getLogger(__name__) bounds = dx = dy = config = paths = gebco = clc = natura = None def init_globals(bounds_xXyY, n_dx, n_dy, n_config, n_paths): @@ -239,7 +240,8 @@ def calculate_potential(gid, save_map=None): if __name__ == '__main__': pgb.streams.wrap_stderr() - logging.basicConfig(level=snakemake.config['logging_level']) + + configure_logging(snakemake) config = snakemake.config['renewable'][snakemake.wildcards.technology] diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 9105c51c..9734e7e9 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -63,6 +63,10 @@ Description """ +import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + import os import numpy as np from operator import attrgetter @@ -221,6 +225,8 @@ if __name__ == "__main__": ) ) + configure_logging(snakemake) + country_shapes = countries() save_to_geojson(country_shapes, snakemake.output.country_shapes) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index d7f32dca..3c67b4b3 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -91,11 +91,12 @@ Description """ -import pandas as pd -idx = pd.IndexSlice - import logging logger = logging.getLogger(__name__) +from _helpers import configure_logging + +import pandas as pd +idx = pd.IndexSlice import os import numpy as np @@ -311,8 +312,8 @@ if __name__ == "__main__": clustermaps='resources/clustermaps_{network}_s{simpl}_{clusters}.h5' ) ) - - logging.basicConfig(level=snakemake.config['logging_level']) + + configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index d3bed616..68d3b5fb 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -49,7 +49,12 @@ Replacing '/summaries/' with '/plots/' creates nice colored maps of the results. """ +import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + import os + from six import iteritems import pandas as pd @@ -478,7 +483,9 @@ if __name__ == "__main__": else: ll = [snakemake.wildcards.ll] - networks_dict = {(simpl,clusters,l,opts) : ('results/networks/{network}_s{simpl}_{clusters}_l{ll}_{opts}.nc' + configure_logging(snakemake) + + networks_dict = {(simpl,clusters,l,opts) : ('results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' .format(network=snakemake.wildcards.network, simpl=simpl, clusters=clusters, diff --git a/scripts/plot_network.py b/scripts/plot_network.py index ec4b8e40..e9e5c2fe 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -15,12 +15,13 @@ Description """ -from _helpers import load_network, aggregate_p, aggregate_costs +import logging +logger = logging.getLogger(__name__) +from _helpers import load_network, aggregate_p, aggregate_costs, configure_logging import pandas as pd import numpy as np from six.moves import zip -import logging import cartopy.crs as ccrs import matplotlib.pyplot as plt @@ -257,8 +258,8 @@ if __name__ == "__main__": ext="results/plots/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{attr}_ext.{ext}") ) - logging.basicConfig(level=snakemake.config['logging_level']) - + configure_logging(snakemake) + set_plot_style() opts = snakemake.config['plotting'] diff --git a/scripts/plot_p_nom_max.py b/scripts/plot_p_nom_max.py index 92f9e49a..3b2134f3 100644 --- a/scripts/plot_p_nom_max.py +++ b/scripts/plot_p_nom_max.py @@ -14,8 +14,12 @@ Description ----------- """ +import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging import pypsa + import pandas as pd import matplotlib.pyplot as plt @@ -56,8 +60,8 @@ if __name__ == "__main__": ), output=['results/plots/cum_p_nom_max_{clusters}_{country}.pdf'] ) - - logging.basicConfig(level=snakemake.config['logging_level']) + + configure_logging(snakemake) plot_kwds = dict(drawstyle="steps-post") diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 9a7aa7e4..2464a054 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -16,6 +16,9 @@ Description """ import os +import logging +logger = logging.getLogger(__name__) + import pandas as pd import matplotlib.pyplot as plt @@ -178,6 +181,9 @@ def plot_energy(infn, fn=None): if __name__ == "__main__": + + configure_logging(snakemake) + summary = snakemake.wildcards.summary try: func = globals()[f"plot_{summary}"] diff --git a/scripts/prepare_links_p_nom.py b/scripts/prepare_links_p_nom.py index 57fa2687..300277fa 100644 --- a/scripts/prepare_links_p_nom.py +++ b/scripts/prepare_links_p_nom.py @@ -31,9 +31,16 @@ Description """ +import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + import pandas as pd if __name__ == "__main__": + + configure_logging(snakemake) + links_p_nom = pd.read_html('https://en.wikipedia.org/wiki/List_of_HVDC_projects', header=0, match="SwePol")[0] def extract_coordinates(s): diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 95e3b887..2a228321 100644 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -38,7 +38,7 @@ Inputs Outputs ------- -- ``networks/{network}_s{simpl}_{clusters}_l{ll}_{opts}.nc``: Complete PyPSA network that will be handed to the ``solve_network`` rule. +- ``networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: Complete PyPSA network that will be handed to the ``solve_network`` rule. Description ----------- @@ -50,6 +50,10 @@ Description """ +import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + from add_electricity import load_costs, update_transmission_costs from six import iteritems @@ -57,10 +61,8 @@ import numpy as np import re import pypsa import pandas as pd -import logging idx = pd.IndexSlice -logger = logging.getLogger(__name__) def add_co2limit(n, Nyears=1., factor=None): @@ -182,10 +184,10 @@ if __name__ == "__main__": snakemake = MockSnakemake( wildcards=dict(network='elec', simpl='', clusters='37', ll='v2', opts='Co2L-3H'), input=['networks/{network}_s{simpl}_{clusters}.nc'], - output=['networks/{network}_s{simpl}_{clusters}_l{ll}_{opts}.nc'] + output=['networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc'] ) - logging.basicConfig(level=snakemake.config['logging_level']) + configure_logging(snakemake) opts = snakemake.wildcards.opts.split('-') diff --git a/scripts/retrieve_cutout.py b/scripts/retrieve_cutout.py index d6ad733f..226d612e 100644 --- a/scripts/retrieve_cutout.py +++ b/scripts/retrieve_cutout.py @@ -37,22 +37,31 @@ The :ref:`tutorial` uses smaller `cutouts