diff --git a/Snakefile b/Snakefile index cd42835e..fb04256b 100644 --- a/Snakefile +++ b/Snakefile @@ -15,26 +15,27 @@ ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4) wildcard_constraints: + year="[0-9]*", simpl="[a-zA-Z0-9]*|all", clusters="[0-9]+m?|all", ll="(v|c)([0-9\.]+|opt|all)|all", opts="[-+a-zA-Z0-9\.]*" -rule cluster_all_elec_networks: - input: expand("networks/elec_s{simpl}_{clusters}.nc", **config['scenario']) +rule cluster_all_networks: + input: expand("networks/elec{year}_s{simpl}_{clusters}.nc", **config['scenario']) -rule extra_components_all_elec_networks: - input: expand("networks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) +rule extra_components_all_networks: + input: expand("networks/elec{year}_s{simpl}_{clusters}_ec.nc", **config['scenario']) -rule prepare_all_elec_networks: - input: expand("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) +rule prepare_all_networks: + input: expand("networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) -rule solve_all_elec_networks: - input: expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) +rule solve_all_networks: + input: expand("results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) if config['enable'].get('prepare_links_p_nom', False): @@ -180,9 +181,9 @@ rule build_renewable_profiles: else "resources/regions_offshore.geojson"), cutout=lambda w: "cutouts/" + config["renewable"][w.technology]['cutout'] output: - profile="resources/profile_{technology}.nc", - log: "logs/build_renewable_profile_{technology}.log" - benchmark: "benchmarks/build_renewable_profiles_{technology}" + profile="resources/profile{year}_{technology}.nc", + log: "logs/build_renewable_profile{year}_{technology}.log" + benchmark: "benchmarks/build_renewable_profiles{year}_{technology}" threads: ATLITE_NPROCESSES resources: mem=ATLITE_NPROCESSES * 5000 script: "scripts/build_renewable_profiles.py" @@ -194,8 +195,8 @@ if 'hydro' in config['renewable'].keys(): country_shapes='resources/country_shapes.geojson', eia_hydro_generation='data/bundle/EIA_hydro_generation_2000_2014.csv', cutout="cutouts/" + config["renewable"]['hydro']['cutout'] - output: 'resources/profile_hydro.nc' - log: "logs/build_hydro_profile.log" + output: 'resources/profile{year}_hydro.nc' + log: "logs/build_hydro_profile{year}.log" resources: mem=5000 script: 'scripts/build_hydro_profile.py' @@ -210,11 +211,11 @@ rule add_electricity: geth_hydro_capacities='data/geth2015_hydro_capacities.csv', opsd_load='data/bundle/time_series_60min_singleindex_filtered.csv', nuts3_shapes='resources/nuts3_shapes.geojson', - **{f"profile_{tech}": f"resources/profile_{tech}.nc" + **{f"profile_{tech}": "resources/profile{year}_" + f"{tech}.nc" for tech in config['renewable']} - output: "networks/elec.nc" - log: "logs/add_electricity.log" - benchmark: "benchmarks/add_electricity" + output: "networks/elec{year}.nc" + log: "logs/add_electricity{year}.log" + benchmark: "benchmarks/add_electricity{year}" threads: 1 resources: mem=3000 script: "scripts/add_electricity.py" @@ -222,17 +223,17 @@ rule add_electricity: rule simplify_network: input: - network='networks/elec.nc', + network='networks/elec{year}.nc', tech_costs=COSTS, regions_onshore="resources/regions_onshore.geojson", regions_offshore="resources/regions_offshore.geojson" output: - network='networks/elec_s{simpl}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", - busmap='resources/busmap_elec_s{simpl}.csv' - log: "logs/simplify_network/elec_s{simpl}.log" - benchmark: "benchmarks/simplify_network/elec_s{simpl}" + network='networks/elec{year}_s{simpl}.nc', + regions_onshore="resources/regions_onshore_elec{year}_s{simpl}.geojson", + regions_offshore="resources/regions_offshore_elec{year}_s{simpl}.geojson", + busmap='resources/busmap_elec{year}_s{simpl}.csv' + log: "logs/simplify_network/elec{year}_s{simpl}.log" + benchmark: "benchmarks/simplify_network/elec{year}_s{simpl}" threads: 1 resources: mem=4000 script: "scripts/simplify_network.py" @@ -240,19 +241,19 @@ rule simplify_network: rule cluster_network: input: - network='networks/elec_s{simpl}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", - busmap=ancient('resources/busmap_elec_s{simpl}.csv'), + network='networks/elec{year}_s{simpl}.nc', + regions_onshore="resources/regions_onshore_elec{year}_s{simpl}.geojson", + regions_offshore="resources/regions_offshore_elec{year}_s{simpl}.geojson", + busmap=ancient('resources/busmap_elec{year}_s{simpl}.csv'), tech_costs=COSTS output: - network='networks/elec_s{simpl}_{clusters}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", - busmap="resources/busmap_elec_s{simpl}_{clusters}.csv", - linemap="resources/linemap_elec_s{simpl}_{clusters}.csv" - log: "logs/cluster_network/elec_s{simpl}_{clusters}.log" - benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}" + network='networks/elec{year}_s{simpl}_{clusters}.nc', + regions_onshore="resources/regions_onshore_elec{year}_s{simpl}_{clusters}.geojson", + regions_offshore="resources/regions_offshore_elec{year}_s{simpl}_{clusters}.geojson", + busmap="resources/busmap_elec{year}_s{simpl}_{clusters}.csv", + linemap="resources/linemap_elec{year}_s{simpl}_{clusters}.csv" + log: "logs/cluster_network/elec{year}_s{simpl}_{clusters}.log" + benchmark: "benchmarks/cluster_network/elec{year}_s{simpl}_{clusters}" threads: 1 resources: mem=3000 script: "scripts/cluster_network.py" @@ -260,21 +261,21 @@ rule cluster_network: rule add_extra_components: input: - network='networks/elec_s{simpl}_{clusters}.nc', + network='networks/elec{year}_s{simpl}_{clusters}.nc', tech_costs=COSTS, - output: 'networks/elec_s{simpl}_{clusters}_ec.nc' - log: "logs/add_extra_components/elec_s{simpl}_{clusters}.log" - benchmark: "benchmarks/add_extra_components/elec_s{simpl}_{clusters}_ec" + output: 'networks/elec{year}_s{simpl}_{clusters}_ec.nc' + log: "logs/add_extra_components/elec{year}_s{simpl}_{clusters}.log" + benchmark: "benchmarks/add_extra_components/elec{year}_s{simpl}_{clusters}_ec" threads: 1 resources: mem=3000 script: "scripts/add_extra_components.py" rule prepare_network: - input: 'networks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS - output: 'networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' - log: "logs/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" - benchmark: "benchmarks/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" + input: 'networks/elec{year}_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS + output: 'networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' + log: "logs/prepare_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" + benchmark: "benchmarks/prepare_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 1 resources: mem=1000 script: "scripts/prepare_network.py" @@ -294,13 +295,13 @@ def memory(w): rule solve_network: - input: "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + input: "networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" log: - solver=normpath("logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), - python="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", - memory="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log" - benchmark: "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" + solver=normpath("logs/solve_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), + python="logs/solve_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", + memory="logs/solve_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log" + benchmark: "benchmarks/solve_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 resources: mem=memory shadow: "shallow" @@ -309,14 +310,14 @@ rule solve_network: rule solve_operations_network: input: - unprepared="networks/elec_s{simpl}_{clusters}_ec.nc", - optimized="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" + unprepared="networks/elec{year}_s{simpl}_{clusters}_ec.nc", + optimized="results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" log: - solver=normpath("logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), - python="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", - memory="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log" - benchmark: "benchmarks/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" + solver=normpath("logs/solve_operations_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), + python="logs/solve_operations_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", + memory="logs/solve_operations_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log" + benchmark: "benchmarks/solve_operations_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 resources: mem=(lambda w: 5000 + 372 * int(w.clusters)) shadow: "shallow" @@ -325,12 +326,12 @@ rule solve_operations_network: rule plot_network: input: - network="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network="results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", tech_costs=COSTS output: - only_map="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", - ext="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_ext.{ext}" - log: "logs/plot_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_{ext}.log" + only_map="results/plots/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", + ext="results/plots/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_ext.{ext}" + log: "logs/plot_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_{ext}.log" script: "scripts/plot_network.py" @@ -343,7 +344,7 @@ def input_make_summary(w): else: ll = w.ll return ([COSTS] + - expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + expand("results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", network=w.network, ll=ll, **{k: config["scenario"][k] if getattr(w, k) == "all" else getattr(w, k) @@ -352,28 +353,28 @@ def input_make_summary(w): rule make_summary: input: input_make_summary - output: directory("results/summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}") - log: "logs/make_summary/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log", + output: directory("results/summaries/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}") + log: "logs/make_summary/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log", script: "scripts/make_summary.py" rule plot_summary: - input: "results/summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}" - output: "results/plots/summary_{summary}_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}" - log: "logs/plot_summary/{summary}_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log" + input: "results/summaries/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}" + output: "results/plots/{year}/summary_{summary}_elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}" + log: "logs/plot_summary/{summary}_elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log" script: "scripts/plot_summary.py" def input_plot_p_nom_max(w): - return [("networks/elec_s{simpl}{maybe_cluster}.nc" + return [("networks/elec{year}_s{simpl}{maybe_cluster}.nc" .format(maybe_cluster=('' if c == 'full' else ('_' + c)), **w)) for c in w.clusts.split(",")] rule plot_p_nom_max: input: input_plot_p_nom_max - output: "results/plots/elec_s{simpl}_cum_p_nom_max_{clusts}_{techs}_{country}.{ext}" - log: "logs/plot_p_nom_max/elec_s{simpl}_{clusts}_{techs}_{country}_{ext}.log" + output: "results/plots/elec{year}_s{simpl}_cum_p_nom_max_{clusts}_{techs}_{country}.{ext}" + log: "logs/plot_p_nom_max/elec{year}_s{simpl}_{clusts}_{techs}_{country}_{ext}.log" script: "scripts/plot_p_nom_max.py" diff --git a/config.default.yaml b/config.default.yaml index b46432c6..0ba49caf 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -12,6 +12,7 @@ logging: summary_dir: results scenario: + year: [''] # for backwards compatibility simpl: [''] ll: ['copt'] clusters: [37, 128, 256, 512, 1024] diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 174708cb..52b5b866 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -11,6 +11,7 @@ logging: summary_dir: results scenario: + year: [''] simpl: [''] ll: ['copt'] clusters: [5] diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index c11c27f5..a13cdb15 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -212,6 +212,7 @@ def attach_load(n): substation_lv_i = n.buses.index[n.buses['substation_lv']] regions = (gpd.read_file(snakemake.input.regions).set_index('name') .reindex(substation_lv_i)) + # TODO: is there a fallback if year not in timeseries_opsd? opsd_load = (timeseries_opsd(slice(*n.snapshots[[0,-1]].year.astype(str)), snakemake.input.opsd_load) * snakemake.config.get('load', {}).get('scaling_factor', 1.0)) @@ -522,6 +523,11 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) + + year = int(snakemake.wildcards.year) + snapshots = dict(start=str(year), end=str(year+1), closed="left") if year else snakememake.config['snapshots'] + n.set_snapshots(pd.date_range(freq='h', **snapshots)) + n.snapshot_weightings[:] *= 8760. / n.snapshot_weightings.sum() Nyears = n.snapshot_weightings.sum() / 8760. costs = load_costs(Nyears) diff --git a/scripts/base_network.py b/scripts/base_network.py index d2ceccf1..0b535218 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -11,8 +11,6 @@ Relevant Settings .. code:: yaml - snapshots: - countries: electricity: @@ -35,7 +33,7 @@ Relevant Settings .. seealso:: Documentation of the configuration file ``config.yaml`` at - :ref:`snapshots_cf`, :ref:`toplevel_cf`, :ref:`electricity_cf`, :ref:`load_cf`, + :ref:`toplevel_cf`, :ref:`electricity_cf`, :ref:`load_cf`, :ref:`lines_cf`, :ref:`links_cf`, :ref:`transformers_cf` Inputs @@ -554,9 +552,6 @@ def base_network(): n = pypsa.Network() n.name = 'PyPSA-Eur' - n.set_snapshots(pd.date_range(freq='h', **snakemake.config['snapshots'])) - n.snapshot_weightings[:] *= 8760. / n.snapshot_weightings.sum() - n.import_components_from_dataframe(buses, "Bus") n.import_components_from_dataframe(lines, "Line") n.import_components_from_dataframe(transformers, "Transformer") diff --git a/scripts/build_country_flh.py b/scripts/build_country_flh.py index 459b8f38..345d1d09 100644 --- a/scripts/build_country_flh.py +++ b/scripts/build_country_flh.py @@ -165,7 +165,9 @@ if __name__ == '__main__': config = snakemake.config['renewable'][snakemake.wildcards.technology] - time = pd.date_range(freq='m', **snakemake.config['snapshots']) + year = int(snakemake.wildcards.year) + snapshots = dict(start=str(year), end=str(year+1), closed="left") if year else snakememake.config['snapshots'] + time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) cutout = atlite.Cutout(config['cutout'], diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 1e55faf5..dc909c09 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -105,6 +105,7 @@ if __name__ == "__main__": snakemake = mock_snakemake('build_cutout', cutout='europe-2013-era5') configure_logging(snakemake) + # TODO make it accept year independent config cutout_params = snakemake.config['atlite']['cutouts'][snakemake.wildcards.cutout] for p in ('xs', 'ys', 'years', 'months'): if p in cutout_params: diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 71adb66e..9b477625 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -284,7 +284,9 @@ if __name__ == '__main__': config = snakemake.config['renewable'][snakemake.wildcards.technology] - time = pd.date_range(freq='m', **snakemake.config['snapshots']) + year = int(snakemake.wildcards.year) + snapshots = dict(start=str(year), end=str(year+1), closed="left") if year else snakememake.config['snapshots'] + time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) cutout = atlite.Cutout(config['cutout'], diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 84d551e6..a1fc59ba 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -11,6 +11,7 @@ logging: summary_dir: results scenario: + year: [''] simpl: [''] ll: ['copt'] clusters: [5]