first step towards sweeping through multiple weather years

This commit is contained in:
Fabian Neumann 2020-10-18 14:25:12 +02:00
parent bcd7f86b5c
commit cb2b78ad97
9 changed files with 87 additions and 77 deletions

139
Snakefile
View File

@ -15,26 +15,27 @@ ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4)
wildcard_constraints: wildcard_constraints:
year="[0-9]*",
simpl="[a-zA-Z0-9]*|all", simpl="[a-zA-Z0-9]*|all",
clusters="[0-9]+m?|all", clusters="[0-9]+m?|all",
ll="(v|c)([0-9\.]+|opt|all)|all", ll="(v|c)([0-9\.]+|opt|all)|all",
opts="[-+a-zA-Z0-9\.]*" opts="[-+a-zA-Z0-9\.]*"
rule cluster_all_elec_networks: rule cluster_all_networks:
input: expand("networks/elec_s{simpl}_{clusters}.nc", **config['scenario']) input: expand("networks/elec{year}_s{simpl}_{clusters}.nc", **config['scenario'])
rule extra_components_all_elec_networks: rule extra_components_all_networks:
input: expand("networks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) input: expand("networks/elec{year}_s{simpl}_{clusters}_ec.nc", **config['scenario'])
rule prepare_all_elec_networks: rule prepare_all_networks:
input: expand("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) input: expand("networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario'])
rule solve_all_elec_networks: rule solve_all_networks:
input: expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) input: expand("results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario'])
if config['enable'].get('prepare_links_p_nom', False): if config['enable'].get('prepare_links_p_nom', False):
@ -180,9 +181,9 @@ rule build_renewable_profiles:
else "resources/regions_offshore.geojson"), else "resources/regions_offshore.geojson"),
cutout=lambda w: "cutouts/" + config["renewable"][w.technology]['cutout'] cutout=lambda w: "cutouts/" + config["renewable"][w.technology]['cutout']
output: output:
profile="resources/profile_{technology}.nc", profile="resources/profile{year}_{technology}.nc",
log: "logs/build_renewable_profile_{technology}.log" log: "logs/build_renewable_profile{year}_{technology}.log"
benchmark: "benchmarks/build_renewable_profiles_{technology}" benchmark: "benchmarks/build_renewable_profiles{year}_{technology}"
threads: ATLITE_NPROCESSES threads: ATLITE_NPROCESSES
resources: mem=ATLITE_NPROCESSES * 5000 resources: mem=ATLITE_NPROCESSES * 5000
script: "scripts/build_renewable_profiles.py" script: "scripts/build_renewable_profiles.py"
@ -194,8 +195,8 @@ if 'hydro' in config['renewable'].keys():
country_shapes='resources/country_shapes.geojson', country_shapes='resources/country_shapes.geojson',
eia_hydro_generation='data/bundle/EIA_hydro_generation_2000_2014.csv', eia_hydro_generation='data/bundle/EIA_hydro_generation_2000_2014.csv',
cutout="cutouts/" + config["renewable"]['hydro']['cutout'] cutout="cutouts/" + config["renewable"]['hydro']['cutout']
output: 'resources/profile_hydro.nc' output: 'resources/profile{year}_hydro.nc'
log: "logs/build_hydro_profile.log" log: "logs/build_hydro_profile{year}.log"
resources: mem=5000 resources: mem=5000
script: 'scripts/build_hydro_profile.py' script: 'scripts/build_hydro_profile.py'
@ -210,11 +211,11 @@ rule add_electricity:
geth_hydro_capacities='data/geth2015_hydro_capacities.csv', geth_hydro_capacities='data/geth2015_hydro_capacities.csv',
opsd_load='data/bundle/time_series_60min_singleindex_filtered.csv', opsd_load='data/bundle/time_series_60min_singleindex_filtered.csv',
nuts3_shapes='resources/nuts3_shapes.geojson', nuts3_shapes='resources/nuts3_shapes.geojson',
**{f"profile_{tech}": f"resources/profile_{tech}.nc" **{f"profile_{tech}": "resources/profile{year}_" + f"{tech}.nc"
for tech in config['renewable']} for tech in config['renewable']}
output: "networks/elec.nc" output: "networks/elec{year}.nc"
log: "logs/add_electricity.log" log: "logs/add_electricity{year}.log"
benchmark: "benchmarks/add_electricity" benchmark: "benchmarks/add_electricity{year}"
threads: 1 threads: 1
resources: mem=3000 resources: mem=3000
script: "scripts/add_electricity.py" script: "scripts/add_electricity.py"
@ -222,17 +223,17 @@ rule add_electricity:
rule simplify_network: rule simplify_network:
input: input:
network='networks/elec.nc', network='networks/elec{year}.nc',
tech_costs=COSTS, tech_costs=COSTS,
regions_onshore="resources/regions_onshore.geojson", regions_onshore="resources/regions_onshore.geojson",
regions_offshore="resources/regions_offshore.geojson" regions_offshore="resources/regions_offshore.geojson"
output: output:
network='networks/elec_s{simpl}.nc', network='networks/elec{year}_s{simpl}.nc',
regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", regions_onshore="resources/regions_onshore_elec{year}_s{simpl}.geojson",
regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", regions_offshore="resources/regions_offshore_elec{year}_s{simpl}.geojson",
busmap='resources/busmap_elec_s{simpl}.csv' busmap='resources/busmap_elec{year}_s{simpl}.csv'
log: "logs/simplify_network/elec_s{simpl}.log" log: "logs/simplify_network/elec{year}_s{simpl}.log"
benchmark: "benchmarks/simplify_network/elec_s{simpl}" benchmark: "benchmarks/simplify_network/elec{year}_s{simpl}"
threads: 1 threads: 1
resources: mem=4000 resources: mem=4000
script: "scripts/simplify_network.py" script: "scripts/simplify_network.py"
@ -240,19 +241,19 @@ rule simplify_network:
rule cluster_network: rule cluster_network:
input: input:
network='networks/elec_s{simpl}.nc', network='networks/elec{year}_s{simpl}.nc',
regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", regions_onshore="resources/regions_onshore_elec{year}_s{simpl}.geojson",
regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", regions_offshore="resources/regions_offshore_elec{year}_s{simpl}.geojson",
busmap=ancient('resources/busmap_elec_s{simpl}.csv'), busmap=ancient('resources/busmap_elec{year}_s{simpl}.csv'),
tech_costs=COSTS tech_costs=COSTS
output: output:
network='networks/elec_s{simpl}_{clusters}.nc', network='networks/elec{year}_s{simpl}_{clusters}.nc',
regions_onshore="resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", regions_onshore="resources/regions_onshore_elec{year}_s{simpl}_{clusters}.geojson",
regions_offshore="resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", regions_offshore="resources/regions_offshore_elec{year}_s{simpl}_{clusters}.geojson",
busmap="resources/busmap_elec_s{simpl}_{clusters}.csv", busmap="resources/busmap_elec{year}_s{simpl}_{clusters}.csv",
linemap="resources/linemap_elec_s{simpl}_{clusters}.csv" linemap="resources/linemap_elec{year}_s{simpl}_{clusters}.csv"
log: "logs/cluster_network/elec_s{simpl}_{clusters}.log" log: "logs/cluster_network/elec{year}_s{simpl}_{clusters}.log"
benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}" benchmark: "benchmarks/cluster_network/elec{year}_s{simpl}_{clusters}"
threads: 1 threads: 1
resources: mem=3000 resources: mem=3000
script: "scripts/cluster_network.py" script: "scripts/cluster_network.py"
@ -260,21 +261,21 @@ rule cluster_network:
rule add_extra_components: rule add_extra_components:
input: input:
network='networks/elec_s{simpl}_{clusters}.nc', network='networks/elec{year}_s{simpl}_{clusters}.nc',
tech_costs=COSTS, tech_costs=COSTS,
output: 'networks/elec_s{simpl}_{clusters}_ec.nc' output: 'networks/elec{year}_s{simpl}_{clusters}_ec.nc'
log: "logs/add_extra_components/elec_s{simpl}_{clusters}.log" log: "logs/add_extra_components/elec{year}_s{simpl}_{clusters}.log"
benchmark: "benchmarks/add_extra_components/elec_s{simpl}_{clusters}_ec" benchmark: "benchmarks/add_extra_components/elec{year}_s{simpl}_{clusters}_ec"
threads: 1 threads: 1
resources: mem=3000 resources: mem=3000
script: "scripts/add_extra_components.py" script: "scripts/add_extra_components.py"
rule prepare_network: rule prepare_network:
input: 'networks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS input: 'networks/elec{year}_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS
output: 'networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' output: 'networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc'
log: "logs/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" log: "logs/prepare_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.log"
benchmark: "benchmarks/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" benchmark: "benchmarks/prepare_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}"
threads: 1 threads: 1
resources: mem=1000 resources: mem=1000
script: "scripts/prepare_network.py" script: "scripts/prepare_network.py"
@ -294,13 +295,13 @@ def memory(w):
rule solve_network: rule solve_network:
input: "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" input: "networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"
output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" output: "results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"
log: log:
solver=normpath("logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), solver=normpath("logs/solve_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"),
python="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", python="logs/solve_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log",
memory="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log" memory="logs/solve_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log"
benchmark: "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" benchmark: "benchmarks/solve_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}"
threads: 4 threads: 4
resources: mem=memory resources: mem=memory
shadow: "shallow" shadow: "shallow"
@ -309,14 +310,14 @@ rule solve_network:
rule solve_operations_network: rule solve_operations_network:
input: input:
unprepared="networks/elec_s{simpl}_{clusters}_ec.nc", unprepared="networks/elec{year}_s{simpl}_{clusters}_ec.nc",
optimized="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" optimized="results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"
output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" output: "results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc"
log: log:
solver=normpath("logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), solver=normpath("logs/solve_operations_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"),
python="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", python="logs/solve_operations_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log",
memory="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log" memory="logs/solve_operations_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log"
benchmark: "benchmarks/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" benchmark: "benchmarks/solve_operations_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}"
threads: 4 threads: 4
resources: mem=(lambda w: 5000 + 372 * int(w.clusters)) resources: mem=(lambda w: 5000 + 372 * int(w.clusters))
shadow: "shallow" shadow: "shallow"
@ -325,12 +326,12 @@ rule solve_operations_network:
rule plot_network: rule plot_network:
input: input:
network="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", network="results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
tech_costs=COSTS tech_costs=COSTS
output: output:
only_map="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", only_map="results/plots/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}",
ext="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_ext.{ext}" ext="results/plots/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_ext.{ext}"
log: "logs/plot_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_{ext}.log" log: "logs/plot_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_{ext}.log"
script: "scripts/plot_network.py" script: "scripts/plot_network.py"
@ -343,7 +344,7 @@ def input_make_summary(w):
else: else:
ll = w.ll ll = w.ll
return ([COSTS] + return ([COSTS] +
expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", expand("results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
network=w.network, network=w.network,
ll=ll, ll=ll,
**{k: config["scenario"][k] if getattr(w, k) == "all" else getattr(w, k) **{k: config["scenario"][k] if getattr(w, k) == "all" else getattr(w, k)
@ -352,28 +353,28 @@ def input_make_summary(w):
rule make_summary: rule make_summary:
input: input_make_summary input: input_make_summary
output: directory("results/summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}") output: directory("results/summaries/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}")
log: "logs/make_summary/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log", log: "logs/make_summary/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log",
script: "scripts/make_summary.py" script: "scripts/make_summary.py"
rule plot_summary: rule plot_summary:
input: "results/summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}" input: "results/summaries/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}"
output: "results/plots/summary_{summary}_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}" output: "results/plots/{year}/summary_{summary}_elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}"
log: "logs/plot_summary/{summary}_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log" log: "logs/plot_summary/{summary}_elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log"
script: "scripts/plot_summary.py" script: "scripts/plot_summary.py"
def input_plot_p_nom_max(w): def input_plot_p_nom_max(w):
return [("networks/elec_s{simpl}{maybe_cluster}.nc" return [("networks/elec{year}_s{simpl}{maybe_cluster}.nc"
.format(maybe_cluster=('' if c == 'full' else ('_' + c)), **w)) .format(maybe_cluster=('' if c == 'full' else ('_' + c)), **w))
for c in w.clusts.split(",")] for c in w.clusts.split(",")]
rule plot_p_nom_max: rule plot_p_nom_max:
input: input_plot_p_nom_max input: input_plot_p_nom_max
output: "results/plots/elec_s{simpl}_cum_p_nom_max_{clusts}_{techs}_{country}.{ext}" output: "results/plots/elec{year}_s{simpl}_cum_p_nom_max_{clusts}_{techs}_{country}.{ext}"
log: "logs/plot_p_nom_max/elec_s{simpl}_{clusts}_{techs}_{country}_{ext}.log" log: "logs/plot_p_nom_max/elec{year}_s{simpl}_{clusts}_{techs}_{country}_{ext}.log"
script: "scripts/plot_p_nom_max.py" script: "scripts/plot_p_nom_max.py"

View File

@ -12,6 +12,7 @@ logging:
summary_dir: results summary_dir: results
scenario: scenario:
year: [''] # for backwards compatibility
simpl: [''] simpl: ['']
ll: ['copt'] ll: ['copt']
clusters: [37, 128, 256, 512, 1024] clusters: [37, 128, 256, 512, 1024]

View File

@ -11,6 +11,7 @@ logging:
summary_dir: results summary_dir: results
scenario: scenario:
year: ['']
simpl: [''] simpl: ['']
ll: ['copt'] ll: ['copt']
clusters: [5] clusters: [5]

View File

@ -212,6 +212,7 @@ def attach_load(n):
substation_lv_i = n.buses.index[n.buses['substation_lv']] substation_lv_i = n.buses.index[n.buses['substation_lv']]
regions = (gpd.read_file(snakemake.input.regions).set_index('name') regions = (gpd.read_file(snakemake.input.regions).set_index('name')
.reindex(substation_lv_i)) .reindex(substation_lv_i))
# TODO: is there a fallback if year not in timeseries_opsd?
opsd_load = (timeseries_opsd(slice(*n.snapshots[[0,-1]].year.astype(str)), opsd_load = (timeseries_opsd(slice(*n.snapshots[[0,-1]].year.astype(str)),
snakemake.input.opsd_load) * snakemake.input.opsd_load) *
snakemake.config.get('load', {}).get('scaling_factor', 1.0)) snakemake.config.get('load', {}).get('scaling_factor', 1.0))
@ -522,6 +523,11 @@ if __name__ == "__main__":
configure_logging(snakemake) configure_logging(snakemake)
n = pypsa.Network(snakemake.input.base_network) n = pypsa.Network(snakemake.input.base_network)
year = int(snakemake.wildcards.year)
snapshots = dict(start=str(year), end=str(year+1), closed="left") if year else snakememake.config['snapshots']
n.set_snapshots(pd.date_range(freq='h', **snapshots))
n.snapshot_weightings[:] *= 8760. / n.snapshot_weightings.sum()
Nyears = n.snapshot_weightings.sum() / 8760. Nyears = n.snapshot_weightings.sum() / 8760.
costs = load_costs(Nyears) costs = load_costs(Nyears)

View File

@ -11,8 +11,6 @@ Relevant Settings
.. code:: yaml .. code:: yaml
snapshots:
countries: countries:
electricity: electricity:
@ -35,7 +33,7 @@ Relevant Settings
.. seealso:: .. seealso::
Documentation of the configuration file ``config.yaml`` at Documentation of the configuration file ``config.yaml`` at
:ref:`snapshots_cf`, :ref:`toplevel_cf`, :ref:`electricity_cf`, :ref:`load_cf`, :ref:`toplevel_cf`, :ref:`electricity_cf`, :ref:`load_cf`,
:ref:`lines_cf`, :ref:`links_cf`, :ref:`transformers_cf` :ref:`lines_cf`, :ref:`links_cf`, :ref:`transformers_cf`
Inputs Inputs
@ -554,9 +552,6 @@ def base_network():
n = pypsa.Network() n = pypsa.Network()
n.name = 'PyPSA-Eur' n.name = 'PyPSA-Eur'
n.set_snapshots(pd.date_range(freq='h', **snakemake.config['snapshots']))
n.snapshot_weightings[:] *= 8760. / n.snapshot_weightings.sum()
n.import_components_from_dataframe(buses, "Bus") n.import_components_from_dataframe(buses, "Bus")
n.import_components_from_dataframe(lines, "Line") n.import_components_from_dataframe(lines, "Line")
n.import_components_from_dataframe(transformers, "Transformer") n.import_components_from_dataframe(transformers, "Transformer")

View File

@ -165,7 +165,9 @@ if __name__ == '__main__':
config = snakemake.config['renewable'][snakemake.wildcards.technology] config = snakemake.config['renewable'][snakemake.wildcards.technology]
time = pd.date_range(freq='m', **snakemake.config['snapshots']) year = int(snakemake.wildcards.year)
snapshots = dict(start=str(year), end=str(year+1), closed="left") if year else snakememake.config['snapshots']
time = pd.date_range(freq='m', **snapshots)
params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]]))
cutout = atlite.Cutout(config['cutout'], cutout = atlite.Cutout(config['cutout'],

View File

@ -105,6 +105,7 @@ if __name__ == "__main__":
snakemake = mock_snakemake('build_cutout', cutout='europe-2013-era5') snakemake = mock_snakemake('build_cutout', cutout='europe-2013-era5')
configure_logging(snakemake) configure_logging(snakemake)
# TODO make it accept year independent config
cutout_params = snakemake.config['atlite']['cutouts'][snakemake.wildcards.cutout] cutout_params = snakemake.config['atlite']['cutouts'][snakemake.wildcards.cutout]
for p in ('xs', 'ys', 'years', 'months'): for p in ('xs', 'ys', 'years', 'months'):
if p in cutout_params: if p in cutout_params:

View File

@ -284,7 +284,9 @@ if __name__ == '__main__':
config = snakemake.config['renewable'][snakemake.wildcards.technology] config = snakemake.config['renewable'][snakemake.wildcards.technology]
time = pd.date_range(freq='m', **snakemake.config['snapshots']) year = int(snakemake.wildcards.year)
snapshots = dict(start=str(year), end=str(year+1), closed="left") if year else snakememake.config['snapshots']
time = pd.date_range(freq='m', **snapshots)
params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]]))
cutout = atlite.Cutout(config['cutout'], cutout = atlite.Cutout(config['cutout'],

View File

@ -11,6 +11,7 @@ logging:
summary_dir: results summary_dir: results
scenario: scenario:
year: ['']
simpl: [''] simpl: ['']
ll: ['copt'] ll: ['copt']
clusters: [5] clusters: [5]