From f6dd2e12f382aa58d599e90c4ce7eb11d844a663 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 11 Sep 2020 12:40:53 +0200 Subject: [PATCH 001/173] some refactoring and code cleaning --- .gitignore | 1 + {LICENSES => .licenses}/CC-BY-4.0.txt | 0 {LICENSES => .licenses}/CC0-1.0.txt | 0 {LICENSES => .licenses}/GPL-3.0-or-later.txt | 0 Snakefile | 129 ++++++++------- cluster.yaml | 22 --- config.default.yaml | 67 +------- config.tutorial.yaml | 160 ------------------- doc/configtables/plotting.csv | 3 +- doc/configtables/scenario.csv | 3 +- doc/index.rst | 8 +- doc/wildcards.rst | 10 -- matplotlibrc | 1 - scripts/_helpers.py | 8 +- scripts/add_electricity.py | 42 ++--- scripts/add_extra_components.py | 12 +- scripts/base_network.py | 34 +++- scripts/build_bus_regions.py | 23 +-- scripts/build_country_flh.py | 5 +- scripts/build_cutout.py | 7 +- scripts/build_hydro_profile.py | 13 +- scripts/build_natura_raster.py | 3 +- scripts/build_powerplants.py | 9 +- scripts/build_renewable_profiles.py | 13 +- scripts/build_shapes.py | 18 ++- scripts/cluster_network.py | 68 ++++---- scripts/make_summary.py | 98 +++--------- scripts/plot_network.py | 36 ++--- scripts/plot_p_nom_max.py | 8 +- scripts/plot_summary.py | 33 +--- scripts/prepare_links_p_nom.py | 33 ++-- scripts/prepare_network.py | 17 +- scripts/retrieve_databundle.py | 7 +- scripts/retrieve_natura_raster.py | 3 +- scripts/simplify_network.py | 12 +- scripts/solve_network.py | 14 +- scripts/solve_operations_network.py | 6 +- test/config.test1.yaml | 141 ---------------- 38 files changed, 331 insertions(+), 736 deletions(-) rename {LICENSES => .licenses}/CC-BY-4.0.txt (100%) rename {LICENSES => .licenses}/CC0-1.0.txt (100%) rename {LICENSES => .licenses}/GPL-3.0-or-later.txt (100%) delete mode 100644 cluster.yaml delete mode 100644 matplotlibrc diff --git a/.gitignore b/.gitignore index 5ed82d0d..b4734ab2 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__ *dconf gurobi.log +.vscode /bak /resources diff --git a/LICENSES/CC-BY-4.0.txt b/.licenses/CC-BY-4.0.txt similarity index 100% rename from LICENSES/CC-BY-4.0.txt rename to .licenses/CC-BY-4.0.txt diff --git a/LICENSES/CC0-1.0.txt b/.licenses/CC0-1.0.txt similarity index 100% rename from LICENSES/CC0-1.0.txt rename to .licenses/CC0-1.0.txt diff --git a/LICENSES/GPL-3.0-or-later.txt b/.licenses/GPL-3.0-or-later.txt similarity index 100% rename from LICENSES/GPL-3.0-or-later.txt rename to .licenses/GPL-3.0-or-later.txt diff --git a/Snakefile b/Snakefile index 72b726fc..ce27174a 100644 --- a/Snakefile +++ b/Snakefile @@ -11,33 +11,31 @@ if not exists("config.yaml"): configfile: "config.yaml" COSTS="data/costs.csv" +ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4) + wildcard_constraints: - ll="(v|c)([0-9\.]+|opt|all)|all", # line limit, can be volume or cost simpl="[a-zA-Z0-9]*|all", clusters="[0-9]+m?|all", - sectors="[+a-zA-Z0-9]+", + ll="(v|c)([0-9\.]+|opt|all)|all", opts="[-+a-zA-Z0-9\.]*" + rule cluster_all_elec_networks: - input: - expand("networks/elec_s{simpl}_{clusters}.nc", - **config['scenario']) + input: expand("networks/elec_s{simpl}_{clusters}.nc", **config['scenario']) + rule extra_components_all_elec_networks: - input: - expand("networks/elec_s{simpl}_{clusters}_ec.nc", - **config['scenario']) + input: expand("networks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) + rule prepare_all_elec_networks: - input: - expand("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", - **config['scenario']) + input: expand("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + rule solve_all_elec_networks: - input: - expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", - **config['scenario']) + input: expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + if config['enable'].get('prepare_links_p_nom', False): rule prepare_links_p_nom: @@ -45,7 +43,6 @@ if config['enable'].get('prepare_links_p_nom', False): log: 'logs/prepare_links_p_nom.log' threads: 1 resources: mem=500 - # group: 'nonfeedin_preparation' script: 'scripts/prepare_links_p_nom.py' @@ -56,15 +53,18 @@ datafiles = ['ch_cantons.csv', 'je-e-21.03.02.xls', 'nama_10r_3gdp.tsv.gz', 'time_series_60min_singleindex_filtered.csv', 'corine/g250_clc06_V18_5.tif'] + if not config.get('tutorial', False): datafiles.extend(["natura/Natura2000_end2015.shp", "GEBCO_2014_2D.nc"]) + if config['enable'].get('retrieve_databundle', True): rule retrieve_databundle: - output: expand('data/bundle/{file}', file=datafiles) + output: expand('data/bundle/{file}', file=datafiles) log: "logs/retrieve_databundle.log" script: 'scripts/retrieve_databundle.py' + rule build_powerplants: input: base_network="networks/base.nc", @@ -73,9 +73,9 @@ rule build_powerplants: log: "logs/build_powerplants.log" threads: 1 resources: mem=500 - # group: 'nonfeedin_preparation' script: "scripts/build_powerplants.py" + rule base_network: input: eg_buses='data/entsoegridkit/buses.csv', @@ -94,9 +94,9 @@ rule base_network: benchmark: "benchmarks/base_network" threads: 1 resources: mem=500 - # group: 'nonfeedin_preparation' script: "scripts/base_network.py" + rule build_shapes: input: naturalearth='data/bundle/naturalearth/ne_10m_admin_0_countries.shp', @@ -114,9 +114,9 @@ rule build_shapes: log: "logs/build_shapes.log" threads: 1 resources: mem=500 - # group: 'nonfeedin_preparation' script: "scripts/build_shapes.py" + rule build_bus_regions: input: country_shapes='resources/country_shapes.geojson', @@ -126,20 +126,21 @@ rule build_bus_regions: regions_onshore="resources/regions_onshore.geojson", regions_offshore="resources/regions_offshore.geojson" log: "logs/build_bus_regions.log" + threads: 1 resources: mem=1000 - # group: 'nonfeedin_preparation' script: "scripts/build_bus_regions.py" -if config['enable'].get('build_cutout', False): + +if config['enable'].get('build_cutout', False): rule build_cutout: output: directory("cutouts/{cutout}") log: "logs/build_cutout/{cutout}.log" - resources: mem=config['atlite'].get('nprocesses', 4) * 1000 - threads: config['atlite'].get('nprocesses', 4) benchmark: "benchmarks/build_cutout_{cutout}" - # group: 'feedin_preparation' + threads: ATLITE_NPROCESSES + resources: mem=ATLITE_NPROCESSES * 1000 script: "scripts/build_cutout.py" + if config['enable'].get('retrieve_cutout', True): rule retrieve_cutout: output: directory(expand("cutouts/{cutouts}", **config['atlite'])), @@ -156,34 +157,37 @@ if config['enable'].get('build_natura_raster', False): log: "logs/build_natura_raster.log" script: "scripts/build_natura_raster.py" + if config['enable'].get('retrieve_natura_raster', True): rule retrieve_natura_raster: output: "resources/natura.tiff" log: "logs/retrieve_natura_raster.log" script: 'scripts/retrieve_natura_raster.py' + rule build_renewable_profiles: input: base_network="networks/base.nc", corine="data/bundle/corine/g250_clc06_V18_5.tif", natura="resources/natura.tiff", - gebco=lambda wildcards: ("data/bundle/GEBCO_2014_2D.nc" - if "max_depth" in config["renewable"][wildcards.technology].keys() - else []), + gebco=lambda w: ("data/bundle/GEBCO_2014_2D.nc" + if "max_depth" in config["renewable"][w.technology].keys() + else []), country_shapes='resources/country_shapes.geojson', offshore_shapes='resources/offshore_shapes.geojson', - regions=lambda wildcards: ("resources/regions_onshore.geojson" - if wildcards.technology in ('onwind', 'solar') - else "resources/regions_offshore.geojson"), - cutout=lambda wildcards: "cutouts/" + config["renewable"][wildcards.technology]['cutout'] - output: profile="resources/profile_{technology}.nc", + regions=lambda w: ("resources/regions_onshore.geojson" + if w.technology in ('onwind', 'solar') + else "resources/regions_offshore.geojson"), + cutout=lambda w: "cutouts/" + config["renewable"][w.technology]['cutout'] + output: + profile="resources/profile_{technology}.nc", log: "logs/build_renewable_profile_{technology}.log" - resources: mem=config['atlite'].get('nprocesses', 2) * 5000 - threads: config['atlite'].get('nprocesses', 2) benchmark: "benchmarks/build_renewable_profiles_{technology}" - # group: 'feedin_preparation' + threads: ATLITE_NPROCESSES + resources: mem=ATLITE_NPROCESSES * 5000 script: "scripts/build_renewable_profiles.py" + if 'hydro' in config['renewable'].keys(): rule build_hydro_profile: input: @@ -193,9 +197,9 @@ if 'hydro' in config['renewable'].keys(): output: 'resources/profile_hydro.nc' log: "logs/build_hydro_profile.log" resources: mem=5000 - # group: 'feedin_preparation' script: 'scripts/build_hydro_profile.py' + rule add_electricity: input: base_network='networks/base.nc', @@ -206,16 +210,16 @@ rule add_electricity: geth_hydro_capacities='data/geth2015_hydro_capacities.csv', opsd_load='data/bundle/time_series_60min_singleindex_filtered.csv', nuts3_shapes='resources/nuts3_shapes.geojson', - **{'profile_' + t: "resources/profile_" + t + ".nc" - for t in config['renewable']} + **{'profile_' + tech: "resources/profile_" + tech + ".nc" + for tech in config['renewable']} output: "networks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" threads: 1 resources: mem=3000 - # group: 'build_pypsa_networks' script: "scripts/add_electricity.py" + rule simplify_network: input: network='networks/{network}.nc', @@ -231,9 +235,9 @@ rule simplify_network: benchmark: "benchmarks/simplify_network/{network}_s{simpl}" threads: 1 resources: mem=4000 - # group: 'build_pypsa_networks' script: "scripts/simplify_network.py" + rule cluster_network: input: network='networks/{network}_s{simpl}.nc', @@ -250,7 +254,6 @@ rule cluster_network: benchmark: "benchmarks/cluster_network/{network}_s{simpl}_{clusters}" threads: 1 resources: mem=3000 - # group: 'build_pypsa_networks' script: "scripts/cluster_network.py" @@ -263,7 +266,6 @@ rule add_extra_components: benchmark: "benchmarks/add_extra_components/{network}_s{simpl}_{clusters}_ec" threads: 1 resources: mem=3000 - # group: 'build_pypsa_networks' script: "scripts/add_extra_components.py" @@ -271,11 +273,12 @@ rule prepare_network: input: 'networks/{network}_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS output: 'networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' log: "logs/prepare_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" + benchmark: "benchmarks/prepare_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 1 resources: mem=1000 - # benchmark: "benchmarks/prepare_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}" script: "scripts/prepare_network.py" + def memory(w): factor = 3. for o in w.opts.split('-'): @@ -287,12 +290,11 @@ def memory(w): return int(factor * (18000 + 180 * int(w.clusters[:-1]))) else: return int(factor * (10000 + 195 * int(w.clusters))) - # return 4890+310 * int(w.clusters) + rule solve_network: input: "networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" output: "results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - shadow: "shallow" log: solver=normpath("logs/solve_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), python="logs/solve_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", @@ -300,15 +302,15 @@ rule solve_network: benchmark: "benchmarks/solve_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 resources: mem=memory - # group: "solve" # with group, threads is ignored https://bitbucket.org/snakemake/snakemake/issues/971/group-job-description-does-not-contain + shadow: "shallow" script: "scripts/solve_network.py" + rule solve_operations_network: input: unprepared="networks/{network}_s{simpl}_{clusters}_ec.nc", optimized="results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" output: "results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" - shadow: "shallow" log: solver=normpath("logs/solve_operations_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), python="logs/solve_operations_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", @@ -316,9 +318,10 @@ rule solve_operations_network: benchmark: "benchmarks/solve_operations_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 resources: mem=(lambda w: 5000 + 372 * int(w.clusters)) - # group: "solve_operations" + shadow: "shallow" script: "scripts/solve_operations_network.py" + rule plot_network: input: network="results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", @@ -329,6 +332,7 @@ rule plot_network: log: "logs/plot_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_{ext}.log" script: "scripts/plot_network.py" + def input_make_summary(w): # It's mildly hacky to include the separate costs input as first entry if w.ll.endswith("all"): @@ -344,42 +348,48 @@ def input_make_summary(w): **{k: config["scenario"][k] if getattr(w, k) == "all" else getattr(w, k) for k in ["simpl", "clusters", "opts"]})) + rule make_summary: input: input_make_summary output: directory("results/summaries/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}") log: "logs/make_summary/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log", script: "scripts/make_summary.py" + rule plot_summary: input: "results/summaries/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}" output: "results/plots/summary_{summary}_{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}" log: "logs/plot_summary/{summary}_{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log" script: "scripts/plot_summary.py" -def input_plot_p_nom_max(wildcards): - return [('networks/{network}_s{simpl}{maybe_cluster}.nc' - .format(maybe_cluster=('' if c == 'full' else ('_' + c)), **wildcards)) - for c in wildcards.clusts.split(",")] + +def input_plot_p_nom_max(w): + return [("networks/{network}_s{simpl}{maybe_cluster}.nc" + .format(maybe_cluster=('' if c == 'full' else ('_' + c)), **w)) + for c in w.clusts.split(",")] + + rule plot_p_nom_max: input: input_plot_p_nom_max output: "results/plots/{network}_s{simpl}_cum_p_nom_max_{clusts}_{techs}_{country}.{ext}" log: "logs/plot_p_nom_max/{network}_s{simpl}_{clusts}_{techs}_{country}_{ext}.log" script: "scripts/plot_p_nom_max.py" + rule build_country_flh: input: base_network="networks/base.nc", corine="data/bundle/corine/g250_clc06_V18_5.tif", natura="resources/natura.tiff", - gebco=lambda wildcards: ("data/bundle/GEBCO_2014_2D.nc" - if "max_depth" in config["renewable"][wildcards.technology].keys() - else []), + gebco=lambda w: ("data/bundle/GEBCO_2014_2D.nc" + if "max_depth" in config["renewable"][w.technology].keys() + else []), country_shapes='resources/country_shapes.geojson', offshore_shapes='resources/offshore_shapes.geojson', pietzker="data/pietzker2014.xlsx", regions=lambda w: ("resources/country_shapes.geojson" - if w.technology in ('onwind', 'solar') - else "resources/offshore_shapes.geojson"), + if w.technology in ('onwind', 'solar') + else "resources/offshore_shapes.geojson"), cutout=lambda w: "cutouts/" + config["renewable"][w.technology]['cutout'] output: area="resources/country_flh_area_{technology}.csv", @@ -390,9 +400,4 @@ rule build_country_flh: log: "logs/build_country_flh_{technology}.log" resources: mem=10000 benchmark: "benchmarks/build_country_flh_{technology}" - # group: 'feedin_preparation' script: "scripts/build_country_flh.py" - -# Local Variables: -# mode: python -# End: diff --git a/cluster.yaml b/cluster.yaml deleted file mode 100644 index b36e6ed2..00000000 --- a/cluster.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: : 2017-2020 The PyPSA-Eur Authors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -__default__: - log: "logs/cluster/{{name}}.log" - -feedin_preparation: - walltime: "12:00:00" - -solve_network: - walltime: "05:00:00:00" - -trace_solve_network: - walltime: "05:00:00:00" - -solve: - walltime: "05:00:00:00" - threads: 4 # Group threads are not aggregated - -solve_operations: - walltime: "01:00:00:00" diff --git a/config.default.yaml b/config.default.yaml index 375bcda6..d94d9842 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -12,7 +12,6 @@ logging: summary_dir: results scenario: - sectors: [E] simpl: [''] ll: ['copt'] clusters: [37, 128, 256, 512, 1024] @@ -257,67 +256,18 @@ plotting: 'waste' : '#68896b' 'geothermal' : '#ba91b1' "OCGT" : "#d35050" - "OCGT marginal" : "#d35050" - "OCGT-heat" : "#d35050" - "gas boiler" : "#d35050" - "gas boilers" : "#d35050" - "gas boiler marginal" : "#d35050" - "gas-to-power/heat" : "#d35050" "gas" : "#d35050" "natural gas" : "#d35050" "CCGT" : "#b20101" - "CCGT marginal" : "#b20101" - "Nuclear" : "#ff9000" - "Nuclear marginal" : "#ff9000" "nuclear" : "#ff9000" "coal" : "#707070" - "Coal" : "#707070" - "Coal marginal" : "#707070" "lignite" : "#9e5a01" - "Lignite" : "#9e5a01" - "Lignite marginal" : "#9e5a01" - "Oil" : "#262626" "oil" : "#262626" "H2" : "#ea048a" "hydrogen storage" : "#ea048a" - "Sabatier" : "#a31597" - "methanation" : "#a31597" - "helmeth" : "#a31597" - "DAC" : "#d284ff" - "co2 stored" : "#e5e5e5" - "CO2 sequestration" : "#e5e5e5" "battery" : "#b8ea04" - "battery storage" : "#b8ea04" - "Li ion" : "#b8ea04" - "BEV charger" : "#e2ff7c" - "V2G" : "#7a9618" - "transport fuel cell" : "#e884be" - "retrofitting" : "#e0d6a8" - "building retrofitting" : "#e0d6a8" - "heat pumps" : "#ff9768" - "heat pump" : "#ff9768" - "air heat pump" : "#ffbea0" - "ground heat pump" : "#ff7a3d" - "power-to-heat" : "#a59e7c" - "power-to-gas" : "#db8585" - "power-to-liquid" : "#a9acd1" - "Fischer-Tropsch" : "#a9acd1" - "resistive heater" : "#aa4925" - "water tanks" : "#401f75" - "hot water storage" : "#401f75" - "hot water charging" : "#351c5e" - "hot water discharging" : "#683ab2" - "CHP" : "#d80a56" - "CHP heat" : "#d80a56" - "CHP electric" : "#d80a56" - "district heating" : "#93864b" - "Ambient" : "#262626" "Electric load" : "#f9d002" "electricity" : "#f9d002" - "Heat load" : "#d35050" - "heat" : "#d35050" - "Transport load" : "#235ebc" - "transport" : "#235ebc" "lines" : "#70af1d" "transmission lines" : "#70af1d" "AC-AC" : "#70af1d" @@ -337,18 +287,5 @@ plotting: hydro: "Reservoir & Dam" battery: "Battery Storage" H2: "Hydrogen Storage" - lines: "Transmission lines" - ror: "Run of river" - nice_names_n: - OCGT: "Open-Cycle\nGas" - CCGT: "Combined-Cycle\nGas" - offwind-ac: "Offshore\nWind (AC)" - offwind-dc: "Offshore\nWind (DC)" - onwind: "Onshore\nWind" - battery: "Battery\nStorage" - H2: "Hydrogen\nStorage" - lines: "Transmission\nlines" - ror: "Run of\nriver" - PHS: "Pumped Hydro\nStorage" - hydro: "Reservoir\n& Dam" - + lines: "Transmission Lines" + ror: "Run of River" diff --git a/config.tutorial.yaml b/config.tutorial.yaml index f77ad2bd..7fda5daf 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -11,7 +11,6 @@ logging: summary_dir: results scenario: - sectors: [E] simpl: [''] ll: ['copt'] clusters: [5] @@ -169,164 +168,5 @@ solving: clip_p_max_pu: 0.01 skip_iterations: false track_iterations: false - #nhours: 10 solver: name: cbc - # solver: - # name: gurobi - # threads: 4 - # method: 2 # barrier - # crossover: 0 - # BarConvTol: 1.e-5 - # FeasibilityTol: 1.e-6 - # AggFill: 0 - # PreDual: 0 - # GURO_PAR_BARDENSETHRESH: 200 - # solver: - # name: cplex - # threads: 4 - # lpmethod: 4 # barrier - # solutiontype: 2 # non basic solution, ie no crossover - # barrier_convergetol: 1.e-5 - # feasopt_tolerance: 1.e-6 - -plotting: - map: - figsize: [7, 7] - boundaries: [-10.2, 29, 35, 72] - p_nom: - bus_size_factor: 5.e+4 - linewidth_factor: 3.e+3 - - costs_max: 800 - costs_threshold: 1 - - energy_max: 15000. - energy_min: -10000. - energy_threshold: 50. - - vre_techs: ["onwind", "offwind-ac", "offwind-dc", "solar", "ror"] - conv_techs: ["OCGT", "CCGT", "Nuclear", "Coal"] - storage_techs: ["hydro+PHS", "battery", "H2"] - load_carriers: ["AC load"] - AC_carriers: ["AC line", "AC transformer"] - link_carriers: ["DC line", "Converter AC-DC"] - tech_colors: - "onwind" : "#235ebc" - "onshore wind" : "#235ebc" - 'offwind' : "#6895dd" - 'offwind-ac' : "#6895dd" - 'offshore wind' : "#6895dd" - 'offshore wind ac' : "#6895dd" - 'offwind-dc' : "#74c6f2" - 'offshore wind dc' : "#74c6f2" - "hydro" : "#08ad97" - "hydro+PHS" : "#08ad97" - "PHS" : "#08ad97" - "hydro reservoir" : "#08ad97" - 'hydroelectricity' : '#08ad97' - "ror" : "#4adbc8" - "run of river" : "#4adbc8" - 'solar' : "#f9d002" - 'solar PV' : "#f9d002" - 'solar thermal' : '#ffef60' - 'biomass' : '#0c6013' - 'solid biomass' : '#06540d' - 'biogas' : '#23932d' - 'waste' : '#68896b' - 'geothermal' : '#ba91b1' - "OCGT" : "#d35050" - "OCGT marginal" : "#d35050" - "OCGT-heat" : "#d35050" - "gas boiler" : "#d35050" - "gas boilers" : "#d35050" - "gas boiler marginal" : "#d35050" - "gas-to-power/heat" : "#d35050" - "gas" : "#d35050" - "natural gas" : "#d35050" - "CCGT" : "#b20101" - "CCGT marginal" : "#b20101" - "Nuclear" : "#ff9000" - "Nuclear marginal" : "#ff9000" - "nuclear" : "#ff9000" - "coal" : "#707070" - "Coal" : "#707070" - "Coal marginal" : "#707070" - "lignite" : "#9e5a01" - "Lignite" : "#9e5a01" - "Lignite marginal" : "#9e5a01" - "Oil" : "#262626" - "oil" : "#262626" - "H2" : "#ea048a" - "hydrogen storage" : "#ea048a" - "Sabatier" : "#a31597" - "methanation" : "#a31597" - "helmeth" : "#a31597" - "DAC" : "#d284ff" - "co2 stored" : "#e5e5e5" - "CO2 sequestration" : "#e5e5e5" - "battery" : "#b8ea04" - "battery storage" : "#b8ea04" - "Li ion" : "#b8ea04" - "BEV charger" : "#e2ff7c" - "V2G" : "#7a9618" - "transport fuel cell" : "#e884be" - "retrofitting" : "#e0d6a8" - "building retrofitting" : "#e0d6a8" - "heat pumps" : "#ff9768" - "heat pump" : "#ff9768" - "air heat pump" : "#ffbea0" - "ground heat pump" : "#ff7a3d" - "power-to-heat" : "#a59e7c" - "power-to-gas" : "#db8585" - "power-to-liquid" : "#a9acd1" - "Fischer-Tropsch" : "#a9acd1" - "resistive heater" : "#aa4925" - "water tanks" : "#401f75" - "hot water storage" : "#401f75" - "hot water charging" : "#351c5e" - "hot water discharging" : "#683ab2" - "CHP" : "#d80a56" - "CHP heat" : "#d80a56" - "CHP electric" : "#d80a56" - "district heating" : "#93864b" - "Ambient" : "#262626" - "Electric load" : "#f9d002" - "electricity" : "#f9d002" - "Heat load" : "#d35050" - "heat" : "#d35050" - "Transport load" : "#235ebc" - "transport" : "#235ebc" - "lines" : "#70af1d" - "transmission lines" : "#70af1d" - "AC-AC" : "#70af1d" - "AC line" : "#70af1d" - "links" : "#8a1caf" - "HVDC links" : "#8a1caf" - "DC-DC" : "#8a1caf" - "DC link" : "#8a1caf" - nice_names: - OCGT: "Open-Cycle Gas" - CCGT: "Combined-Cycle Gas" - offwind-ac: "Offshore Wind (AC)" - offwind-dc: "Offshore Wind (DC)" - onwind: "Onshore Wind" - solar: "Solar" - PHS: "Pumped Hydro Storage" - hydro: "Reservoir & Dam" - battery: "Battery Storage" - H2: "Hydrogen Storage" - lines: "Transmission lines" - ror: "Run of river" - nice_names_n: - OCGT: "Open-Cycle\nGas" - CCGT: "Combined-Cycle\nGas" - offwind-ac: "Offshore\nWind (AC)" - offwind-dc: "Offshore\nWind (DC)" - onwind: "Onshore\nWind" - battery: "Battery\nStorage" - H2: "Hydrogen\nStorage" - lines: "Transmission\nlines" - ror: "Run of\nriver" - PHS: "Pumped Hydro\nStorage" - hydro: "Reservoir\n& Dam" diff --git a/doc/configtables/plotting.csv b/doc/configtables/plotting.csv index 0f21c9a8..754f2e83 100644 --- a/doc/configtables/plotting.csv +++ b/doc/configtables/plotting.csv @@ -11,5 +11,4 @@ energy_max,TWh,float,"Upper y-axis limit in energy bar plots." energy_min,TWh,float,"Lower y-axis limit in energy bar plots." energy_threshold,TWh,float,"Threshold below which technologies will not be shown in energy bar plots." tech_colors,--,"carrier -> HEX colour code","Mapping from network ``carrier`` to a colour (`HEX colour code `_)." -nice_names,--,"str -> str","Mapping from network ``carrier`` to a more readable name." -nice_names_n,--,"str -> str","Same as nice_names, but with linebreaks." \ No newline at end of file +nice_names,--,"str -> str","Mapping from network ``carrier`` to a more readable name." \ No newline at end of file diff --git a/doc/configtables/scenario.csv b/doc/configtables/scenario.csv index 52dafa56..7a2d3bcd 100644 --- a/doc/configtables/scenario.csv +++ b/doc/configtables/scenario.csv @@ -1,6 +1,5 @@ ,Unit,Values,Description -sectors,--,"Must be 'elec'","Placeholder for integration of other energy sectors." simpl,--,cf. :ref:`simpl`,"List of ``{simpl}`` wildcards to run." -ll,--,cf. :ref:`ll`,"List of ``{ll}`` wildcards to run." clusters,--,cf. :ref:`clusters`,"List of ``{clusters}`` wildcards to run." +ll,--,cf. :ref:`ll`,"List of ``{ll}`` wildcards to run." opts,--,cf. :ref:`opts`,"List of ``{opts}`` wildcards to run." \ No newline at end of file diff --git a/doc/index.rst b/doc/index.rst index bd2ec5f2..d375cfac 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -194,10 +194,10 @@ Licence PyPSA-Eur work is released under multiple licenses: -* All original source code is licensed as free software under `GPL-3.0-or-later `_. -* The documentation is licensed under `CC-BY-4.0 `_. -* Configuration files are mostly licensed under `CC0-1.0 `_. -* Data files are licensed under `CC-BY-4.0 `_. +* All original source code is licensed as free software under `GPL-3.0-or-later <.licenses/GPL-3.0-or-later.txt>`_. +* The documentation is licensed under `CC-BY-4.0 <.licenses/CC-BY-4.0.txt>`_. +* Configuration files are mostly licensed under `CC0-1.0 <.licenses/CC0-1.0.txt>`_. +* Data files are licensed under `CC-BY-4.0 <.licenses/CC-BY-4.0.txt>`_. See the individual files and the `dep5 <.reuse/dep5>`_ file for license details. diff --git a/doc/wildcards.rst b/doc/wildcards.rst index 9f1c2248..a180d04b 100644 --- a/doc/wildcards.rst +++ b/doc/wildcards.rst @@ -18,16 +18,6 @@ what data to retrieve and what files to produce. Detailed explanations of how wildcards work in ``snakemake`` can be found in the `relevant section of the documentation `_. -.. _network: - -The ``{network}`` wildcard -========================== - -The ``{network}`` wildcard specifies the considered energy sector(s) -and, as currently only ``elec`` (for electricity) is included, -it currently represents rather a placeholder wildcard to facilitate -future extensions including multiple energy sectors at once. - .. _simpl: The ``{simpl}`` wildcard diff --git a/matplotlibrc b/matplotlibrc deleted file mode 100644 index 13468274..00000000 --- a/matplotlibrc +++ /dev/null @@ -1 +0,0 @@ -backend : Agg diff --git a/scripts/_helpers.py b/scripts/_helpers.py index fff8143d..e0e16d20 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -44,6 +44,7 @@ def configure_logging(snakemake, skip_handlers=False): }) logging.basicConfig(**kwargs) + def load_network(import_name=None, custom_components=None): """ Helper for importing a pypsa.Network with additional custom components. @@ -70,7 +71,6 @@ def load_network(import_name=None, custom_components=None): ------- pypsa.Network """ - import pypsa from pypsa.descriptors import Dict @@ -90,10 +90,12 @@ def load_network(import_name=None, custom_components=None): override_components=override_components, override_component_attrs=override_component_attrs) + def pdbcast(v, h): return pd.DataFrame(v.values.reshape((-1, 1)) * h.values, index=v.index, columns=h.index) + def load_network_for_plots(fn, tech_costs, config, combine_hydro_ps=True): import pypsa from add_electricity import update_transmission_costs, load_costs @@ -113,11 +115,11 @@ def load_network_for_plots(fn, tech_costs, config, combine_hydro_ps=True): if combine_hydro_ps: n.storage_units.loc[n.storage_units.carrier.isin({'PHS', 'hydro'}), 'carrier'] = 'hydro+PHS' - # #if the carrier was not set on the heat storage units + # if the carrier was not set on the heat storage units # bus_carrier = n.storage_units.bus.map(n.buses.carrier) # n.storage_units.loc[bus_carrier == "heat","carrier"] = "water tanks" - Nyears = n.snapshot_weightings.sum()/8760. + Nyears = n.snapshot_weightings.sum() / 8760. costs = load_costs(Nyears, tech_costs, config['costs'], config['electricity']) update_transmission_costs(n, costs) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 7a32e628..14667679 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -90,26 +90,28 @@ It further adds extendable ``generators`` with **zero** capacity for - additional open- and combined-cycle gas turbines (if ``OCGT`` and/or ``CCGT`` is listed in the config setting ``electricity: extendable_carriers``) """ -from vresutils.costdata import annuity -from vresutils.load import timeseries_opsd -from vresutils import transfer as vtransfer - import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging +import pypsa import pandas as pd import numpy as np import xarray as xr import geopandas as gpd -import pypsa import powerplantmatching as ppm +from vresutils.costdata import annuity +from vresutils.load import timeseries_opsd +from vresutils import transfer as vtransfer + idx = pd.IndexSlice +logger = logging.getLogger(__name__) + def normed(s): return s/s.sum() + def _add_missing_carriers_from_costs(n, costs, carriers): missing_carriers = pd.Index(carriers).difference(n.carriers.index) if missing_carriers.empty: return @@ -121,6 +123,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers): emissions.index = missing_carriers n.import_components_from_dataframe(emissions, 'Carrier') + def load_costs(Nyears=1., tech_costs=None, config=None, elec_config=None): if tech_costs is None: tech_costs = snakemake.input.tech_costs @@ -193,10 +196,11 @@ def load_costs(Nyears=1., tech_costs=None, config=None, elec_config=None): return costs + def load_powerplants(ppl_fn=None): if ppl_fn is None: ppl_fn = snakemake.input.powerplants - carrier_dict = {'ocgt': 'OCGT', 'ccgt': 'CCGT', 'bioenergy':'biomass', + carrier_dict = {'ocgt': 'OCGT', 'ccgt': 'CCGT', 'bioenergy': 'biomass', 'ccgt, thermal': 'CCGT', 'hard coal': 'coal'} return (pd.read_csv(ppl_fn, index_col=0, dtype={'bus': 'str'}) .powerplant.to_pypsa_names() @@ -204,12 +208,6 @@ def load_powerplants(ppl_fn=None): .replace({'carrier': carrier_dict})) -# ============================================================================= -# Attach components -# ============================================================================= - -# ### Load - def attach_load(n): substation_lv_i = n.buses.index[n.buses['substation_lv']] regions = (gpd.read_file(snakemake.input.regions).set_index('name') @@ -249,7 +247,6 @@ def attach_load(n): n.madd("Load", substation_lv_i, bus=substation_lv_i, p_set=load) -### Set line costs def update_transmission_costs(n, costs, length_factor=1.0, simple_hvdc_costs=False): n.lines['capital_cost'] = (n.lines['length'] * length_factor * @@ -270,7 +267,6 @@ def update_transmission_costs(n, costs, length_factor=1.0, simple_hvdc_costs=Fal costs.at['HVDC inverter pair', 'capital_cost']) n.links.loc[dc_b, 'capital_cost'] = costs -### Generators def attach_wind_and_solar(n, costs): for tech in snakemake.config['renewable']: @@ -309,15 +305,17 @@ def attach_wind_and_solar(n, costs): p_max_pu=ds['profile'].transpose('time', 'bus').to_pandas()) - def attach_conventional_generators(n, costs, ppl): carriers = snakemake.config['electricity']['conventional_carriers'] + _add_missing_carriers_from_costs(n, costs, carriers) + ppl = (ppl.query('carrier in @carriers').join(costs, on='carrier') .rename(index=lambda s: 'C' + str(s))) logger.info('Adding {} generators with capacities\n{}' .format(len(ppl), ppl.groupby('carrier').p_nom.sum())) + n.madd("Generator", ppl.index, carrier=ppl.carrier, bus=ppl.bus, @@ -325,6 +323,7 @@ def attach_conventional_generators(n, costs, ppl): efficiency=ppl.efficiency, marginal_cost=ppl.marginal_cost, capital_cost=0) + logger.warning(f'Capital costs for conventional generators put to 0 EUR/MW.') @@ -374,8 +373,8 @@ def attach_hydro(n, costs, ppl): .where(lambda df: df<=1., other=1.))) if 'PHS' in carriers and not phs.empty: - # fill missing max hours to config value and assume no natural inflow - # due to lack of data + # fill missing max hours to config value and + # assume no natural inflow due to lack of data phs = phs.replace({'max_hours': {0: c['PHS_max_hours']}}) n.madd('StorageUnit', phs.index, carrier='PHS', @@ -413,7 +412,6 @@ def attach_hydro(n, costs, ppl): hydro_max_hours = hydro.max_hours.where(hydro.max_hours > 0, hydro.country.map(max_hours_country)).fillna(6) - n.madd('StorageUnit', hydro.index, carrier='hydro', bus=hydro['bus'], p_nom=hydro['p_nom'], @@ -432,6 +430,7 @@ def attach_hydro(n, costs, ppl): def attach_extendable_generators(n, costs, ppl): elec_opts = snakemake.config['electricity'] carriers = pd.Index(elec_opts['extendable_carriers']['Generator']) + _add_missing_carriers_from_costs(n, costs, carriers) for tech in carriers: @@ -497,10 +496,11 @@ def estimate_renewable_capacities(n, tech_map=None): n.generators.loc[tech_i, 'p_nom'] = ( (n.generators_t.p_max_pu[tech_i].mean() * n.generators.loc[tech_i, 'p_nom_max']) # maximal yearly generation - .groupby(n.generators.bus.map(n.buses.country)) # for each country + .groupby(n.generators.bus.map(n.buses.country)) .transform(lambda s: normed(s) * tech_capacities.at[s.name]) .where(lambda s: s>0.1, 0.)) # only capacities above 100kW + def add_nice_carrier_names(n, config=None): if config is None: config = snakemake.config carrier_i = n.carriers.index @@ -522,7 +522,7 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) - Nyears = n.snapshot_weightings.sum()/8760. + Nyears = n.snapshot_weightings.sum() / 8760. costs = load_costs(Nyears) ppl = load_powerplants() diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 219c082d..d0b627a3 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -50,17 +50,20 @@ The rule :mod:`add_extra_components` attaches additional extendable components t - ``Stores`` of carrier 'H2' and/or 'battery' in combination with ``Links``. If this option is chosen, the script adds extra buses with corresponding carrier where energy ``Stores`` are attached and which are connected to the corresponding power buses via two links, one each for charging and discharging. This leads to three investment variables for the energy capacity, charging and discharging capacity of the storage unit. """ import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging +import pypsa import pandas as pd import numpy as np -import pypsa + from add_electricity import (load_costs, add_nice_carrier_names, _add_missing_carriers_from_costs) idx = pd.IndexSlice +logger = logging.getLogger(__name__) + + def attach_storageunits(n, costs): elec_opts = snakemake.config['electricity'] carriers = elec_opts['extendable_carriers']['StorageUnit'] @@ -82,6 +85,7 @@ def attach_storageunits(n, costs): max_hours=max_hours[carrier], cyclic_state_of_charge=True) + def attach_stores(n, costs): elec_opts = snakemake.config['electricity'] carriers = elec_opts['extendable_carriers']['Store'] @@ -144,6 +148,7 @@ def attach_stores(n, costs): capital_cost=costs.at['battery inverter', 'capital_cost'], p_nom_extendable=True) + def attach_hydrogen_pipelines(n, costs): elec_opts = snakemake.config['electricity'] ext_carriers = elec_opts['extendable_carriers'] @@ -176,6 +181,7 @@ def attach_hydrogen_pipelines(n, costs): efficiency=costs.at['H2 pipeline','efficiency'], carrier="H2 pipeline") + if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake @@ -184,7 +190,7 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) - Nyears = n.snapshot_weightings.sum()/8760. + Nyears = n.snapshot_weightings.sum() / 8760. costs = load_costs(Nyears, tech_costs=snakemake.input.tech_costs, config=snakemake.config['costs'], elec_config=snakemake.config['electricity']) diff --git a/scripts/base_network.py b/scripts/base_network.py index 1d5fddd7..034eeb2f 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -63,14 +63,16 @@ Description """ import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging +import pypsa import yaml import pandas as pd import geopandas as gpd import numpy as np import scipy as sp +import networkx as nx + from scipy.sparse import csgraph from six import iteritems from itertools import product @@ -78,9 +80,8 @@ from itertools import product from shapely.geometry import Point, LineString import shapely, shapely.prepared, shapely.wkt -import networkx as nx +logger = logging.getLogger(__name__) -import pypsa def _get_oid(df): if "tags" in df.columns: @@ -88,12 +89,14 @@ def _get_oid(df): else: return pd.Series(np.nan, df.index) + def _get_country(df): if "tags" in df.columns: return df.tags.str.extract('"country"=>"([A-Z]{2})"', expand=False) else: return pd.Series(np.nan, df.index) + def _find_closest_links(links, new_links, distance_upper_bound=1.5): treecoords = np.asarray([np.asarray(shapely.wkt.loads(s))[[0, -1]].flatten() for s in links.geometry]) @@ -109,6 +112,7 @@ def _find_closest_links(links, new_links, distance_upper_bound=1.5): [lambda ds: ~ds.index.duplicated(keep='first')]\ .sort_index()['i'] + def _load_buses_from_eg(): buses = (pd.read_csv(snakemake.input.eg_buses, quotechar="'", true_values='t', false_values='f', @@ -130,6 +134,7 @@ def _load_buses_from_eg(): return pd.DataFrame(buses.loc[buses_in_europe_b & buses_with_v_nom_to_keep_b]) + def _load_transformers_from_eg(buses): transformers = (pd.read_csv(snakemake.input.eg_transformers, quotechar="'", true_values='t', false_values='f', @@ -140,6 +145,7 @@ def _load_transformers_from_eg(buses): return transformers + def _load_converters_from_eg(buses): converters = (pd.read_csv(snakemake.input.eg_converters, quotechar="'", true_values='t', false_values='f', @@ -241,6 +247,7 @@ def _add_links_from_tyndp(buses, links): return buses, links.append(links_tyndp, sort=True) + def _load_lines_from_eg(buses): lines = (pd.read_csv(snakemake.input.eg_lines, quotechar="'", true_values='t', false_values='f', dtype=dict(line_id='str', bus0='str', bus1='str', @@ -254,11 +261,13 @@ def _load_lines_from_eg(buses): return lines + def _apply_parameter_corrections(n): with open(snakemake.input.parameter_corrections) as f: corrections = yaml.safe_load(f) if corrections is None: return + for component, attrs in iteritems(corrections): df = n.df(component) oid = _get_oid(df) @@ -275,6 +284,7 @@ def _apply_parameter_corrections(n): inds = r.index.intersection(df.index) df.loc[inds, attr] = r[inds].astype(df[attr].dtype) + def _set_electrical_parameters_lines(lines): v_noms = snakemake.config['electricity']['voltages'] linetypes = snakemake.config['lines']['types'] @@ -286,12 +296,14 @@ def _set_electrical_parameters_lines(lines): return lines + def _set_lines_s_nom_from_linetypes(n): n.lines['s_nom'] = ( np.sqrt(3) * n.lines['type'].map(n.line_types.i_nom) * n.lines['v_nom'] * n.lines.num_parallel ) + def _set_electrical_parameters_links(links): if links.empty: return links @@ -301,11 +313,11 @@ def _set_electrical_parameters_links(links): links_p_nom = pd.read_csv(snakemake.input.links_p_nom) - #Filter links that are not in operation anymore + # filter links that are not in operation anymore removed_b = links_p_nom.Remarks.str.contains('Shut down|Replaced', na=False) links_p_nom = links_p_nom[~removed_b] - #find closest link for all links in links_p_nom + # find closest link for all links in links_p_nom links_p_nom['j'] = _find_closest_links(links, links_p_nom) links_p_nom = links_p_nom.groupby(['j'],as_index=False).agg({'Power (MW)': 'sum'}) @@ -318,6 +330,7 @@ def _set_electrical_parameters_links(links): return links + def _set_electrical_parameters_converters(converters): p_max_pu = snakemake.config['links'].get('p_max_pu', 1.) converters['p_max_pu'] = p_max_pu @@ -331,6 +344,7 @@ def _set_electrical_parameters_converters(converters): return converters + def _set_electrical_parameters_transformers(transformers): config = snakemake.config['transformers'] @@ -341,9 +355,11 @@ def _set_electrical_parameters_transformers(transformers): return transformers + def _remove_dangling_branches(branches, buses): return pd.DataFrame(branches.loc[branches.bus0.isin(buses.index) & branches.bus1.isin(buses.index)]) + def _remove_unconnected_components(network): _, labels = csgraph.connected_components(network.adjacency_matrix(), directed=False) component = pd.Series(labels, index=network.buses.index) @@ -356,6 +372,7 @@ def _remove_unconnected_components(network): return network[component == component_sizes.index[0]] + def _set_countries_and_substations(n): buses = n.buses @@ -442,6 +459,7 @@ def _set_countries_and_substations(n): return buses + def _replace_b2b_converter_at_country_border_by_link(n): # Affects only the B2B converter in Lithuania at the Polish border at the moment buscntry = n.buses.country @@ -479,6 +497,7 @@ def _replace_b2b_converter_at_country_border_by_link(n): logger.info("Replacing B2B converter `{}` together with bus `{}` and line `{}` by an HVDC tie-line {}-{}" .format(i, b0, line, linkcntry.at[i], buscntry.at[b1])) + def _set_links_underwater_fraction(n): if n.links.empty: return @@ -489,6 +508,7 @@ def _set_links_underwater_fraction(n): links = gpd.GeoSeries(n.links.geometry.dropna().map(shapely.wkt.loads)) n.links['underwater_fraction'] = links.intersection(offshore_shape).length / links.length + def _adjust_capacities_of_under_construction_branches(n): lines_mode = snakemake.config['lines'].get('under_construction', 'undef') if lines_mode == 'zero': @@ -513,6 +533,7 @@ def _adjust_capacities_of_under_construction_branches(n): return n + def base_network(): buses = _load_buses_from_eg() @@ -534,7 +555,7 @@ def base_network(): n.name = 'PyPSA-Eur' n.set_snapshots(pd.date_range(freq='h', **snakemake.config['snapshots'])) - n.snapshot_weightings[:] *= 8760./n.snapshot_weightings.sum() + n.snapshot_weightings[:] *= 8760. / n.snapshot_weightings.sum() n.import_components_from_dataframe(buses, "Bus") n.import_components_from_dataframe(lines, "Line") @@ -565,4 +586,5 @@ if __name__ == "__main__": configure_logging(snakemake) n = base_network() + n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index ce012c39..87890d92 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -42,17 +42,24 @@ Description """ import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging -from vresutils.graph import voronoi_partition_pts - +import pypsa import os - import pandas as pd import geopandas as gpd -import pypsa +from vresutils.graph import voronoi_partition_pts + +logger = logging.getLogger(__name__) + + +def save_to_geojson(s, fn): + if os.path.exists(fn): + os.unlink(fn) + schema = {**gpd.io.file.infer_schema(s), 'geometry': 'Unknown'} + s.to_file(fn, driver='GeoJSON', schema=schema) + if __name__ == "__main__": if 'snakemake' not in globals(): @@ -96,12 +103,6 @@ if __name__ == "__main__": offshore_regions_c = offshore_regions_c.loc[offshore_regions_c.area > 1e-2] offshore_regions.append(offshore_regions_c) - def save_to_geojson(s, fn): - if os.path.exists(fn): - os.unlink(fn) - schema = {**gpd.io.file.infer_schema(s), 'geometry': 'Unknown'} - s.to_file(fn, driver='GeoJSON', schema=schema) - save_to_geojson(pd.concat(onshore_regions, ignore_index=True), snakemake.output.regions_onshore) save_to_geojson(pd.concat(offshore_regions, ignore_index=True), snakemake.output.regions_offshore) diff --git a/scripts/build_country_flh.py b/scripts/build_country_flh.py index 2fb8a173..459b8f38 100644 --- a/scripts/build_country_flh.py +++ b/scripts/build_country_flh.py @@ -63,7 +63,6 @@ Description """ import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging import os @@ -84,6 +83,9 @@ import progressbar as pgb from build_renewable_profiles import init_globals, calculate_potential +logger = logging.getLogger(__name__) + + def build_area(flh, countries, areamatrix, breaks, fn): area_unbinned = xr.DataArray(areamatrix.todense(), [countries, capacity_factor.coords['spatial']]) bins = xr.DataArray(pd.cut(flh.to_series(), bins=breaks), flh.coords, name="bins") @@ -92,6 +94,7 @@ def build_area(flh, countries, areamatrix, breaks, fn): area.columns = area.columns.map(lambda s: s.left) return area + def plot_area_not_solar(area, countries): # onshore wind/offshore wind a = area.T diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index b6fc6761..1e55faf5 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -92,12 +92,13 @@ Description """ import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging import os import atlite +logger = logging.getLogger(__name__) + if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake @@ -113,4 +114,6 @@ if __name__ == "__main__": cutout_dir=os.path.dirname(snakemake.output[0]), **cutout_params) - cutout.prepare(nprocesses=snakemake.config['atlite'].get('nprocesses', 4)) + nprocesses = snakemake.config['atlite'].get('nprocesses', 4) + + cutout.prepare(nprocesses=nprocesses) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 0736511a..339fccaf 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -60,7 +60,6 @@ Description """ import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging import os @@ -68,6 +67,8 @@ import atlite import geopandas as gpd from vresutils import hydro as vhydro +logger = logging.getLogger(__name__) + if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake @@ -75,8 +76,8 @@ if __name__ == "__main__": configure_logging(snakemake) config = snakemake.config['renewable']['hydro'] - cutout = atlite.Cutout(config['cutout'], - cutout_dir=os.path.dirname(snakemake.input.cutout)) + cutout_dir = os.path.dirname(snakemake.input.cutout) + cutout = atlite.Cutout(config['cutout'], cutout_dir=cutout_dir) countries = snakemake.config['countries'] country_shapes = gpd.read_file(snakemake.input.country_shapes).set_index('name')['geometry'].reindex(countries) @@ -84,9 +85,9 @@ if __name__ == "__main__": eia_stats = vhydro.get_eia_annual_hydro_generation(snakemake.input.eia_hydro_generation).reindex(columns=countries) inflow = cutout.runoff(shapes=country_shapes, - smooth=True, - lower_threshold_quantile=True, - normalize_using_yearly=eia_stats) + smooth=True, + lower_threshold_quantile=True, + normalize_using_yearly=eia_stats) if 'clip_min_inflow' in config: inflow.values[inflow.values < config['clip_min_inflow']] = 0. diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py index f2ee491b..39667ca0 100644 --- a/scripts/build_natura_raster.py +++ b/scripts/build_natura_raster.py @@ -41,6 +41,7 @@ Description import logging from _helpers import configure_logging + import atlite import geokit as gk from pathlib import Path @@ -58,7 +59,7 @@ def determine_cutout_xXyY(cutout_name): if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('build_natura_raster') #has to be enabled + snakemake = mock_snakemake('build_natura_raster') configure_logging(snakemake) cutout_dir = Path(snakemake.input.cutouts[0]).parent.resolve() diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 67bdaeb9..8b329469 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -72,16 +72,18 @@ The configuration options ``electricity: powerplants_filter`` and ``electricity: """ import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging -from scipy.spatial import cKDTree as KDTree - import pypsa import powerplantmatching as pm import pandas as pd import numpy as np +from scipy.spatial import cKDTree as KDTree + +logger = logging.getLogger(__name__) + + def add_custom_powerplants(ppl): custom_ppl_query = snakemake.config['electricity']['custom_powerplants'] if not custom_ppl_query: @@ -94,7 +96,6 @@ def add_custom_powerplants(ppl): if __name__ == "__main__": - if 'snakemake' not in globals(): from _helpers import mock_snakemake snakemake = mock_snakemake('build_powerplants') diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 6578fd75..71adb66e 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -181,27 +181,28 @@ node (`p_nom_max`): ``simple`` and ``conservative``: """ import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging -import matplotlib.pyplot as plt - import os import atlite + import numpy as np import xarray as xr import pandas as pd import multiprocessing as mp +import matplotlib.pyplot as plt +import progressbar as pgb from scipy.sparse import csr_matrix, vstack - from pypsa.geo import haversine from vresutils import landuse as vlanduse from vresutils.array import spdiag -import progressbar as pgb +logger = logging.getLogger(__name__) bounds = dx = dy = config = paths = gebco = clc = natura = None + + def init_globals(bounds_xXyY, n_dx, n_dy, n_config, n_paths): # Late import so that the GDAL Context is only created in the new processes global gl, gk, gdal @@ -227,6 +228,7 @@ def init_globals(bounds_xXyY, n_dx, n_dy, n_config, n_paths): natura = gk.raster.loadRaster(paths["natura"]) + def downsample_to_coarse_grid(bounds, dx, dy, mask, data): # The GDAL warp function with the 'average' resample algorithm needs a band of zero values of at least # the size of one coarse cell around the original raster or it produces erroneous results @@ -238,6 +240,7 @@ def downsample_to_coarse_grid(bounds, dx, dy, mask, data): assert gdal.Warp(average, padded, resampleAlg='average') == 1, "gdal warp failed: %s" % gdal.GetLastErrorMsg() return average + def calculate_potential(gid, save_map=None): feature = gk.vector.extractFeature(paths["regions"], where=gid) ec = gl.ExclusionCalculator(feature.geom) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 1d6fc5e1..2651837b 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -92,6 +92,7 @@ def _get_country(target, **keys): except (KeyError, AttributeError): return np.nan + def _simplify_polys(polys, minarea=0.1, tolerance=0.01, filterremote=True): if isinstance(polys, MultiPolygon): polys = sorted(polys, key=attrgetter('area'), reverse=True) @@ -105,6 +106,7 @@ def _simplify_polys(polys, minarea=0.1, tolerance=0.01, filterremote=True): polys = mainpoly return polys.simplify(tolerance=tolerance) + def countries(): cntries = snakemake.config['countries'] if 'RS' in cntries: cntries.append('KV') @@ -121,6 +123,7 @@ def countries(): return s + def eez(country_shapes): df = gpd.read_file(snakemake.input.eez) df = df.loc[df['ISO_3digit'].isin([_get_country('alpha_3', alpha_2=c) for c in snakemake.config['countries']])] @@ -130,6 +133,7 @@ def eez(country_shapes): s.index.name = "name" return s + def country_cover(country_shapes, eez_shapes=None): shapes = list(country_shapes) if eez_shapes is not None: @@ -140,6 +144,7 @@ def country_cover(country_shapes, eez_shapes=None): europe_shape = max(europe_shape, key=attrgetter('area')) return Polygon(shell=europe_shape.exterior) + def nuts3(country_shapes): df = gpd.read_file(snakemake.input.nuts3) df = df.loc[df['STAT_LEVL_'] == 3] @@ -158,7 +163,6 @@ def nuts3(country_shapes): .applymap(lambda x: pd.to_numeric(x, errors='coerce')) .fillna(method='bfill', axis=1))['2014'] - # Swiss data cantons = pd.read_csv(snakemake.input.ch_cantons) cantons = cantons.set_index(cantons['HASC'].str[3:])['NUTS'] cantons = cantons.str.pad(5, side='right', fillchar='0') @@ -197,6 +201,7 @@ def nuts3(country_shapes): return df + def save_to_geojson(df, fn): if os.path.exists(fn): os.unlink(fn) @@ -206,20 +211,23 @@ def save_to_geojson(df, fn): schema = {**gpd.io.file.infer_schema(df), 'geometry': 'Unknown'} df.to_file(fn, driver='GeoJSON', schema=schema) + if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake snakemake = mock_snakemake('build_shapes') configure_logging(snakemake) + out = snakemake.output + country_shapes = countries() - save_to_geojson(country_shapes, snakemake.output.country_shapes) + save_to_geojson(country_shapes, out.country_shapes) offshore_shapes = eez(country_shapes) - save_to_geojson(offshore_shapes, snakemake.output.offshore_shapes) + save_to_geojson(offshore_shapes, out.offshore_shapes) europe_shape = country_cover(country_shapes, offshore_shapes) - save_to_geojson(gpd.GeoSeries(europe_shape), snakemake.output.europe_shape) + save_to_geojson(gpd.GeoSeries(europe_shape), out.europe_shape) nuts3_shapes = nuts3(country_shapes) - save_to_geojson(nuts3_shapes, snakemake.output.nuts3_shapes) + save_to_geojson(nuts3_shapes, out.nuts3_shapes) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index ff85bcf3..2c384926 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -120,31 +120,33 @@ Exemplary unsolved network clustered to 37 nodes: """ import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging -import pandas as pd -idx = pd.IndexSlice - +import pypsa import os +import shapely + +import pandas as pd import numpy as np import geopandas as gpd -import shapely +import pyomo.environ as po import matplotlib.pyplot as plt import seaborn as sns from six.moves import reduce -import pyomo.environ as po - -import pypsa from pypsa.networkclustering import (busmap_by_kmeans, busmap_by_spectral_clustering, _make_consense, get_clustering_from_busmap) from add_electricity import load_costs -def normed(x): - return (x/x.sum()).fillna(0.) +idx = pd.IndexSlice + +logger = logging.getLogger(__name__) + + +def normed(x): return (x/x.sum()).fillna(0.) + def weighting_for_country(n, x): conv_carriers = {'OCGT','CCGT','PHS', 'hydro'} @@ -162,22 +164,13 @@ def weighting_for_country(n, x): g = normed(gen.reindex(b_i, fill_value=0)) l = normed(load.reindex(b_i, fill_value=0)) - w= g + l + w = g + l return (w * (100. / w.max())).clip(lower=1.).astype(int) -## Plot weighting for Germany - -def plot_weighting(n, country, country_shape=None): - n.plot(bus_sizes=(2*weighting_for_country(n.buses.loc[n.buses.country == country])).reindex(n.buses.index, fill_value=1)) - if country_shape is not None: - plt.xlim(country_shape.bounds[0], country_shape.bounds[2]) - plt.ylim(country_shape.bounds[1], country_shape.bounds[3]) - - -# # Determining the number of clusters per country - def distribute_clusters(n, n_clusters, focus_weights=None, solver_name=None): + """Determine the number of clusters per country""" + if solver_name is None: solver_name = snakemake.config['solving']['solver']['name'] @@ -189,7 +182,7 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name=None): N = n.buses.groupby(['country', 'sub_network']).size() assert n_clusters >= len(N) and n_clusters <= N.sum(), \ - "Number of clusters must be {} <= n_clusters <= {} for this selection of countries.".format(len(N), N.sum()) + f"Number of clusters must be {len(N)} <= n_clusters <= {N.sum()} for this selection of countries." if focus_weights is not None: @@ -205,7 +198,7 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name=None): logger.warning('Using custom focus weights for determining number of clusters.') - assert np.isclose(L.sum(), 1.0, rtol=1e-3), "Country weights L must sum up to 1.0 when distributing clusters. Is {}.".format(L.sum()) + assert np.isclose(L.sum(), 1.0, rtol=1e-3), f"Country weights L must sum up to 1.0 when distributing clusters. Is {L.sum()}." m = po.ConcreteModel() def n_bounds(model, *n_id): @@ -221,10 +214,11 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name=None): opt = po.SolverFactory('ipopt') results = opt.solve(m) - assert results['Solver'][0]['Status'] == 'ok', "Solver returned non-optimally: {}".format(results) + assert results['Solver'][0]['Status'] == 'ok', f"Solver returned non-optimally: {results}" return pd.Series(m.n.get_values(), index=L.index).astype(int) + def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algorithm="kmeans", **algorithm_kwds): if algorithm == "kmeans": algorithm_kwds.setdefault('n_init', 1000) @@ -243,7 +237,7 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori def busmap_for_country(x): prefix = x.name[0] + x.name[1] + ' ' - logger.debug("Determining busmap for country {}".format(prefix[:-1])) + logger.debug(f"Determining busmap for country {prefix[:-1]}") if len(x) == 1: return pd.Series(prefix + '0', index=x.index) weight = weighting_for_country(n, x) @@ -260,12 +254,6 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori return (n.buses.groupby(['country', 'sub_network'], group_keys=False, squeeze=True) .apply(busmap_for_country).rename('busmap')) -def plot_busmap_for_n_clusters(n, n_clusters=50): - busmap = busmap_for_n_clusters(n, n_clusters) - cs = busmap.unique() - cr = sns.color_palette("hls", len(cs)) - n.plot(bus_colors=busmap.map(dict(zip(cs, cr)))) - del cs, cr def clustering_for_n_clusters(n, n_clusters, aggregate_carriers=None, line_length_factor=1.25, potential_mode='simple', @@ -277,8 +265,7 @@ def clustering_for_n_clusters(n, n_clusters, aggregate_carriers=None, elif potential_mode == 'conservative': p_nom_max_strategy = np.min else: - raise AttributeError("potential_mode should be one of 'simple' or 'conservative', " - "but is '{}'".format(potential_mode)) + raise AttributeError(f"potential_mode should be one of 'simple' or 'conservative' but is '{potential_mode}'") clustering = get_clustering_from_busmap( n, busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm), @@ -301,6 +288,7 @@ def clustering_for_n_clusters(n, n_clusters, aggregate_carriers=None, return clustering + def save_to_geojson(s, fn): if os.path.exists(fn): os.unlink(fn) @@ -308,6 +296,7 @@ def save_to_geojson(s, fn): schema = {**gpd.io.file.infer_schema(df), 'geometry': 'Unknown'} df.to_file(fn, driver='GeoJSON', schema=schema) + def cluster_regions(busmaps, input=None, output=None): if input is None: input = snakemake.input if output is None: output = snakemake.output @@ -321,6 +310,17 @@ def cluster_regions(busmaps, input=None, output=None): regions_c.index.name = 'name' save_to_geojson(regions_c, getattr(output, which)) + +def plot_busmap_for_n_clusters(n, n_clusters, fn=None): + busmap = busmap_for_n_clusters(n, n_clusters) + cs = busmap.unique() + cr = sns.color_palette("hls", len(cs)) + n.plot(bus_colors=busmap.map(dict(zip(cs, cr)))) + if fn is not None: + plt.savefig(fn, bbox_inches='tight') + del cs, cr + + if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake diff --git a/scripts/make_summary.py b/scripts/make_summary.py index db9eff46..ac0119ab 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -54,22 +54,22 @@ Replacing '/summaries/' with '/plots/' creates nice colored maps of the results. """ import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging import os - -from six import iteritems +import pypsa import pandas as pd -import pypsa - +from six import iteritems from add_electricity import load_costs, update_transmission_costs idx = pd.IndexSlice +logger = logging.getLogger(__name__) + opt_name = {"Store": "e", "Line" : "s", "Transformer" : "s"} + def _add_indexed_rows(df, raw_index): new_index = df.index|pd.MultiIndex.from_product(raw_index) if isinstance(new_index, pd.Index): @@ -77,6 +77,7 @@ def _add_indexed_rows(df, raw_index): return df.reindex(new_index) + def assign_carriers(n): if "carrier" not in n.loads: @@ -97,7 +98,8 @@ def assign_carriers(n): if "EU gas store" in n.stores.index and n.stores.loc["EU gas Store","carrier"] == "": n.stores.loc["EU gas Store","carrier"] = "gas Store" -def calculate_costs(n,label,costs): + +def calculate_costs(n, label, costs): for c in n.iterate_components(n.branch_components|n.controllable_one_port_components^{"Load"}): capital_costs = c.df.capital_cost*c.df[opt_name.get(c.name,"p") + "_nom_opt"] @@ -130,7 +132,7 @@ def calculate_costs(n,label,costs): return costs -def calculate_curtailment(n,label,curtailment): +def calculate_curtailment(n, label, curtailment): avail = n.generators_t.p_max_pu.multiply(n.generators.p_nom_opt).sum().groupby(n.generators.carrier).sum() used = n.generators_t.p.sum().groupby(n.generators.carrier).sum() @@ -139,7 +141,7 @@ def calculate_curtailment(n,label,curtailment): return curtailment -def calculate_energy(n,label,energy): +def calculate_energy(n, label, energy): for c in n.iterate_components(n.one_port_components|n.branch_components): @@ -159,6 +161,7 @@ def include_in_summary(summary, multiindexprefix, label, item): summary = _add_indexed_rows(summary, raw_index) summary.loc[idx[raw_index], label] = item.values + return summary def calculate_capacity(n,label,capacity): @@ -178,7 +181,7 @@ def calculate_capacity(n,label,capacity): return capacity -def calculate_supply(n,label,supply): +def calculate_supply(n, label, supply): """calculate the max dispatch of each component at the buses where the loads are attached""" load_types = n.loads.carrier.value_counts().index @@ -224,7 +227,8 @@ def calculate_supply(n,label,supply): return supply -def calculate_supply_energy(n,label,supply_energy): + +def calculate_supply_energy(n, label, supply_energy): """calculate the total dispatch of each component at the buses where the loads are attached""" load_types = n.loads.carrier.value_counts().index @@ -269,6 +273,7 @@ def calculate_supply_energy(n,label,supply_energy): return supply_energy + def calculate_metrics(n,label,metrics): metrics = metrics.reindex(metrics.index|pd.Index(["line_volume","line_volume_limit","line_volume_AC","line_volume_DC","line_volume_shadow","co2_shadow"])) @@ -295,16 +300,15 @@ def calculate_prices(n,label,prices): prices = prices.reindex(prices.index|bus_type.value_counts().index) - #WARNING: this is time-averaged, should really be load-weighted average + logger.warning("Prices are time-averaged, not load-weighted") prices[label] = n.buses_t.marginal_price.mean().groupby(bus_type).mean() return prices - def calculate_weighted_prices(n,label,weighted_prices): - # Warning: doesn't include storage units as loads - + + logger.warning("Weighted prices don't include storage units as loads") weighted_prices = weighted_prices.reindex(pd.Index(["electricity","heat","space heat","urban heat","space urban heat","gas","H2"])) @@ -347,7 +351,7 @@ def calculate_weighted_prices(n,label,weighted_prices): load += n.links_t.p0[names].groupby(n.links.loc[names,"bus0"],axis=1).sum(axis=1) - #Add H2 Store when charging + # Add H2 Store when charging if carrier == "H2": stores = n.stores_t.p[buses+ " Store"].groupby(n.stores.loc[buses+ " Store","bus"],axis=1).sum(axis=1) stores[stores > 0.] = 0. @@ -361,62 +365,6 @@ def calculate_weighted_prices(n,label,weighted_prices): return weighted_prices - -# BROKEN don't use -# -# def calculate_market_values(n, label, market_values): -# # Warning: doesn't include storage units - -# n.buses["suffix"] = n.buses.index.str[2:] -# suffix = "" -# buses = n.buses.index[n.buses.suffix == suffix] - -# ## First do market value of generators ## -# generators = n.generators.index[n.buses.loc[n.generators.bus,"suffix"] == suffix] -# techs = n.generators.loc[generators,"carrier"].value_counts().index -# market_values = market_values.reindex(market_values.index | techs) - -# for tech in techs: -# gens = generators[n.generators.loc[generators,"carrier"] == tech] -# dispatch = n.generators_t.p[gens].groupby(n.generators.loc[gens,"bus"],axis=1).sum().reindex(columns=buses,fill_value=0.) -# revenue = dispatch*n.buses_t.marginal_price[buses] -# market_values.at[tech,label] = revenue.sum().sum()/dispatch.sum().sum() - -# ## Now do market value of links ## - -# for i in ["0","1"]: -# all_links = n.links.index[n.buses.loc[n.links["bus"+i],"suffix"] == suffix] -# techs = n.links.loc[all_links,"carrier"].value_counts().index -# market_values = market_values.reindex(market_values.index | techs) - -# for tech in techs: -# links = all_links[n.links.loc[all_links,"carrier"] == tech] -# dispatch = n.links_t["p"+i][links].groupby(n.links.loc[links,"bus"+i],axis=1).sum().reindex(columns=buses,fill_value=0.) -# revenue = dispatch*n.buses_t.marginal_price[buses] -# market_values.at[tech,label] = revenue.sum().sum()/dispatch.sum().sum() - -# return market_values - - -# OLD CODE must be adapted - -# def calculate_price_statistics(n, label, price_statistics): - - -# price_statistics = price_statistics.reindex(price_statistics.index|pd.Index(["zero_hours","mean","standard_deviation"])) -# n.buses["suffix"] = n.buses.index.str[2:] -# suffix = "" -# buses = n.buses.index[n.buses.suffix == suffix] - -# threshold = 0.1 #higher than phoney marginal_cost of wind/solar -# df = pd.DataFrame(data=0.,columns=buses,index=n.snapshots) -# df[n.buses_t.marginal_price[buses] < threshold] = 1. -# price_statistics.at["zero_hours", label] = df.sum().sum()/(df.shape[0]*df.shape[1]) -# price_statistics.at["mean", label] = n.buses_t.marginal_price[buses].unstack().mean() -# price_statistics.at["standard_deviation", label] = n.buses_t.marginal_price[buses].unstack().std() -# return price_statistics - - outputs = ["costs", "curtailment", "energy", @@ -425,11 +373,10 @@ outputs = ["costs", "supply_energy", "prices", "weighted_prices", - # "price_statistics", - # "market_values", "metrics", ] + def make_summaries(networks_dict, country='all'): columns = pd.MultiIndex.from_tuples(networks_dict.keys(),names=["simpl","clusters","ll","opts"]) @@ -454,7 +401,7 @@ def make_summaries(networks_dict, country='all'): if country != 'all': n = n[n.buses.country == country] - Nyears = n.snapshot_weightings.sum()/8760. + Nyears = n.snapshot_weightings.sum() / 8760. costs = load_costs(Nyears, snakemake.input[0], snakemake.config['costs'], snakemake.config['electricity']) update_transmission_costs(n, costs, simple_hvdc_costs=False) @@ -484,7 +431,6 @@ if __name__ == "__main__": network_dir = os.path.join('results', 'networks') configure_logging(snakemake) - def expand_from_wildcard(key): w = getattr(snakemake.wildcards, key) return snakemake.config["scenario"][key] if w == "all" else [w] @@ -504,8 +450,6 @@ if __name__ == "__main__": for l in ll for opts in expand_from_wildcard("opts")} - print(networks_dict) - dfs = make_summaries(networks_dict, country=snakemake.wildcards.country) to_csv(dfs) diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 4e162e2f..84423916 100755 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -20,7 +20,6 @@ Description """ import logging -logger = logging.getLogger(__name__) from _helpers import (load_network_for_plots, aggregate_p, aggregate_costs, configure_logging) @@ -35,6 +34,9 @@ from matplotlib.patches import Circle, Ellipse from matplotlib.legend_handler import HandlerPatch to_rgba = mpl.colors.colorConverter.to_rgba +logger = logging.getLogger(__name__) + + def make_handler_map_to_scale_circles_as_in(ax, dont_resize_actively=False): fig = ax.get_figure() def axes2pt(): @@ -57,9 +59,11 @@ def make_handler_map_to_scale_circles_as_in(ax, dont_resize_actively=False): return e return {Circle: HandlerPatch(patch_func=legend_circle_handler)} + def make_legend_circles_for(sizes, scale=1.0, **kw): return [Circle((0,0), radius=(s/scale)**0.5, **kw) for s in sizes] + def set_plot_style(): plt.style.use(['classic', 'seaborn-white', {'axes.grid': False, 'grid.linestyle': '--', 'grid.color': u'0.6', @@ -69,9 +73,9 @@ def set_plot_style(): 'legend.fontsize': 'medium', 'lines.linewidth': 1.5, 'pdf.fonttype': 42, - # 'font.family': 'Times New Roman' }]) + def plot_map(n, ax=None, attribute='p_nom', opts={}): if ax is None: ax = plt.gca() @@ -114,16 +118,11 @@ def plot_map(n, ax=None, attribute='p_nom', opts={}): bus_sizes=0, bus_colors=tech_colors, boundaries=map_boundaries, - geomap=True, # TODO : Turn to False, after the release of PyPSA 0.14.2 (refer to https://github.com/PyPSA/PyPSA/issues/75) + geomap=False, ax=ax) ax.set_aspect('equal') ax.axis('off') - # x1, y1, x2, y2 = map_boundaries - # ax.set_xlim(x1, x2) - # ax.set_ylim(y1, y2) - - # Rasterize basemap # TODO : Check if this also works with cartopy for c in ax.collections[:2]: c.set_rasterized(True) @@ -176,13 +175,9 @@ def plot_map(n, ax=None, attribute='p_nom', opts={}): return fig -#n = load_network_for_plots(snakemake.input.network, opts, combine_hydro_ps=False) - def plot_total_energy_pie(n, ax=None): - """Add total energy pie plot""" - if ax is None: - ax = plt.gca() + if ax is None: ax = plt.gca() ax.set_title('Energy per technology', fontdict=dict(fontsize="medium")) @@ -190,7 +185,7 @@ def plot_total_energy_pie(n, ax=None): patches, texts, autotexts = ax.pie(e_primary, startangle=90, - labels = e_primary.rename(opts['nice_names_n']).index, + labels = e_primary.rename(opts['nice_names']).index, autopct='%.0f%%', shadow=False, colors = [opts['tech_colors'][tech] for tech in e_primary.index]) @@ -200,9 +195,7 @@ def plot_total_energy_pie(n, ax=None): t2.remove() def plot_total_cost_bar(n, ax=None): - """Add average system cost bar plot""" - if ax is None: - ax = plt.gca() + if ax is None: ax = plt.gca() total_load = (n.snapshot_weightings * n.loads_t.p.sum(axis=1)).sum() tech_colors = opts['tech_colors'] @@ -240,14 +233,13 @@ def plot_total_cost_bar(n, ax=None): if abs(data[-1]) < 5: continue - text = ax.text(1.1,(bottom-0.5*data)[-1]-3,opts['nice_names_n'].get(ind,ind)) + text = ax.text(1.1,(bottom-0.5*data)[-1]-3,opts['nice_names'].get(ind,ind)) texts.append(text) ax.set_ylabel("Average system cost [Eur/MWh]") - ax.set_ylim([0, 80]) # opts['costs_max']]) + ax.set_ylim([0, opts.get('costs_max', 80)]) ax.set_xlim([0, 1]) - #ax.set_xticks([0.5]) - ax.set_xticklabels([]) #["w/o\nEp", "w/\nEp"]) + ax.set_xticklabels([]) ax.grid(True, axis="y", color='k', linestyle='dotted') @@ -280,8 +272,6 @@ if __name__ == "__main__": ax2 = fig.add_axes([-0.075, 0.1, 0.1, 0.45]) plot_total_cost_bar(n, ax2) - #fig.tight_layout() - ll = snakemake.wildcards.ll ll_type = ll[0] ll_factor = ll[1:] diff --git a/scripts/plot_p_nom_max.py b/scripts/plot_p_nom_max.py index 0c2e06f2..bc346785 100644 --- a/scripts/plot_p_nom_max.py +++ b/scripts/plot_p_nom_max.py @@ -19,19 +19,19 @@ Description """ import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging import pypsa - import pandas as pd import matplotlib.pyplot as plt +logger = logging.getLogger(__name__) + + def cum_p_nom_max(net, tech, country=None): carrier_b = net.generators.carrier == tech - generators = \ - pd.DataFrame(dict( + generators = pd.DataFrame(dict( p_nom_max=net.generators.loc[carrier_b, 'p_nom_max'], p_max_pu=net.generators_t.p_max_pu.loc[:,carrier_b].mean(), country=net.generators.loc[carrier_b, 'bus'].map(net.buses.country) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 8eceea91..4aaa0d37 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -21,41 +21,19 @@ Description import os import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging import pandas as pd import matplotlib.pyplot as plt -#consolidate and rename -def rename_techs(label): - if label.startswith("central "): - label = label[len("central "):] - elif label.startswith("urban "): - label = label[len("urban "):] +logger = logging.getLogger(__name__) - if "retrofitting" in label: - label = "building retrofitting" + +def rename_techs(label): elif "H2" in label: label = "hydrogen storage" - elif "CHP" in label: - label = "CHP" - elif "water tank" in label: - label = "water tanks" - elif label == "water tanks": - label = "hot water storage" - elif "gas" in label and label != "gas boiler": - label = "natural gas" - elif "solar thermal" in label: - label = "solar thermal" elif label == "solar": label = "solar PV" - elif label == "heat pump": - label = "air heat pump" - elif label == "Sabatier": - label = "methanation" - elif label == "offwind": - label = "offshore wind" elif label == "offwind-ac": label = "offshore wind ac" elif label == "offwind-dc": @@ -68,15 +46,14 @@ def rename_techs(label): label = "hydroelectricity" elif label == "PHS": label = "hydroelectricity" - elif label == "co2 Store": - label = "DAC" elif "battery" in label: label = "battery storage" return label -preferred_order = pd.Index(["transmission lines","hydroelectricity","hydro reservoir","run of river","pumped hydro storage","onshore wind","offshore wind ac", "offshore wind dc","solar PV","solar thermal","building retrofitting","ground heat pump","air heat pump","resistive heater","CHP","OCGT","gas boiler","gas","natural gas","methanation","hydrogen storage","battery storage","hot water storage"]) +preferred_order = pd.Index(["transmission lines","hydroelectricity","hydro reservoir","run of river","pumped hydro storage","onshore wind","offshore wind ac", "offshore wind dc","solar PV","solar thermal","OCGT","hydrogen storage","battery storage"]) + def plot_costs(infn, fn=None): diff --git a/scripts/prepare_links_p_nom.py b/scripts/prepare_links_p_nom.py index 757e8345..7623d1bf 100644 --- a/scripts/prepare_links_p_nom.py +++ b/scripts/prepare_links_p_nom.py @@ -37,11 +37,26 @@ Description """ import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging import pandas as pd +logger = logging.getLogger(__name__) + + +def multiply(s): + return s.str[0].astype(float) * s.str[1].astype(float) + + +def extract_coordinates(s): + regex = (r"(\d{1,2})°(\d{1,2})′(\d{1,2})″(N|S) " + r"(\d{1,2})°(\d{1,2})′(\d{1,2})″(E|W)") + e = s.str.extract(regex, expand=True) + lat = (e[0].astype(float) + (e[1].astype(float) + e[2].astype(float)/60.)/60.)*e[3].map({'N': +1., 'S': -1.}) + lon = (e[4].astype(float) + (e[5].astype(float) + e[6].astype(float)/60.)/60.)*e[7].map({'E': +1., 'W': -1.}) + return lon, lat + + if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake #rule must be enabled in config @@ -50,19 +65,11 @@ if __name__ == "__main__": links_p_nom = pd.read_html('https://en.wikipedia.org/wiki/List_of_HVDC_projects', header=0, match="SwePol")[0] - def extract_coordinates(s): - regex = (r"(\d{1,2})°(\d{1,2})′(\d{1,2})″(N|S) " - r"(\d{1,2})°(\d{1,2})′(\d{1,2})″(E|W)") - e = s.str.extract(regex, expand=True) - lat = (e[0].astype(float) + (e[1].astype(float) + e[2].astype(float)/60.)/60.)*e[3].map({'N': +1., 'S': -1.}) - lon = (e[4].astype(float) + (e[5].astype(float) + e[6].astype(float)/60.)/60.)*e[7].map({'E': +1., 'W': -1.}) - return lon, lat + mw = "Power (MW)" + m_b = links_p_nom[mw].str.contains('x').fillna(False) - m_b = links_p_nom["Power (MW)"].str.contains('x').fillna(False) - def multiply(s): return s.str[0].astype(float) * s.str[1].astype(float) - - links_p_nom.loc[m_b, "Power (MW)"] = links_p_nom.loc[m_b, "Power (MW)"].str.split('x').pipe(multiply) - links_p_nom["Power (MW)"] = links_p_nom["Power (MW)"].str.extract("[-/]?([\d.]+)", expand=False).astype(float) + links_p_nom.loc[m_b, mw] = links_p_nom.loc[m_b, mw].str.split('x').pipe(multiply) + links_p_nom[mw] = links_p_nom[mw].str.extract("[-/]?([\d.]+)", expand=False).astype(float) links_p_nom['x1'], links_p_nom['y1'] = extract_coordinates(links_p_nom['Converterstation 1']) links_p_nom['x2'], links_p_nom['y2'] = extract_coordinates(links_p_nom['Converterstation 2']) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 1fcba99f..bb7640be 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -55,19 +55,21 @@ Description """ import logging -logger = logging.getLogger(__name__) from _helpers import configure_logging -from add_electricity import load_costs, update_transmission_costs -from six import iteritems - -import numpy as np import re import pypsa +import numpy as np import pandas as pd +from six import iteritems + +from add_electricity import load_costs, update_transmission_costs idx = pd.IndexSlice +logger = logging.getLogger(__name__) + + def add_co2limit(n, Nyears=1., factor=None): if factor is not None: @@ -129,10 +131,10 @@ def set_transmission_limit(n, ll_type, factor, Nyears=1): n.add('GlobalConstraint', f'l{ll_type}_limit', type=f'transmission_{con_type}_limit', sense='<=', constant=rhs, carrier_attribute='AC, DC') + return n - def average_every_nhours(n, offset): logger.info('Resampling the network to {}'.format(offset)) m = n.copy(with_time=False) @@ -160,7 +162,7 @@ if __name__ == "__main__": opts = snakemake.wildcards.opts.split('-') n = pypsa.Network(snakemake.input[0]) - Nyears = n.snapshot_weightings.sum()/8760. + Nyears = n.snapshot_weightings.sum() / 8760. set_line_s_max_pu(n) @@ -179,6 +181,7 @@ if __name__ == "__main__": add_co2limit(n, Nyears, float(m[0])) else: add_co2limit(n, Nyears) + break for o in opts: oo = o.split("+") diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 9bb85833..7ee6c2b1 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -33,14 +33,15 @@ The :ref:`tutorial` uses a smaller `data bundle Date: Fri, 11 Sep 2020 15:26:43 +0200 Subject: [PATCH 002/173] execute pre-commit --- .github/ISSUE_TEMPLATE/bug_report.md | 4 +- .github/ISSUE_TEMPLATE/config.yml | 2 +- .github/pull_request_template.md | 4 +- .pre-commit-config.yaml | 8 ++++ .readthedocs.yml | 2 +- .reuse/dep5 | 2 +- .travis.yml | 5 ++- README.md | 2 +- Snakefile | 20 +++++----- data/links_tyndp.csv | 2 +- data/parameter_corrections.yaml | 12 +++--- doc/_static/theme_overrides.css | 4 +- doc/cloudcomputing.rst | 40 +++++++++---------- doc/configtables/atlite.csv | 2 +- doc/configtables/costs.csv | 2 +- doc/configtables/hydro.csv | 2 +- doc/configtables/lines.csv | 2 +- doc/configtables/links.csv | 2 +- doc/configtables/load.csv | 2 +- doc/configtables/offwind-ac.csv | 2 +- doc/configtables/offwind-dc.csv | 2 +- doc/configtables/onwind.csv | 2 +- doc/configtables/opts.csv | 2 +- doc/configtables/plotting.csv | 2 +- doc/configtables/scenario.csv | 2 +- doc/configtables/snapshots.csv | 2 +- doc/configtables/solar.csv | 2 +- doc/configtables/solving-options.csv | 2 +- doc/configtables/solving-solver.csv | 2 +- doc/configtables/toplevel.csv | 2 +- doc/configtables/transformers.csv | 2 +- doc/configuration.rst | 4 +- doc/costs.rst | 2 +- doc/index.rst | 10 ++--- doc/installation.rst | 16 ++++---- doc/introduction.rst | 4 +- doc/limitations.rst | 3 +- doc/plotting.rst | 2 +- doc/preparation/base_network.rst | 4 +- doc/preparation/build_bus_regions.rst | 2 +- doc/preparation/build_cutout.rst | 2 +- doc/release_notes.rst | 8 ++-- doc/simplification.rst | 2 +- doc/tutorial.rst | 16 ++++---- doc/wildcards.rst | 4 +- .../environment.docs.yaml | 29 ++++---------- .../environment.fixed.yaml | 0 environment.yaml => envs/environment.yaml | 0 scripts/add_electricity.py | 2 +- scripts/base_network.py | 16 ++++---- scripts/cluster_network.py | 2 +- scripts/make_summary.py | 2 +- scripts/prepare_network.py | 2 +- test/config.test1.yaml | 1 - 54 files changed, 135 insertions(+), 141 deletions(-) create mode 100644 .pre-commit-config.yaml rename environment.docs.yaml => envs/environment.docs.yaml (60%) rename environment.fixedversions.yaml => envs/environment.fixed.yaml (100%) rename environment.yaml => envs/environment.yaml (100%) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index a1671b69..2654595c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -11,7 +11,7 @@ assignees: '' ## Checklist - [ ] I am using the current [`master`](https://github.com/PyPSA/pypsa-eur/tree/master) branch or the latest [release](https://github.com/PyPSA/pypsa-eur/releases). Please indicate. -- [ ] I am running on an up-to-date [`pypsa-eur` environment](https://github.com/PyPSA/pypsa-eur/blob/master/environment.yaml). Update via `conda env update -f environment.yaml`. +- [ ] I am running on an up-to-date [`pypsa-eur` environment](https://github.com/PyPSA/pypsa-eur/blob/master/envs/environment.yaml). Update via `conda env update -f envs/environment.yaml`. ## Describe the Bug @@ -24,4 +24,4 @@ assignees: '' ``` -``` \ No newline at end of file +``` diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index bb1951c0..5b64d242 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,4 +2,4 @@ blank_issues_enabled: false contact_links: - name: PyPSA Mailing List url: https://groups.google.com/forum/#!forum/pypsa - about: Please ask and answer general usage questions here. \ No newline at end of file + about: Please ask and answer general usage questions here. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index b431c08a..05601c58 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -7,7 +7,7 @@ Closes # (if applicable). - [ ] I tested my contribution locally and it seems to work fine. - [ ] Code and workflow changes are sufficiently documented. -- [ ] Newly introduced dependencies are added to `environment.yaml` and `environment.docs.yaml`. +- [ ] Newly introduced dependencies are added to `envs/environment.yaml` and `envs/environment.docs.yaml`. - [ ] Changes in configuration options are added in all of `config.default.yaml`, `config.tutorial.yaml`, and `test/config.test1.yaml`. - [ ] Changes in configuration options are also documented in `doc/configtables/*.csv` and line references are adjusted in `doc/configuration.rst` and `doc/tutorial.rst`. -- [ ] A note for the release notes `doc/release_notes.rst` is amended in the format of previous release notes. \ No newline at end of file +- [ ] A note for the release notes `doc/release_notes.rst` is amended in the format of previous release notes. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..c5d05ee9 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,8 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files diff --git a/.readthedocs.yml b/.readthedocs.yml index 0ccb7def..173d21d7 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,4 +5,4 @@ version: 2 conda: - environment: environment.docs.yaml \ No newline at end of file + environment: envs/environment.docs.yaml diff --git a/.reuse/dep5 b/.reuse/dep5 index ef2e4f85..eb64a172 100644 --- a/.reuse/dep5 +++ b/.reuse/dep5 @@ -25,4 +25,4 @@ License: CC0-1.0 Files: borg-it Copyright: : 2017-2020 The PyPSA-Eur Authors -License: CC0-1.0 \ No newline at end of file +License: CC0-1.0 diff --git a/.travis.yml b/.travis.yml index a8e6e36e..75c5d524 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,11 +15,12 @@ before_install: - source conda4travis.sh # install conda environment - - conda env create -f ./environment.yaml + - conda install -c conda-forge mamba + - mamba env create -f ./envs/environment.yaml - conda activate pypsa-eur # install open-source solver - - conda install -c conda-forge ipopt glpk + - mamba install -c conda-forge ipopt glpk script: - cp ./test/config.test1.yaml ./config.yaml diff --git a/README.md b/README.md index dcc7c6fc..c281e589 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ and local grid bottlenecks may cause unrealistic load-shedding or generator curtailment. We recommend to cluster the network to a couple of hundred nodes to remove these local inconsistencies. See the -discussion in Section 3.4 "Model validation" of the paper. +discussion in Section 3.4 "Model validation" of the paper. ![PyPSA-Eur Grid Model](doc/img/base.png) diff --git a/Snakefile b/Snakefile index ce27174a..3eddad7c 100644 --- a/Snakefile +++ b/Snakefile @@ -46,11 +46,11 @@ if config['enable'].get('prepare_links_p_nom', False): script: 'scripts/prepare_links_p_nom.py' -datafiles = ['ch_cantons.csv', 'je-e-21.03.02.xls', - 'eez/World_EEZ_v8_2014.shp', 'EIA_hydro_generation_2000_2014.csv', - 'hydro_capacities.csv', 'naturalearth/ne_10m_admin_0_countries.shp', - 'NUTS_2013_60M_SH/data/NUTS_RG_60M_2013.shp', 'nama_10r_3popgdp.tsv.gz', - 'nama_10r_3gdp.tsv.gz', 'time_series_60min_singleindex_filtered.csv', +datafiles = ['ch_cantons.csv', 'je-e-21.03.02.xls', + 'eez/World_EEZ_v8_2014.shp', 'EIA_hydro_generation_2000_2014.csv', + 'hydro_capacities.csv', 'naturalearth/ne_10m_admin_0_countries.shp', + 'NUTS_2013_60M_SH/data/NUTS_RG_60M_2013.shp', 'nama_10r_3popgdp.tsv.gz', + 'nama_10r_3gdp.tsv.gz', 'time_series_60min_singleindex_filtered.csv', 'corine/g250_clc06_V18_5.tif'] @@ -131,7 +131,7 @@ rule build_bus_regions: script: "scripts/build_bus_regions.py" -if config['enable'].get('build_cutout', False): +if config['enable'].get('build_cutout', False): rule build_cutout: output: directory("cutouts/{cutout}") log: "logs/build_cutout/{cutout}.log" @@ -148,9 +148,9 @@ if config['enable'].get('retrieve_cutout', True): script: 'scripts/retrieve_cutout.py' -if config['enable'].get('build_natura_raster', False): +if config['enable'].get('build_natura_raster', False): rule build_natura_raster: - input: + input: natura="data/bundle/natura/Natura2000_end2015.shp", cutouts=expand("cutouts/{cutouts}", **config['atlite']) output: "resources/natura.tiff" @@ -179,7 +179,7 @@ rule build_renewable_profiles: if w.technology in ('onwind', 'solar') else "resources/regions_offshore.geojson"), cutout=lambda w: "cutouts/" + config["renewable"][w.technology]['cutout'] - output: + output: profile="resources/profile_{technology}.nc", log: "logs/build_renewable_profile_{technology}.log" benchmark: "benchmarks/build_renewable_profiles_{technology}" @@ -210,7 +210,7 @@ rule add_electricity: geth_hydro_capacities='data/geth2015_hydro_capacities.csv', opsd_load='data/bundle/time_series_60min_singleindex_filtered.csv', nuts3_shapes='resources/nuts3_shapes.geojson', - **{'profile_' + tech: "resources/profile_" + tech + ".nc" + **{f"profile_{tech}": f"resources/profile_{tech}.nc" for tech in config['renewable']} output: "networks/elec.nc" log: "logs/add_electricity.log" diff --git a/data/links_tyndp.csv b/data/links_tyndp.csv index 5d0aec28..ca977c27 100644 --- a/data/links_tyndp.csv +++ b/data/links_tyndp.csv @@ -23,4 +23,4 @@ HVDC Ultranet,Osterath (DE),Philippsburg (DE),,314,600,in permitting,,https://ty Gridlink,Kingsnorth (UK),Warande (FR),160,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/285,0.596111111111111,51.41972,2.376776,51.034368 NeuConnect,Grain (UK),Fedderwarden (DE),680,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/309,0.716666666666667,51.44,8.046524,53.562763 NordBalt,Klaipeda (LT),Nybro (SE),450,,700,built,,https://en.wikipedia.org/wiki/NordBalt,21.256667,55.681667,15.854167,56.767778 -Estlink 1,Harku (EE),Espoo (FI),105,,350,built,,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889 \ No newline at end of file +Estlink 1,Harku (EE),Espoo (FI),105,,350,built,,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889 diff --git a/data/parameter_corrections.yaml b/data/parameter_corrections.yaml index 83a17d51..0962b1b8 100644 --- a/data/parameter_corrections.yaml +++ b/data/parameter_corrections.yaml @@ -33,12 +33,12 @@ Link: "14559": "6240" # fix wrong bus allocation from 6241 "12998": "1333" # combine link 12998 + 12997 in 12998 "5627": '2309' # combine link 5627 + 5628 in 5627 - length: + length: index: - "12998": 409.0 + "12998": 409.0 "5627": 26.39 - bus0: - index: + bus0: + index: # set bus0 == bus1 for removing the link in remove_unconnected_components - "5628": "7276" - "12997": "7276" \ No newline at end of file + "5628": "7276" + "12997": "7276" diff --git a/doc/_static/theme_overrides.css b/doc/_static/theme_overrides.css index efbe909c..778756c1 100644 --- a/doc/_static/theme_overrides.css +++ b/doc/_static/theme_overrides.css @@ -11,7 +11,7 @@ white-space: normal !important; /* background: #eeeeee !important; */ } - + .wy-table-responsive { max-width: 100%; overflow: visible !important; @@ -20,4 +20,4 @@ .wy-nav-content { max-width: 910px !important; } - } \ No newline at end of file + } diff --git a/doc/cloudcomputing.rst b/doc/cloudcomputing.rst index 3da6b9c2..f91441a5 100644 --- a/doc/cloudcomputing.rst +++ b/doc/cloudcomputing.rst @@ -1,6 +1,6 @@ .. SPDX-FileCopyrightText: 2020 Maximilian Parzen and Emmanuel Paez - + SPDX-License-Identifier: CC-BY-4.0 @@ -18,7 +18,7 @@ Google Cloud Platform (GCP) We are happy to take pull requests explaining where the procedures deviate from the descriptions below for other operating systems. Likewise, tutorials for other cloud computing solutions are also highly welcome. -The Google Cloud Platform (GCP) is a cloud computing service you can use to run PyPSA-Eur calculations, especially if +The Google Cloud Platform (GCP) is a cloud computing service you can use to run PyPSA-Eur calculations, especially if - you do not have immediate access to high-performance computating facilities, - you have problems with the Windows operating system and want a quick run on a linux-based system, @@ -26,7 +26,7 @@ The Google Cloud Platform (GCP) is a cloud computing service you can use to run - you need quick results (trial version includes 32 vCPU cores and up to 800 GB of memory). With the Google Cloud Platform you set up a virtual machine in the cloud which can store and operate data. -Like on your local computer, you have to install all software and solvers, and create paths on the virtual machine to set up the required environment. +Like on your local computer, you have to install all software and solvers, and create paths on the virtual machine to set up the required environment. The 300$ free trial budget is offered which equals roughly 10-20 simulations with 180 nodes at hourly basis. The following steps are required: @@ -38,7 +38,7 @@ The following steps are required: Step 1 - Google Cloud Platform registration ------------------------------------------- -First, register at the `Google Cloud Platform `_ (GCP). +First, register at the `Google Cloud Platform `_ (GCP). Ann active bank account is required, which will not be charged unless you exceed the trial budget. Step 2 - Create your Virtual Machine instance @@ -73,26 +73,26 @@ Step 3 - Installation of Cloud SDK - Download Google Cloud SDK `SDK `_. Check that you are logged in in your Google account. The link should lead you to the Windows installation of Google Cloud SDK. - Follow the "Quickstart for Windows - Before you begin" steps. - After the successfull installation and initialization, close the Google Cloud SDK reopen it again. Type the following command into the "Google Cloud SDK Shell": - + .. code:: bash - + gcloud compute ssh -- -L 8888:localhost:8888 - + - This command above will open a PuTTy command window that is connected to your Virtual Machine. Time to celebrate if it works! -- Now install all necessary tools. As little help, the first steps: +- Now install all necessary tools. As little help, the first steps: .. code:: bash - + sudo apt-get update sudo apt-get install bzip2 libxml2-dev sudo apt-get install wget wget https://repo.anaconda.com/archive/Anaconda3-2020.07-Linux-x86_64.sh (Check the link. To be up to date with anaconda, check the Anaconda website https://www.anaconda.com/products/individual ) ls (to see what anaconda file to bash) - bash Anaconda3-2020.07-Linux-x86_64.sh - source ~/.bashrc - + bash Anaconda3-2020.07-Linux-x86_64.sh + source ~/.bashrc + - Close and reopen the PuTTy file (-> open Google Cloud SDK -> initialize again with the command above to open the PuTTY command window). Now ``conda`` can be listed with ``conda list``. Noq you can follow the standard installation instructions to finalize your machine (don't forget the solvers - for bigger simulations use commercial solvers such as Gurobi). - + Step 4 - Installation of WinSCP ------------------------------- @@ -102,22 +102,22 @@ Make sure that your instance is operating for the next steps. - Download `WinSCP `_ and follow the default installation steps. - Open WinSCP after the installation. A login window will open. - Keep SFTP as file protocol. -- As host name insert the External IP of your VM (click in your internet browser on your GCP VM instance to see the external IP) +- As host name insert the External IP of your VM (click in your internet browser on your GCP VM instance to see the external IP) - Set the User name in WinSCP to the name you see in your PuTTy window (check step 3 - for instance [username]@[VM-name]:~$) -- Click on the advanced setting. SSH -> Authentication. -- Option 1. Click on the Tools button and "Install Public Key into Server..". Somewhere in your folder structure must be a public key. I found it with the following folder syntax on my local windows computer -> :\Users\...\.ssh (there should be a PKK file). -- Option 2. Click on the Tools button and "Generate new key pair...". Save the private key at a folder you remember and add it to the "private key file" field in WinSCP. Upload the public key to the metadeta of your instance. +- Click on the advanced setting. SSH -> Authentication. +- Option 1. Click on the Tools button and "Install Public Key into Server..". Somewhere in your folder structure must be a public key. I found it with the following folder syntax on my local windows computer -> :\Users\...\.ssh (there should be a PKK file). +- Option 2. Click on the Tools button and "Generate new key pair...". Save the private key at a folder you remember and add it to the "private key file" field in WinSCP. Upload the public key to the metadeta of your instance. - Click ok and save. Then click Login. If successfull WinSCP will open on the left side your local computer folder structure and on the right side the folder strucutre of your VM. (If you followed Option 2 and its not initially working. Stop your instance, refresh the website, reopen the WinSCP field. Afterwards your your Login should be successfull) If you had struggle with the above steps, you could also try `this video `_. .. note:: Double check the External IP of your VM before you try to login with WinSCP. It's often a cause for an error. - + Step 5 - Extra. Copying your instance with all its data and paths included -------------------------------------------------------------------------- Especially if you think about operating several instance for quicker simulations, you can create a so called `"image" `_ of the virtual machine. -The "image" includes all the data and software set-ups from your VM. Afterwards you can create a VM from an image and avoid all the installation steps above. +The "image" includes all the data and software set-ups from your VM. Afterwards you can create a VM from an image and avoid all the installation steps above. Important points when to solve networks in PyPSA ------------------------------------------------ @@ -126,4 +126,4 @@ If you use the GCP with the default PyPSA-Eur settings, your budget will be used - Always test using low resolution networks; i.e. a single country at 5 nodes and 24h resolution for 2 month of weather data. - Adjust your solver in the ``config.yaml`` file. Set ``solving: skip_iterations: true``. - This will lead to a single solver iteration which is often precise enough. + This will lead to a single solver iteration which is often precise enough. diff --git a/doc/configtables/atlite.csv b/doc/configtables/atlite.csv index b60b15e5..5f21bb05 100644 --- a/doc/configtables/atlite.csv +++ b/doc/configtables/atlite.csv @@ -5,4 +5,4 @@ cutouts,,, -- -- module,--,"One of {'era5','sarah'}","Source of the reanalysis weather dataset (e.g. `ERA5 `_ or `SARAH-2 `_)" -- -- xs,°,"Float interval within [-180, 180]","Range of longitudes to download weather data for." -- -- ys,°,"Float interval within [-90, 90]","Range of latitudes to download weather data for." --- -- years,--,"Integer interval within [1979,2018]","Range of years to download weather data for." \ No newline at end of file +-- -- years,--,"Integer interval within [1979,2018]","Range of years to download weather data for." diff --git a/doc/configtables/costs.csv b/doc/configtables/costs.csv index 383a6423..ed2d56e4 100644 --- a/doc/configtables/costs.csv +++ b/doc/configtables/costs.csv @@ -5,4 +5,4 @@ USD2013_to_EUR2013,--,float,"Exchange rate from USD :math:`_{2013}` to EUR :math capital_cost,EUR/MW,"Keys should be in the 'technology' column of ``data/costs.csv``. Values can be any float.","For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``data/costs.csv``." marginal_cost,EUR/MWh,"Keys should be in the 'technology' column of ``data/costs.csv``. Values can be any float.","For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``data/costs.csv``." emission_prices,,,"Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs." --- co2,EUR/t,float,"Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``." \ No newline at end of file +-- co2,EUR/t,float,"Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``." diff --git a/doc/configtables/hydro.csv b/doc/configtables/hydro.csv index 801b27ff..fc53334e 100644 --- a/doc/configtables/hydro.csv +++ b/doc/configtables/hydro.csv @@ -3,4 +3,4 @@ cutout,--,"Must be 'europe-2013-era5'","Specifies the directory where the releva carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydro power plants to build per-unit availability time series for. 'ror' stands for run-of-river plants, 'PHS' represents pumped-hydro storage, and 'hydro' stands for hydroelectric dams." PHS_max_hours,h,float,"Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_." hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}","Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation `_." -clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero." \ No newline at end of file +clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero." diff --git a/doc/configtables/lines.csv b/doc/configtables/lines.csv index e5067867..59cc4fa3 100644 --- a/doc/configtables/lines.csv +++ b/doc/configtables/lines.csv @@ -2,4 +2,4 @@ types,--,"Values should specify a `line type in PyPSA `_. Keys should specify the corresponding voltage level (e.g. 220., 300. and 380. kV)","Specifies line types to assume for the different voltage levels of the ENTSO-E grid extraction. Should normally handle voltage levels 220, 300, and 380 kV" s_max_pu,--,"Value in [0.,1.]","Correction factor for line capacities (``s_nom``) to approximate :math:`N-1` security and reserve capacity for reactive power flows" length_factor,--,float,"Correction factor to account for the fact that buses are *not* connected by lines through air-line distance." -under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}","Specifies how to handle lines which are currently under construction." \ No newline at end of file +under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}","Specifies how to handle lines which are currently under construction." diff --git a/doc/configtables/links.csv b/doc/configtables/links.csv index d77c9ddd..96f6c2f5 100644 --- a/doc/configtables/links.csv +++ b/doc/configtables/links.csv @@ -1,4 +1,4 @@ ,Unit,Values,Description p_max_pu,--,"Value in [0.,1.]","Correction factor for link capacities ``p_nom``." include_tyndp,bool,"{'true', 'false'}","Specifies whether to add HVDC link projects from the `TYNDP 2018 `_ which are at least in permitting." -under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}","Specifies how to handle lines which are currently under construction." \ No newline at end of file +under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}","Specifies how to handle lines which are currently under construction." diff --git a/doc/configtables/load.csv b/doc/configtables/load.csv index 035b27a1..760f912c 100644 --- a/doc/configtables/load.csv +++ b/doc/configtables/load.csv @@ -1,2 +1,2 @@ ,Unit,Values,Description -scaling_factor,--,float,"Global correction factor for the load time series." \ No newline at end of file +scaling_factor,--,float,"Global correction factor for the load time series." diff --git a/doc/configtables/offwind-ac.csv b/doc/configtables/offwind-ac.csv index 02544506..e5bbc847 100644 --- a/doc/configtables/offwind-ac.csv +++ b/doc/configtables/offwind-ac.csv @@ -10,4 +10,4 @@ max_depth,m,float,"Maximum sea water depth at which wind turbines can be build. min_shore_distance,m,float,"Minimum distance to the shore below which wind turbines cannot be build. Such areas close to the shore are excluded in the process of calculating the AC-connected offshore wind potential." potential,--,"One of {'simple', 'conservative'}","Method to compute the maximal installable potential for a node; confer :ref:`renewableprofiles`" clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero." -keep_all_available_areas,bool,"{'true', 'false'}","Use all availabe weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed." \ No newline at end of file +keep_all_available_areas,bool,"{'true', 'false'}","Use all availabe weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed." diff --git a/doc/configtables/offwind-dc.csv b/doc/configtables/offwind-dc.csv index 02544506..e5bbc847 100644 --- a/doc/configtables/offwind-dc.csv +++ b/doc/configtables/offwind-dc.csv @@ -10,4 +10,4 @@ max_depth,m,float,"Maximum sea water depth at which wind turbines can be build. min_shore_distance,m,float,"Minimum distance to the shore below which wind turbines cannot be build. Such areas close to the shore are excluded in the process of calculating the AC-connected offshore wind potential." potential,--,"One of {'simple', 'conservative'}","Method to compute the maximal installable potential for a node; confer :ref:`renewableprofiles`" clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero." -keep_all_available_areas,bool,"{'true', 'false'}","Use all availabe weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed." \ No newline at end of file +keep_all_available_areas,bool,"{'true', 'false'}","Use all availabe weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed." diff --git a/doc/configtables/onwind.csv b/doc/configtables/onwind.csv index 11f05cfa..149dc0c4 100644 --- a/doc/configtables/onwind.csv +++ b/doc/configtables/onwind.csv @@ -11,4 +11,4 @@ corine,,, natura,bool,"{true, false}","Switch to exclude `Natura 2000 `_ natural protection areas. Area is excluded if ``true``." potential,--,"One of {'simple', 'conservative'}","Method to compute the maximal installable potential for a node; confer :ref:`renewableprofiles`" clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero." -keep_all_available_areas,bool,"{'true', 'false'}","Use all availabe weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed." \ No newline at end of file +keep_all_available_areas,bool,"{'true', 'false'}","Use all availabe weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed." diff --git a/doc/configtables/opts.csv b/doc/configtables/opts.csv index 4d699034..427951e5 100644 --- a/doc/configtables/opts.csv +++ b/doc/configtables/opts.csv @@ -5,4 +5,4 @@ Trigger, Description, Definition, Status ``CCL``, Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use ``BAU``, Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() `_, Untested ``SAFE``, Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() `_, Untested -``carrier+factor``, "Alter the capital cost of a carrier by a factor. Example: ``solar+0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use \ No newline at end of file +``carrier+factor``, "Alter the capital cost of a carrier by a factor. Example: ``solar+0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use diff --git a/doc/configtables/plotting.csv b/doc/configtables/plotting.csv index 754f2e83..f5910e16 100644 --- a/doc/configtables/plotting.csv +++ b/doc/configtables/plotting.csv @@ -11,4 +11,4 @@ energy_max,TWh,float,"Upper y-axis limit in energy bar plots." energy_min,TWh,float,"Lower y-axis limit in energy bar plots." energy_threshold,TWh,float,"Threshold below which technologies will not be shown in energy bar plots." tech_colors,--,"carrier -> HEX colour code","Mapping from network ``carrier`` to a colour (`HEX colour code `_)." -nice_names,--,"str -> str","Mapping from network ``carrier`` to a more readable name." \ No newline at end of file +nice_names,--,"str -> str","Mapping from network ``carrier`` to a more readable name." diff --git a/doc/configtables/scenario.csv b/doc/configtables/scenario.csv index 7a2d3bcd..a82462e4 100644 --- a/doc/configtables/scenario.csv +++ b/doc/configtables/scenario.csv @@ -2,4 +2,4 @@ simpl,--,cf. :ref:`simpl`,"List of ``{simpl}`` wildcards to run." clusters,--,cf. :ref:`clusters`,"List of ``{clusters}`` wildcards to run." ll,--,cf. :ref:`ll`,"List of ``{ll}`` wildcards to run." -opts,--,cf. :ref:`opts`,"List of ``{opts}`` wildcards to run." \ No newline at end of file +opts,--,cf. :ref:`opts`,"List of ``{opts}`` wildcards to run." diff --git a/doc/configtables/snapshots.csv b/doc/configtables/snapshots.csv index 14fd8001..4d917f4d 100644 --- a/doc/configtables/snapshots.csv +++ b/doc/configtables/snapshots.csv @@ -1,4 +1,4 @@ ,Unit,Values,Description start,--,"str or datetime-like; e.g. YYYY-MM-DD","Left bound of date range" end,--,"str or datetime-like; e.g. YYYY-MM-DD","Right bound of date range" -closed,--,"One of {None, ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``None``." \ No newline at end of file +closed,--,"One of {None, ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``None``." diff --git a/doc/configtables/solar.csv b/doc/configtables/solar.csv index 93bfd04a..8e57e066 100644 --- a/doc/configtables/solar.csv +++ b/doc/configtables/solar.csv @@ -12,4 +12,4 @@ corine,--,"Any subset of the `CORINE Land Cover code list `_ natural protection areas. Area is excluded if ``true``." potential,--,"One of {'simple', 'conservative'}","Method to compute the maximal installable potential for a node; confer :ref:`renewableprofiles`" clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero." -keep_all_available_areas,bool,"{'true', 'false'}","Use all availabe weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed." \ No newline at end of file +keep_all_available_areas,bool,"{'true', 'false'}","Use all availabe weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed." diff --git a/doc/configtables/solving-options.csv b/doc/configtables/solving-options.csv index 0c640684..f77eec24 100644 --- a/doc/configtables/solving-options.csv +++ b/doc/configtables/solving-options.csv @@ -7,4 +7,4 @@ max_iterations,--,int,"Maximum number of solving iterations in between which res nhours,--,int,"Specifies the :math:`n` first snapshots to take into account. Must be less than the total number of snapshots. Rather recommended only for debugging." clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero." skip_iterations,bool,"{'true','false'}","Skip iterating, do not update impedances of branches." -track_iterations,bool,"{'true','false'}","Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration)" \ No newline at end of file +track_iterations,bool,"{'true','false'}","Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration)" diff --git a/doc/configtables/solving-solver.csv b/doc/configtables/solving-solver.csv index db16d867..3eae6310 100644 --- a/doc/configtables/solving-solver.csv +++ b/doc/configtables/solving-solver.csv @@ -1,3 +1,3 @@ ,Unit,Values,Description name,--,"One of {'gurobi', 'cplex', 'cbc', 'glpk', 'ipopt'}; potentially more possible","Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow." -opts,--,"Parameter list for `Gurobi `_ and `CPLEX `_","Solver specific parameter settings." \ No newline at end of file +opts,--,"Parameter list for `Gurobi `_ and `CPLEX `_","Solver specific parameter settings." diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv index 7eb7dd61..27df2344 100644 --- a/doc/configtables/toplevel.csv +++ b/doc/configtables/toplevel.csv @@ -13,4 +13,4 @@ enable,,, -- build_cutout,bool,"{true, false}","Switch to enable the building of cutouts via the rule :mod:`build_cutout`." -- retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts from zenodo with :mod:`retrieve_cutout`." -- build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`." --- retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`." \ No newline at end of file +-- retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`." diff --git a/doc/configtables/transformers.csv b/doc/configtables/transformers.csv index b58ae8f3..0ccd8e87 100644 --- a/doc/configtables/transformers.csv +++ b/doc/configtables/transformers.csv @@ -1,4 +1,4 @@ ,Unit,Values,Description x,p.u.,float,"Series reactance (per unit, using ``s_nom`` as base power of the transformer. Overwritten if ``type`` is specified." s_nom,MVA,float,"Limit of apparent power which can pass through branch. Overwritten if ``type`` is specified." -type,--,"A `transformer type in PyPSA `_.","Specifies transformer types to assume for the transformers of the ENTSO-E grid extraction." \ No newline at end of file +type,--,"A `transformer type in PyPSA `_.","Specifies transformer types to assume for the transformers of the ENTSO-E grid extraction." diff --git a/doc/configuration.rst b/doc/configuration.rst index 265943e8..fa51b51a 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -36,10 +36,10 @@ investment changes as more ambitious greenhouse-gas emission reduction targets a The ``scenario`` section is an extraordinary section of the config file that is strongly connected to the :ref:`wildcards` and is designed to -facilitate running multiple scenarios through a single command +facilitate running multiple scenarios through a single command .. code:: bash - + snakemake -j 1 solve_all_elec_networks For each wildcard, a **list of values** is provided. The rule ``solve_all_elec_networks`` will trigger the rules for creating ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc`` for **all combinations** of the provided wildcard values as defined by Python's `itertools.product(...) `_ function that snakemake's `expand(...) function `_ uses. diff --git a/doc/costs.rst b/doc/costs.rst index a51acb85..70bd12b8 100644 --- a/doc/costs.rst +++ b/doc/costs.rst @@ -50,4 +50,4 @@ Default Cost Assumptions .. csv-table:: :header-rows: 1 :widths: 10,3,5,4,6,8 - :file: ../data/costs.csv \ No newline at end of file + :file: ../data/costs.csv diff --git a/doc/index.rst b/doc/index.rst index d375cfac..4c15ddf5 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -8,7 +8,7 @@ PyPSA-Eur: An Open Optimisation Model of the European Transmission System .. image:: https://img.shields.io/github/v/release/pypsa/pypsa-eur?include_prereleases :alt: GitHub release (latest by date including pre-releases) - + .. image:: https://travis-ci.org/PyPSA/pypsa-eur.svg?branch=master :target: https://travis-ci.org/PyPSA/pypsa-eur @@ -101,9 +101,9 @@ Documentation simplification solving plotting - + **References** - + * :doc:`release_notes` * :doc:`limitations` * :doc:`contributing` @@ -183,7 +183,7 @@ There are pre-built networks available as a dataset on Zenodo as well for every The included ``.nc`` files are PyPSA network files which can be imported with PyPSA via: .. code:: python - + import pypsa filename = "elec_s_1024_ec.nc" # example @@ -202,7 +202,7 @@ PyPSA-Eur work is released under multiple licenses: See the individual files and the `dep5 <.reuse/dep5>`_ file for license details. Additionally, different licenses and terms of use also apply to the various input data, which are summarised below. -More details are included in +More details are included in `the description of the data bundles on zenodo `_. .. csv-table:: diff --git a/doc/installation.rst b/doc/installation.rst index f50083d3..e1de3f4b 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -1,6 +1,6 @@ .. SPDX-FileCopyrightText: 2019-2020 The PyPSA-Eur Authors - + SPDX-License-Identifier: CC-BY-4.0 .. _installation: @@ -37,12 +37,12 @@ We recommend using the package manager and environment management system ``conda Install `miniconda `_, which is a mini version of `Anaconda `_ that includes only ``conda`` and its dependencies or make sure ``conda`` is already installed on your system. For instructions for your operating system follow the ``conda`` `installation guide `_. -The python package requirements are curated in the `environment.yaml `_ file. +The python package requirements are curated in the `envs/environment.yaml `_ file. The environment can be installed and activated using .. code:: bash - .../pypsa-eur % conda env create -f environment.yaml + .../pypsa-eur % conda env create -f envs/environment.yaml .../pypsa-eur % conda activate pypsa-eur @@ -55,14 +55,14 @@ The environment can be installed and activated using `mamba `_ as a fast drop-in replacement via .. code:: bash - + conda install -c conda-forge mamba and then install the environment with .. code:: bash - mamba env create -f environment.yaml + mamba env create -f envs/environment.yaml Install a Solver ================ @@ -102,10 +102,10 @@ For installation instructions of these solvers for your operating system, follow and on Windows .. code:: bash - + conda activate pypsa-eur conda install -c conda-forge ipopt glpk - + .. _defaultconfig: @@ -113,7 +113,7 @@ Set Up the Default Configuration ================================ PyPSA-Eur has several configuration options that must be specified in a ``config.yaml`` file located in the root directory. -An example configuration ``config.default.yaml`` is maintained in the repository. +An example configuration ``config.default.yaml`` is maintained in the repository. More details on the configuration options are in :ref:`config`. Before first use, create a ``config.yaml`` by copying the example. diff --git a/doc/introduction.rst b/doc/introduction.rst index 96094a9f..9ca049e7 100644 --- a/doc/introduction.rst +++ b/doc/introduction.rst @@ -1,6 +1,6 @@ .. SPDX-FileCopyrightText: 2019-2020 The PyPSA-Eur Authors - + SPDX-License-Identifier: CC-BY-4.0 .. _intro: @@ -64,4 +64,4 @@ Folder Structure System Requirements =================== -Building the model with the scripts in this repository uses up to 20 GB of memory. Computing optimal investment and operation scenarios requires a strong interior-point solver compatible with the modelling library `Pyomo `_ like `Gurobi `_ or `CPLEX `_ with up to 100 GB of memory. \ No newline at end of file +Building the model with the scripts in this repository uses up to 20 GB of memory. Computing optimal investment and operation scenarios requires a strong interior-point solver compatible with the modelling library `Pyomo `_ like `Gurobi `_ or `CPLEX `_ with up to 100 GB of memory. diff --git a/doc/limitations.rst b/doc/limitations.rst index 16c57ac9..3b6c03d7 100644 --- a/doc/limitations.rst +++ b/doc/limitations.rst @@ -37,7 +37,7 @@ improving the approximations. not spatially disaggregated; assuming, as we have done, that the load time series shape is the same at each node within each country ignores local differences. -- **Currently installed renewable capacities:** +- **Currently installed renewable capacities:** Information on existing wind, solar and small hydro, geothermal, marine and biomass power plants are excluded from the dataset because of a lack of data availability in many countries. Approximate distributions of wind and solar @@ -56,4 +56,3 @@ improving the approximations. Belarus, Ukraine, Turkey and Morocco have not been taken into account; islands which are not connected to the main European system, such as Malta, Crete and Cyprus, are also excluded from the model. - \ No newline at end of file diff --git a/doc/plotting.rst b/doc/plotting.rst index 641d3962..cd404226 100644 --- a/doc/plotting.rst +++ b/doc/plotting.rst @@ -173,4 +173,4 @@ Rule ``plot_network`` .. automodule:: plot_network .. image:: img/tech-colors.png - :align: center \ No newline at end of file + :align: center diff --git a/doc/preparation/base_network.rst b/doc/preparation/base_network.rst index 464b003d..1afc4e05 100644 --- a/doc/preparation/base_network.rst +++ b/doc/preparation/base_network.rst @@ -2,7 +2,7 @@ SPDX-FileCopyrightText: 2019-2020 The PyPSA-Eur Authors SPDX-License-Identifier: CC-BY-4.0 - + .. _base: Rule ``base_network`` @@ -51,4 +51,4 @@ Rule ``base_network`` | -.. automodule:: base_network \ No newline at end of file +.. automodule:: base_network diff --git a/doc/preparation/build_bus_regions.rst b/doc/preparation/build_bus_regions.rst index 6935832e..16aab725 100644 --- a/doc/preparation/build_bus_regions.rst +++ b/doc/preparation/build_bus_regions.rst @@ -48,4 +48,4 @@ Rule ``build_bus_regions`` | -.. automodule:: build_bus_regions \ No newline at end of file +.. automodule:: build_bus_regions diff --git a/doc/preparation/build_cutout.rst b/doc/preparation/build_cutout.rst index 5ad29a97..da2c04d1 100644 --- a/doc/preparation/build_cutout.rst +++ b/doc/preparation/build_cutout.rst @@ -39,4 +39,4 @@ Rule ``build_cutout`` | -.. automodule:: build_cutout \ No newline at end of file +.. automodule:: build_cutout diff --git a/doc/release_notes.rst b/doc/release_notes.rst index f5e3c377..1cae0807 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -23,7 +23,7 @@ Upcoming Release * Corrected setting of exogenous emission price (in config -> cost -> emission price). This was not weighted by the efficiency and effective emission of the generators. Fixed in `#171 `_. -* Don't remove capital costs from lines and links, when imposing a line volume limit (wildcard ``lv``) or a line cost limit (``lc``). Previously, these were removed to move the expansion in direction of the limit. +* Don't remove capital costs from lines and links, when imposing a line volume limit (wildcard ``lv``) or a line cost limit (``lc``). Previously, these were removed to move the expansion in direction of the limit. PyPSA-Eur 0.2.0 (8th June 2020) ================================== @@ -86,7 +86,7 @@ This is the first release of PyPSA-Eur, a model of the European power system at * Hydrogen pipelines (including cost assumptions) can now be added alongside clustered network connections in the rule :mod:`add_extra_components` . Set ``electricity: extendable_carriers: Link: [H2 pipeline]`` and ensure hydrogen storage is modelled as a ``Store``. This is a first simplified stage (`#108 `_). -* Logfiles for all rules of the ``snakemake`` workflow are now written in the folder ``log/`` (`#102 `_). +* Logfiles for all rules of the ``snakemake`` workflow are now written in the folder ``log/`` (`#102 `_). * The new function ``_helpers.mock_snakemake`` creates a ``snakemake`` object which mimics the actual ``snakemake`` object produced by workflow by parsing the ``Snakefile`` and setting all paths for inputs, outputs, and logs. This allows running all scripts within a (I)python terminal (or just by calling ``python ``) and thereby facilitates developing and debugging scripts significantly (`#107 `_). @@ -97,8 +97,8 @@ Release Process * Finalise release notes at ``doc/release_notes.rst``. -* Update ``environment.fixedversions.yaml`` via - ``conda env export -n pypsa-eur -f environment.fixedversions.yaml --no-builds`` +* Update ``envs/environment.fixed.yaml`` via + ``conda env export -n pypsa-eur -f envs/environment.fixed.yaml --no-builds`` from an up-to-date `pypsa-eur` environment. * Update version number in ``doc/conf.py`` and ``*config.*.yaml``. diff --git a/doc/simplification.rst b/doc/simplification.rst index 11d4867f..280b1da0 100644 --- a/doc/simplification.rst +++ b/doc/simplification.rst @@ -22,7 +22,7 @@ After simplification and clustering of the network, additional components may be .. toctree:: :caption: Overview - + simplification/simplify_network simplification/cluster_network simplification/add_extra_components diff --git a/doc/tutorial.rst b/doc/tutorial.rst index eca7dd05..2e96eb5c 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -1,6 +1,6 @@ .. SPDX-FileCopyrightText: 2019-2020 The PyPSA-Eur Authors - + SPDX-License-Identifier: CC-BY-4.0 .. _tutorial: @@ -48,7 +48,7 @@ The model can be adapted to only include selected countries (e.g. Germany) inste .. literalinclude:: ../config.tutorial.yaml :language: yaml :lines: 16 - + Likewise, the example's temporal scope can be restricted (e.g. to a single month). .. literalinclude:: ../config.tutorial.yaml @@ -132,7 +132,7 @@ orders ``snakemake`` to run the script ``solve_network`` that produces the solve Windows users should add the option ``--keep-target-files`` to the command or instead run ``snakemake -j 1 solve_all_elec_networks``. This triggers a workflow of multiple preceding jobs that depend on each rule's inputs and outputs: - + .. graphviz:: :align: center @@ -184,7 +184,7 @@ This triggers a workflow of multiple preceding jobs that depend on each rule's i 7 -> 11 5 -> 11 12 -> 11 - } + } | @@ -229,8 +229,8 @@ A job (here ``simplify_network``) will display its attributes and normally some INFO:__main__:Mapping all network lines onto a single 380kV layer INFO:__main__:Simplifying connected link components INFO:__main__:Removing stubs - INFO:__main__:Displacing offwind-ac generator(s) and adding connection costs to capital_costs: 20128 Eur/MW/a for `5718 offwind-ac` - INFO:__main__:Displacing offwind-dc generator(s) and adding connection costs to capital_costs: 14994 Eur/MW/a for `5718 offwind-dc`, 26939 Eur/MW/a for `5724 offwind-dc`, 29621 Eur/MW/a for `5725 offwind-dc` + INFO:__main__:Displacing offwind-ac generator(s) and adding connection costs to capital_costs: 20128 Eur/MW/a for `5718 offwind-ac` + INFO:__main__:Displacing offwind-dc generator(s) and adding connection costs to capital_costs: 14994 Eur/MW/a for `5718 offwind-dc`, 26939 Eur/MW/a for `5724 offwind-dc`, 29621 Eur/MW/a for `5725 offwind-dc` INFO:pypsa.io:Exported network elec_s.nc has lines, carriers, links, storage_units, loads, buses, generators [] Finished job 3. @@ -293,5 +293,5 @@ For inspiration, read the `examples section in the PyPSA documentation =0.17.1 - atlite=0.0.3 # Dependencies of the workflow itself - #- xlrd - scikit-learn - pycountry - seaborn - #- snakemake-minimal - memory_profiler - yaml - pytables @@ -25,28 +21,19 @@ dependencies: # Second order dependencies which should really be deps of atlite - xarray - #- netcdf4 - #- bottleneck - #- toolz - #- dask - progressbar2 - pyyaml>=5.1.0 - # Include ipython so that one does not inadvertently drop out of the conda - # environment by calling ipython - # - ipython - # GIS dependencies have to come all from conda-forge - - conda-forge::cartopy - - conda-forge::fiona - - conda-forge::proj - - conda-forge::pyshp - - conda-forge::geopandas - - conda-forge::rasterio - - conda-forge::shapely - - conda-forge::libgdal + - cartopy + - fiona + - proj + - pyshp + - geopandas + - rasterio + - shapely + - libgdal - # The FRESNA/KIT stuff is not packaged for conda yet - pip: - vresutils==0.3.1 - git+https://github.com/PyPSA/glaes.git#egg=glaes diff --git a/environment.fixedversions.yaml b/envs/environment.fixed.yaml similarity index 100% rename from environment.fixedversions.yaml rename to envs/environment.fixed.yaml diff --git a/environment.yaml b/envs/environment.yaml similarity index 100% rename from environment.yaml rename to envs/environment.yaml diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 14667679..c11c27f5 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -373,7 +373,7 @@ def attach_hydro(n, costs, ppl): .where(lambda df: df<=1., other=1.))) if 'PHS' in carriers and not phs.empty: - # fill missing max hours to config value and + # fill missing max hours to config value and # assume no natural inflow due to lack of data phs = phs.replace({'max_hours': {0: c['PHS_max_hours']}}) n.madd('StorageUnit', phs.index, diff --git a/scripts/base_network.py b/scripts/base_network.py index 034eeb2f..d2ceccf1 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -312,18 +312,18 @@ def _set_electrical_parameters_links(links): links['p_min_pu'] = -p_max_pu links_p_nom = pd.read_csv(snakemake.input.links_p_nom) - - # filter links that are not in operation anymore + + # filter links that are not in operation anymore removed_b = links_p_nom.Remarks.str.contains('Shut down|Replaced', na=False) links_p_nom = links_p_nom[~removed_b] - - # find closest link for all links in links_p_nom + + # find closest link for all links in links_p_nom links_p_nom['j'] = _find_closest_links(links, links_p_nom) - - links_p_nom = links_p_nom.groupby(['j'],as_index=False).agg({'Power (MW)': 'sum'}) - + + links_p_nom = links_p_nom.groupby(['j'],as_index=False).agg({'Power (MW)': 'sum'}) + p_nom = links_p_nom.dropna(subset=["j"]).set_index("j")["Power (MW)"] - + # Don't update p_nom if it's already set p_nom_unset = p_nom.drop(links.index[links.p_nom.notnull()], errors='ignore') if "p_nom" in links else p_nom links.loc[p_nom_unset.index, "p_nom"] = p_nom_unset diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 2c384926..fc054a10 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -115,7 +115,7 @@ Exemplary unsolved network clustered to 37 nodes: .. image:: ../img/elec_s_37.png :scale: 40 % - :align: center + :align: center """ diff --git a/scripts/make_summary.py b/scripts/make_summary.py index ac0119ab..ada3fa8a 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -307,7 +307,7 @@ def calculate_prices(n,label,prices): def calculate_weighted_prices(n,label,weighted_prices): - + logger.warning("Weighted prices don't include storage units as loads") weighted_prices = weighted_prices.reindex(pd.Index(["electricity","heat","space heat","urban heat","space urban heat","gas","H2"])) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index bb7640be..f790db8f 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -131,7 +131,7 @@ def set_transmission_limit(n, ll_type, factor, Nyears=1): n.add('GlobalConstraint', f'l{ll_type}_limit', type=f'transmission_{con_type}_limit', sense='<=', constant=rhs, carrier_attribute='AC, DC') - + return n diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 048b0ed3..5276adfb 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -186,4 +186,3 @@ solving: # solutiontype: 2 # non basic solution, ie no crossover # barrier_convergetol: 1.e-5 # feasopt_tolerance: 1.e-6 - From dd4621095210445b72cf1136fc88ab9d4aea4f5b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 11 Sep 2020 15:34:03 +0200 Subject: [PATCH 003/173] pre-commit: limit large files --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5d05ee9..d04178b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,4 +5,6 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml + - id: check-merge-conflict - id: check-added-large-files + args: ['--maxkb=2000'] From 570154fa78fdc77056e6bec8d8dd1a3cf365e9e2 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 11 Sep 2020 15:41:23 +0200 Subject: [PATCH 004/173] add license to .pre-commit --- .pre-commit-config.yaml | 8 ++++++++ {.licenses => LICENSES}/CC-BY-4.0.txt | 0 {.licenses => LICENSES}/CC0-1.0.txt | 0 {.licenses => LICENSES}/GPL-3.0-or-later.txt | 0 4 files changed, 8 insertions(+) rename {.licenses => LICENSES}/CC-BY-4.0.txt (100%) rename {.licenses => LICENSES}/CC0-1.0.txt (100%) rename {.licenses => LICENSES}/GPL-3.0-or-later.txt (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d04178b6..f590f447 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: : 2017-2020 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.4.0 @@ -8,3 +12,7 @@ repos: - id: check-merge-conflict - id: check-added-large-files args: ['--maxkb=2000'] +- repo: https://github.com/fsfe/reuse-tool + rev: latest + hooks: + - id: reuse diff --git a/.licenses/CC-BY-4.0.txt b/LICENSES/CC-BY-4.0.txt similarity index 100% rename from .licenses/CC-BY-4.0.txt rename to LICENSES/CC-BY-4.0.txt diff --git a/.licenses/CC0-1.0.txt b/LICENSES/CC0-1.0.txt similarity index 100% rename from .licenses/CC0-1.0.txt rename to LICENSES/CC0-1.0.txt diff --git a/.licenses/GPL-3.0-or-later.txt b/LICENSES/GPL-3.0-or-later.txt similarity index 100% rename from .licenses/GPL-3.0-or-later.txt rename to LICENSES/GPL-3.0-or-later.txt From 098448668b650b922fc8efa268f2dd34a2efb3bb Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 11 Sep 2020 15:43:14 +0200 Subject: [PATCH 005/173] add pre-commit to env --- envs/environment.docs.yaml | 1 + envs/environment.yaml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/envs/environment.docs.yaml b/envs/environment.docs.yaml index 11597d92..fc141a02 100755 --- a/envs/environment.docs.yaml +++ b/envs/environment.docs.yaml @@ -9,6 +9,7 @@ dependencies: - pip - pypsa>=0.17.1 - atlite=0.0.3 + - pre-commit # Dependencies of the workflow itself - scikit-learn diff --git a/envs/environment.yaml b/envs/environment.yaml index ce49420f..fb4019a3 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -11,6 +11,7 @@ channels: dependencies: - python - pip + - pre-commit - pypsa>=0.17.1 - atlite=0.0.3 @@ -36,8 +37,7 @@ dependencies: - progressbar2 - pyyaml>=5.1.0 - # Include ipython so that one does not inadvertently drop out of the conda - # environment by calling ipython + # Keep in conda environment when calling ipython - ipython # GIS dependencies: From 78ca48c602a10470a2220eda7aac0f7eed628e51 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 11 Sep 2020 15:44:25 +0200 Subject: [PATCH 006/173] solve: tidy memory logger --- scripts/solve_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 5ec515c3..ace8575f 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -254,8 +254,8 @@ if __name__ == "__main__": opts = snakemake.wildcards.opts.split('-') solve_opts = snakemake.config['solving']['options'] - with memory_logger(filename=getattr(snakemake.log, 'memory', None), - interval=30.) as mem: + fn = getattr(snakemake.log, 'memory', None) + with memory_logger(filename=fn, interval=30.) as mem: n = pypsa.Network(snakemake.input[0]) n = prepare_network(n, solve_opts) n = solve_network(n, config=snakemake.config, solver_dir=tmpdir, From 1e4f6f9568b1049c4da45d70356dd446a45723c0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 25 Sep 2020 12:45:43 +0200 Subject: [PATCH 007/173] travis: add conda list for easier debugging --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 75c5d524..011753f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,9 @@ before_install: # install open-source solver - mamba install -c conda-forge ipopt glpk + # list packages for easier debugging + - conda list + script: - cp ./test/config.test1.yaml ./config.yaml - snakemake -j all solve_all_elec_networks From f3b1d3d5133e1d9a635d72c6d35cafceaddae74b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 3 Oct 2020 11:38:51 +0200 Subject: [PATCH 008/173] undo config test/tutorial without plotting, rm matplotlibrc, .licenses --- config.tutorial.yaml | 80 ++++++++++++++++++++++++++++++++++++++++++ doc/index.rst | 8 ++--- matplotlibrc | 1 + test/config.test1.yaml | 80 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 matplotlibrc diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 3f4b6853..174708cb 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -172,3 +172,83 @@ solving: track_iterations: false solver: name: cbc + +plotting: + map: + figsize: [7, 7] + boundaries: [-10.2, 29, 35, 72] + p_nom: + bus_size_factor: 5.e+4 + linewidth_factor: 3.e+3 + + costs_max: 800 + costs_threshold: 1 + + energy_max: 15000. + energy_min: -10000. + energy_threshold: 50. + + vre_techs: ["onwind", "offwind-ac", "offwind-dc", "solar", "ror"] + conv_techs: ["OCGT", "CCGT", "Nuclear", "Coal"] + storage_techs: ["hydro+PHS", "battery", "H2"] + load_carriers: ["AC load"] + AC_carriers: ["AC line", "AC transformer"] + link_carriers: ["DC line", "Converter AC-DC"] + tech_colors: + "onwind" : "#235ebc" + "onshore wind" : "#235ebc" + 'offwind' : "#6895dd" + 'offwind-ac' : "#6895dd" + 'offshore wind' : "#6895dd" + 'offshore wind ac' : "#6895dd" + 'offwind-dc' : "#74c6f2" + 'offshore wind dc' : "#74c6f2" + "hydro" : "#08ad97" + "hydro+PHS" : "#08ad97" + "PHS" : "#08ad97" + "hydro reservoir" : "#08ad97" + 'hydroelectricity' : '#08ad97' + "ror" : "#4adbc8" + "run of river" : "#4adbc8" + 'solar' : "#f9d002" + 'solar PV' : "#f9d002" + 'solar thermal' : '#ffef60' + 'biomass' : '#0c6013' + 'solid biomass' : '#06540d' + 'biogas' : '#23932d' + 'waste' : '#68896b' + 'geothermal' : '#ba91b1' + "OCGT" : "#d35050" + "gas" : "#d35050" + "natural gas" : "#d35050" + "CCGT" : "#b20101" + "nuclear" : "#ff9000" + "coal" : "#707070" + "lignite" : "#9e5a01" + "oil" : "#262626" + "H2" : "#ea048a" + "hydrogen storage" : "#ea048a" + "battery" : "#b8ea04" + "Electric load" : "#f9d002" + "electricity" : "#f9d002" + "lines" : "#70af1d" + "transmission lines" : "#70af1d" + "AC-AC" : "#70af1d" + "AC line" : "#70af1d" + "links" : "#8a1caf" + "HVDC links" : "#8a1caf" + "DC-DC" : "#8a1caf" + "DC link" : "#8a1caf" + nice_names: + OCGT: "Open-Cycle Gas" + CCGT: "Combined-Cycle Gas" + offwind-ac: "Offshore Wind (AC)" + offwind-dc: "Offshore Wind (DC)" + onwind: "Onshore Wind" + solar: "Solar" + PHS: "Pumped Hydro Storage" + hydro: "Reservoir & Dam" + battery: "Battery Storage" + H2: "Hydrogen Storage" + lines: "Transmission Lines" + ror: "Run of River" diff --git a/doc/index.rst b/doc/index.rst index 4c15ddf5..d8d9c852 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -194,10 +194,10 @@ Licence PyPSA-Eur work is released under multiple licenses: -* All original source code is licensed as free software under `GPL-3.0-or-later <.licenses/GPL-3.0-or-later.txt>`_. -* The documentation is licensed under `CC-BY-4.0 <.licenses/CC-BY-4.0.txt>`_. -* Configuration files are mostly licensed under `CC0-1.0 <.licenses/CC0-1.0.txt>`_. -* Data files are licensed under `CC-BY-4.0 <.licenses/CC-BY-4.0.txt>`_. +* All original source code is licensed as free software under `GPL-3.0-or-later `_. +* The documentation is licensed under `CC-BY-4.0 `_. +* Configuration files are mostly licensed under `CC0-1.0 `_. +* Data files are licensed under `CC-BY-4.0 `_. See the individual files and the `dep5 <.reuse/dep5>`_ file for license details. diff --git a/matplotlibrc b/matplotlibrc new file mode 100644 index 00000000..13468274 --- /dev/null +++ b/matplotlibrc @@ -0,0 +1 @@ +backend : Agg diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 011169b9..84d551e6 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -188,3 +188,83 @@ solving: # solutiontype: 2 # non basic solution, ie no crossover # barrier_convergetol: 1.e-5 # feasopt_tolerance: 1.e-6 + +plotting: + map: + figsize: [7, 7] + boundaries: [-10.2, 29, 35, 72] + p_nom: + bus_size_factor: 5.e+4 + linewidth_factor: 3.e+3 + + costs_max: 800 + costs_threshold: 1 + + energy_max: 15000. + energy_min: -10000. + energy_threshold: 50. + + vre_techs: ["onwind", "offwind-ac", "offwind-dc", "solar", "ror"] + conv_techs: ["OCGT", "CCGT", "Nuclear", "Coal"] + storage_techs: ["hydro+PHS", "battery", "H2"] + load_carriers: ["AC load"] + AC_carriers: ["AC line", "AC transformer"] + link_carriers: ["DC line", "Converter AC-DC"] + tech_colors: + "onwind" : "#235ebc" + "onshore wind" : "#235ebc" + 'offwind' : "#6895dd" + 'offwind-ac' : "#6895dd" + 'offshore wind' : "#6895dd" + 'offshore wind ac' : "#6895dd" + 'offwind-dc' : "#74c6f2" + 'offshore wind dc' : "#74c6f2" + "hydro" : "#08ad97" + "hydro+PHS" : "#08ad97" + "PHS" : "#08ad97" + "hydro reservoir" : "#08ad97" + 'hydroelectricity' : '#08ad97' + "ror" : "#4adbc8" + "run of river" : "#4adbc8" + 'solar' : "#f9d002" + 'solar PV' : "#f9d002" + 'solar thermal' : '#ffef60' + 'biomass' : '#0c6013' + 'solid biomass' : '#06540d' + 'biogas' : '#23932d' + 'waste' : '#68896b' + 'geothermal' : '#ba91b1' + "OCGT" : "#d35050" + "gas" : "#d35050" + "natural gas" : "#d35050" + "CCGT" : "#b20101" + "nuclear" : "#ff9000" + "coal" : "#707070" + "lignite" : "#9e5a01" + "oil" : "#262626" + "H2" : "#ea048a" + "hydrogen storage" : "#ea048a" + "battery" : "#b8ea04" + "Electric load" : "#f9d002" + "electricity" : "#f9d002" + "lines" : "#70af1d" + "transmission lines" : "#70af1d" + "AC-AC" : "#70af1d" + "AC line" : "#70af1d" + "links" : "#8a1caf" + "HVDC links" : "#8a1caf" + "DC-DC" : "#8a1caf" + "DC link" : "#8a1caf" + nice_names: + OCGT: "Open-Cycle Gas" + CCGT: "Combined-Cycle Gas" + offwind-ac: "Offshore Wind (AC)" + offwind-dc: "Offshore Wind (DC)" + onwind: "Onshore Wind" + solar: "Solar" + PHS: "Pumped Hydro Storage" + hydro: "Reservoir & Dam" + battery: "Battery Storage" + H2: "Hydrogen Storage" + lines: "Transmission Lines" + ror: "Run of River" From bcd7f86b5c6928ce7c03c4a9b0cad27419e76040 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 9 Oct 2020 17:03:49 +0200 Subject: [PATCH 009/173] remove {networks} wildcard --- Snakefile | 104 ++++++++++++++-------------- doc/tutorial.rst | 4 +- scripts/add_extra_components.py | 4 +- scripts/cluster_network.py | 16 ++--- scripts/prepare_network.py | 4 +- scripts/simplify_network.py | 10 +-- scripts/solve_network.py | 4 +- scripts/solve_operations_network.py | 6 +- 8 files changed, 76 insertions(+), 76 deletions(-) diff --git a/Snakefile b/Snakefile index 7a532f30..cd42835e 100644 --- a/Snakefile +++ b/Snakefile @@ -222,17 +222,17 @@ rule add_electricity: rule simplify_network: input: - network='networks/{network}.nc', + network='networks/elec.nc', tech_costs=COSTS, regions_onshore="resources/regions_onshore.geojson", regions_offshore="resources/regions_offshore.geojson" output: - network='networks/{network}_s{simpl}.nc', - regions_onshore="resources/regions_onshore_{network}_s{simpl}.geojson", - regions_offshore="resources/regions_offshore_{network}_s{simpl}.geojson", - busmap='resources/busmap_{network}_s{simpl}.csv' - log: "logs/simplify_network/{network}_s{simpl}.log" - benchmark: "benchmarks/simplify_network/{network}_s{simpl}" + network='networks/elec_s{simpl}.nc', + regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", + regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", + busmap='resources/busmap_elec_s{simpl}.csv' + log: "logs/simplify_network/elec_s{simpl}.log" + benchmark: "benchmarks/simplify_network/elec_s{simpl}" threads: 1 resources: mem=4000 script: "scripts/simplify_network.py" @@ -240,19 +240,19 @@ rule simplify_network: rule cluster_network: input: - network='networks/{network}_s{simpl}.nc', - regions_onshore="resources/regions_onshore_{network}_s{simpl}.geojson", - regions_offshore="resources/regions_offshore_{network}_s{simpl}.geojson", - busmap=ancient('resources/busmap_{network}_s{simpl}.csv'), + network='networks/elec_s{simpl}.nc', + regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", + regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", + busmap=ancient('resources/busmap_elec_s{simpl}.csv'), tech_costs=COSTS output: - network='networks/{network}_s{simpl}_{clusters}.nc', - regions_onshore="resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson", - regions_offshore="resources/regions_offshore_{network}_s{simpl}_{clusters}.geojson", - busmap="resources/busmap_{network}_s{simpl}_{clusters}.csv", - linemap="resources/linemap_{network}_s{simpl}_{clusters}.csv" - log: "logs/cluster_network/{network}_s{simpl}_{clusters}.log" - benchmark: "benchmarks/cluster_network/{network}_s{simpl}_{clusters}" + network='networks/elec_s{simpl}_{clusters}.nc', + regions_onshore="resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", + regions_offshore="resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", + busmap="resources/busmap_elec_s{simpl}_{clusters}.csv", + linemap="resources/linemap_elec_s{simpl}_{clusters}.csv" + log: "logs/cluster_network/elec_s{simpl}_{clusters}.log" + benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}" threads: 1 resources: mem=3000 script: "scripts/cluster_network.py" @@ -260,21 +260,21 @@ rule cluster_network: rule add_extra_components: input: - network='networks/{network}_s{simpl}_{clusters}.nc', + network='networks/elec_s{simpl}_{clusters}.nc', tech_costs=COSTS, - output: 'networks/{network}_s{simpl}_{clusters}_ec.nc' - log: "logs/add_extra_components/{network}_s{simpl}_{clusters}.log" - benchmark: "benchmarks/add_extra_components/{network}_s{simpl}_{clusters}_ec" + output: 'networks/elec_s{simpl}_{clusters}_ec.nc' + log: "logs/add_extra_components/elec_s{simpl}_{clusters}.log" + benchmark: "benchmarks/add_extra_components/elec_s{simpl}_{clusters}_ec" threads: 1 resources: mem=3000 script: "scripts/add_extra_components.py" rule prepare_network: - input: 'networks/{network}_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS - output: 'networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' - log: "logs/prepare_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" - benchmark: "benchmarks/prepare_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}" + input: 'networks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS + output: 'networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' + log: "logs/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" + benchmark: "benchmarks/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 1 resources: mem=1000 script: "scripts/prepare_network.py" @@ -294,13 +294,13 @@ def memory(w): rule solve_network: - input: "networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + input: "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" log: - solver=normpath("logs/solve_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), - python="logs/solve_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", - memory="logs/solve_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log" - benchmark: "benchmarks/solve_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}" + solver=normpath("logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), + python="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", + memory="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log" + benchmark: "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 resources: mem=memory shadow: "shallow" @@ -309,14 +309,14 @@ rule solve_network: rule solve_operations_network: input: - unprepared="networks/{network}_s{simpl}_{clusters}_ec.nc", - optimized="results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" + unprepared="networks/elec_s{simpl}_{clusters}_ec.nc", + optimized="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" log: - solver=normpath("logs/solve_operations_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), - python="logs/solve_operations_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", - memory="logs/solve_operations_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log" - benchmark: "benchmarks/solve_operations_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}" + solver=normpath("logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), + python="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", + memory="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log" + benchmark: "benchmarks/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 resources: mem=(lambda w: 5000 + 372 * int(w.clusters)) shadow: "shallow" @@ -325,12 +325,12 @@ rule solve_operations_network: rule plot_network: input: - network="results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", tech_costs=COSTS output: - only_map="results/plots/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", - ext="results/plots/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_ext.{ext}" - log: "logs/plot_network/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_{ext}.log" + only_map="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", + ext="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_ext.{ext}" + log: "logs/plot_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_{ext}.log" script: "scripts/plot_network.py" @@ -343,7 +343,7 @@ def input_make_summary(w): else: ll = w.ll return ([COSTS] + - expand("results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", network=w.network, ll=ll, **{k: config["scenario"][k] if getattr(w, k) == "all" else getattr(w, k) @@ -352,28 +352,28 @@ def input_make_summary(w): rule make_summary: input: input_make_summary - output: directory("results/summaries/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}") - log: "logs/make_summary/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log", + output: directory("results/summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}") + log: "logs/make_summary/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log", script: "scripts/make_summary.py" rule plot_summary: - input: "results/summaries/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}" - output: "results/plots/summary_{summary}_{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}" - log: "logs/plot_summary/{summary}_{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log" + input: "results/summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}" + output: "results/plots/summary_{summary}_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}" + log: "logs/plot_summary/{summary}_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log" script: "scripts/plot_summary.py" def input_plot_p_nom_max(w): - return [("networks/{network}_s{simpl}{maybe_cluster}.nc" + return [("networks/elec_s{simpl}{maybe_cluster}.nc" .format(maybe_cluster=('' if c == 'full' else ('_' + c)), **w)) for c in w.clusts.split(",")] rule plot_p_nom_max: input: input_plot_p_nom_max - output: "results/plots/{network}_s{simpl}_cum_p_nom_max_{clusts}_{techs}_{country}.{ext}" - log: "logs/plot_p_nom_max/{network}_s{simpl}_{clusts}_{techs}_{country}_{ext}.log" + output: "results/plots/elec_s{simpl}_cum_p_nom_max_{clusts}_{techs}_{country}.{ext}" + log: "logs/plot_p_nom_max/elec_s{simpl}_{clusts}_{techs}_{country}_{ext}.log" script: "scripts/plot_p_nom_max.py" diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 67dab57c..3772bc43 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -119,8 +119,8 @@ orders ``snakemake`` to run the script ``solve_network`` that produces the solve .. code:: rule solve_network: - input: "networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + input: "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" [...] script: "scripts/solve_network.py" diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index d0b627a3..a7814ac6 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -37,13 +37,13 @@ Inputs Outputs ------- -- ``networks/{network}_s{simpl}_{clusters}_ec.nc``: +- ``networks/elec_s{simpl}_{clusters}_ec.nc``: Description ----------- -The rule :mod:`add_extra_components` attaches additional extendable components to the clustered and simplified network. These can be configured in the ``config.yaml`` at ``electricity: extendable_carriers: ``. It processes ``networks/{network}_s{simpl}_{clusters}.nc`` to build ``networks/{network}_s{simpl}_{clusters}_ec.nc``, which in contrast to the former (depending on the configuration) contain with **zero** initial capacity +The rule :mod:`add_extra_components` attaches additional extendable components to the clustered and simplified network. These can be configured in the ``config.yaml`` at ``electricity: extendable_carriers: ``. It processes ``networks/elec_s{simpl}_{clusters}.nc`` to build ``networks/elec_s{simpl}_{clusters}_ec.nc``, which in contrast to the former (depending on the configuration) contain with **zero** initial capacity - ``StorageUnits`` of carrier 'H2' and/or 'battery'. If this option is chosen, every bus is given an extendable ``StorageUnit`` of the corresponding carrier. The energy and power capacities are linked through a parameter that specifies the energy capacity as maximum hours at full dispatch power and is configured in ``electricity: max_hours:``. This linkage leads to one investment variable per storage unit. The default ``max_hours`` lead to long-term hydrogen and short-term battery storage units. diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 948b7cf3..47da0b24 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -31,26 +31,26 @@ Relevant Settings Inputs ------ -- ``resources/regions_onshore_{network}_s{simpl}.geojson``: confer :ref:`simplify` -- ``resources/regions_offshore_{network}_s{simpl}.geojson``: confer :ref:`simplify` -- ``resources/clustermaps_{network}_s{simpl}.h5``: confer :ref:`simplify` -- ``networks/{network}_s{simpl}.nc``: confer :ref:`simplify` +- ``resources/regions_onshore_elec_s{simpl}.geojson``: confer :ref:`simplify` +- ``resources/regions_offshore_elec_s{simpl}.geojson``: confer :ref:`simplify` +- ``resources/clustermaps_elec_s{simpl}.h5``: confer :ref:`simplify` +- ``networks/elec_s{simpl}.nc``: confer :ref:`simplify` Outputs ------- -- ``resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson``: +- ``resources/regions_onshore_elec_s{simpl}_{clusters}.geojson``: .. image:: ../img/regions_onshore_elec_s_X.png :scale: 33 % -- ``resources/regions_offshore_{network}_s{simpl}_{clusters}.geojson``: +- ``resources/regions_offshore_elec_s{simpl}_{clusters}.geojson``: .. image:: ../img/regions_offshore_elec_s_X.png :scale: 33 % -- ``resources/clustermaps_{network}_s{simpl}_{clusters}.h5``: Mapping of buses and lines from ``networks/elec_s{simpl}.nc`` to ``networks/elec_s{simpl}_{clusters}.nc``; has keys ['/busmap', '/busmap_s', '/linemap', '/linemap_negative', '/linemap_positive'] -- ``networks/{network}_s{simpl}_{clusters}.nc``: +- ``resources/clustermaps_elec_s{simpl}_{clusters}.h5``: Mapping of buses and lines from ``networks/elec_s{simpl}.nc`` to ``networks/elec_s{simpl}_{clusters}.nc``; has keys ['/busmap', '/busmap_s', '/linemap', '/linemap_negative', '/linemap_positive'] +- ``networks/elec_s{simpl}_{clusters}.nc``: .. image:: ../img/elec_s_X.png :scale: 40 % diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 7baab77f..a98c63fa 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -37,12 +37,12 @@ Inputs ------ - ``data/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity. -- ``networks/{network}_s{simpl}_{clusters}.nc``: confer :ref:`cluster` +- ``networks/elec_s{simpl}_{clusters}.nc``: confer :ref:`cluster` Outputs ------- -- ``networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: Complete PyPSA network that will be handed to the ``solve_network`` rule. +- ``networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: Complete PyPSA network that will be handed to the ``solve_network`` rule. Description ----------- diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index a6d6e613..eb6dbe6b 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -47,23 +47,23 @@ Inputs - ``data/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity. - ``resources/regions_onshore.geojson``: confer :ref:`busregions` - ``resources/regions_offshore.geojson``: confer :ref:`busregions` -- ``networks/{network}.nc``: confer :ref:`electricity` +- ``networks/elec.nc``: confer :ref:`electricity` Outputs ------- -- ``resources/regions_onshore_{network}_s{simpl}.geojson``: +- ``resources/regions_onshore_elec_s{simpl}.geojson``: .. image:: ../img/regions_onshore_elec_s.png :scale: 33 % -- ``resources/regions_offshore_{network}_s{simpl}.geojson``: +- ``resources/regions_offshore_elec_s{simpl}.geojson``: .. image:: ../img/regions_offshore_elec_s .png :scale: 33 % -- ``resources/clustermaps_{network}_s{simpl}.h5``: Mapping of buses from ``networks/elec.nc`` to ``networks/elec_s{simpl}.nc``; has keys ['/busmap_s'] -- ``networks/{network}_s{simpl}.nc``: +- ``resources/clustermaps_elec_s{simpl}.h5``: Mapping of buses from ``networks/elec.nc`` to ``networks/elec_s{simpl}.nc``; has keys ['/busmap_s'] +- ``networks/elec_s{simpl}.nc``: .. image:: ../img/elec_s.png :scale: 33 % diff --git a/scripts/solve_network.py b/scripts/solve_network.py index d88b7007..e6936db6 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -40,12 +40,12 @@ Relevant Settings Inputs ------ -- ``networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: confer :ref:`prepare` +- ``networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: confer :ref:`prepare` Outputs ------- -- ``results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: Solved PyPSA network including optimisation results +- ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: Solved PyPSA network including optimisation results .. image:: ../img/results.png :scale: 40 % diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 3b2089ca..c65e6889 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -32,13 +32,13 @@ Relevant Settings Inputs ------ -- ``networks/{network}_s{simpl}_{clusters}.nc``: confer :ref:`cluster` -- ``results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: confer :ref:`solve` +- ``networks/elec_s{simpl}_{clusters}.nc``: confer :ref:`cluster` +- ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: confer :ref:`solve` Outputs ------- -- ``results/networks/{network}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc``: Solved PyPSA network for optimal dispatch including optimisation results +- ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc``: Solved PyPSA network for optimal dispatch including optimisation results Description ----------- From cb2b78ad97f0ee876df963fbad59957485c4cfd7 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 18 Oct 2020 14:25:12 +0200 Subject: [PATCH 010/173] first step towards sweeping through multiple weather years --- Snakefile | 139 ++++++++++++++-------------- config.default.yaml | 1 + config.tutorial.yaml | 1 + scripts/add_electricity.py | 6 ++ scripts/base_network.py | 7 +- scripts/build_country_flh.py | 4 +- scripts/build_cutout.py | 1 + scripts/build_renewable_profiles.py | 4 +- test/config.test1.yaml | 1 + 9 files changed, 87 insertions(+), 77 deletions(-) diff --git a/Snakefile b/Snakefile index cd42835e..fb04256b 100644 --- a/Snakefile +++ b/Snakefile @@ -15,26 +15,27 @@ ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4) wildcard_constraints: + year="[0-9]*", simpl="[a-zA-Z0-9]*|all", clusters="[0-9]+m?|all", ll="(v|c)([0-9\.]+|opt|all)|all", opts="[-+a-zA-Z0-9\.]*" -rule cluster_all_elec_networks: - input: expand("networks/elec_s{simpl}_{clusters}.nc", **config['scenario']) +rule cluster_all_networks: + input: expand("networks/elec{year}_s{simpl}_{clusters}.nc", **config['scenario']) -rule extra_components_all_elec_networks: - input: expand("networks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) +rule extra_components_all_networks: + input: expand("networks/elec{year}_s{simpl}_{clusters}_ec.nc", **config['scenario']) -rule prepare_all_elec_networks: - input: expand("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) +rule prepare_all_networks: + input: expand("networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) -rule solve_all_elec_networks: - input: expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) +rule solve_all_networks: + input: expand("results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) if config['enable'].get('prepare_links_p_nom', False): @@ -180,9 +181,9 @@ rule build_renewable_profiles: else "resources/regions_offshore.geojson"), cutout=lambda w: "cutouts/" + config["renewable"][w.technology]['cutout'] output: - profile="resources/profile_{technology}.nc", - log: "logs/build_renewable_profile_{technology}.log" - benchmark: "benchmarks/build_renewable_profiles_{technology}" + profile="resources/profile{year}_{technology}.nc", + log: "logs/build_renewable_profile{year}_{technology}.log" + benchmark: "benchmarks/build_renewable_profiles{year}_{technology}" threads: ATLITE_NPROCESSES resources: mem=ATLITE_NPROCESSES * 5000 script: "scripts/build_renewable_profiles.py" @@ -194,8 +195,8 @@ if 'hydro' in config['renewable'].keys(): country_shapes='resources/country_shapes.geojson', eia_hydro_generation='data/bundle/EIA_hydro_generation_2000_2014.csv', cutout="cutouts/" + config["renewable"]['hydro']['cutout'] - output: 'resources/profile_hydro.nc' - log: "logs/build_hydro_profile.log" + output: 'resources/profile{year}_hydro.nc' + log: "logs/build_hydro_profile{year}.log" resources: mem=5000 script: 'scripts/build_hydro_profile.py' @@ -210,11 +211,11 @@ rule add_electricity: geth_hydro_capacities='data/geth2015_hydro_capacities.csv', opsd_load='data/bundle/time_series_60min_singleindex_filtered.csv', nuts3_shapes='resources/nuts3_shapes.geojson', - **{f"profile_{tech}": f"resources/profile_{tech}.nc" + **{f"profile_{tech}": "resources/profile{year}_" + f"{tech}.nc" for tech in config['renewable']} - output: "networks/elec.nc" - log: "logs/add_electricity.log" - benchmark: "benchmarks/add_electricity" + output: "networks/elec{year}.nc" + log: "logs/add_electricity{year}.log" + benchmark: "benchmarks/add_electricity{year}" threads: 1 resources: mem=3000 script: "scripts/add_electricity.py" @@ -222,17 +223,17 @@ rule add_electricity: rule simplify_network: input: - network='networks/elec.nc', + network='networks/elec{year}.nc', tech_costs=COSTS, regions_onshore="resources/regions_onshore.geojson", regions_offshore="resources/regions_offshore.geojson" output: - network='networks/elec_s{simpl}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", - busmap='resources/busmap_elec_s{simpl}.csv' - log: "logs/simplify_network/elec_s{simpl}.log" - benchmark: "benchmarks/simplify_network/elec_s{simpl}" + network='networks/elec{year}_s{simpl}.nc', + regions_onshore="resources/regions_onshore_elec{year}_s{simpl}.geojson", + regions_offshore="resources/regions_offshore_elec{year}_s{simpl}.geojson", + busmap='resources/busmap_elec{year}_s{simpl}.csv' + log: "logs/simplify_network/elec{year}_s{simpl}.log" + benchmark: "benchmarks/simplify_network/elec{year}_s{simpl}" threads: 1 resources: mem=4000 script: "scripts/simplify_network.py" @@ -240,19 +241,19 @@ rule simplify_network: rule cluster_network: input: - network='networks/elec_s{simpl}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", - busmap=ancient('resources/busmap_elec_s{simpl}.csv'), + network='networks/elec{year}_s{simpl}.nc', + regions_onshore="resources/regions_onshore_elec{year}_s{simpl}.geojson", + regions_offshore="resources/regions_offshore_elec{year}_s{simpl}.geojson", + busmap=ancient('resources/busmap_elec{year}_s{simpl}.csv'), tech_costs=COSTS output: - network='networks/elec_s{simpl}_{clusters}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", - busmap="resources/busmap_elec_s{simpl}_{clusters}.csv", - linemap="resources/linemap_elec_s{simpl}_{clusters}.csv" - log: "logs/cluster_network/elec_s{simpl}_{clusters}.log" - benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}" + network='networks/elec{year}_s{simpl}_{clusters}.nc', + regions_onshore="resources/regions_onshore_elec{year}_s{simpl}_{clusters}.geojson", + regions_offshore="resources/regions_offshore_elec{year}_s{simpl}_{clusters}.geojson", + busmap="resources/busmap_elec{year}_s{simpl}_{clusters}.csv", + linemap="resources/linemap_elec{year}_s{simpl}_{clusters}.csv" + log: "logs/cluster_network/elec{year}_s{simpl}_{clusters}.log" + benchmark: "benchmarks/cluster_network/elec{year}_s{simpl}_{clusters}" threads: 1 resources: mem=3000 script: "scripts/cluster_network.py" @@ -260,21 +261,21 @@ rule cluster_network: rule add_extra_components: input: - network='networks/elec_s{simpl}_{clusters}.nc', + network='networks/elec{year}_s{simpl}_{clusters}.nc', tech_costs=COSTS, - output: 'networks/elec_s{simpl}_{clusters}_ec.nc' - log: "logs/add_extra_components/elec_s{simpl}_{clusters}.log" - benchmark: "benchmarks/add_extra_components/elec_s{simpl}_{clusters}_ec" + output: 'networks/elec{year}_s{simpl}_{clusters}_ec.nc' + log: "logs/add_extra_components/elec{year}_s{simpl}_{clusters}.log" + benchmark: "benchmarks/add_extra_components/elec{year}_s{simpl}_{clusters}_ec" threads: 1 resources: mem=3000 script: "scripts/add_extra_components.py" rule prepare_network: - input: 'networks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS - output: 'networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' - log: "logs/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" - benchmark: "benchmarks/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" + input: 'networks/elec{year}_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS + output: 'networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' + log: "logs/prepare_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" + benchmark: "benchmarks/prepare_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 1 resources: mem=1000 script: "scripts/prepare_network.py" @@ -294,13 +295,13 @@ def memory(w): rule solve_network: - input: "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + input: "networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" log: - solver=normpath("logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), - python="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", - memory="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log" - benchmark: "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" + solver=normpath("logs/solve_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), + python="logs/solve_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", + memory="logs/solve_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log" + benchmark: "benchmarks/solve_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 resources: mem=memory shadow: "shallow" @@ -309,14 +310,14 @@ rule solve_network: rule solve_operations_network: input: - unprepared="networks/elec_s{simpl}_{clusters}_ec.nc", - optimized="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" + unprepared="networks/elec{year}_s{simpl}_{clusters}_ec.nc", + optimized="results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" log: - solver=normpath("logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), - python="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", - memory="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log" - benchmark: "benchmarks/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" + solver=normpath("logs/solve_operations_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), + python="logs/solve_operations_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", + memory="logs/solve_operations_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log" + benchmark: "benchmarks/solve_operations_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 resources: mem=(lambda w: 5000 + 372 * int(w.clusters)) shadow: "shallow" @@ -325,12 +326,12 @@ rule solve_operations_network: rule plot_network: input: - network="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network="results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", tech_costs=COSTS output: - only_map="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", - ext="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_ext.{ext}" - log: "logs/plot_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_{ext}.log" + only_map="results/plots/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", + ext="results/plots/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_ext.{ext}" + log: "logs/plot_network/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_{ext}.log" script: "scripts/plot_network.py" @@ -343,7 +344,7 @@ def input_make_summary(w): else: ll = w.ll return ([COSTS] + - expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + expand("results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", network=w.network, ll=ll, **{k: config["scenario"][k] if getattr(w, k) == "all" else getattr(w, k) @@ -352,28 +353,28 @@ def input_make_summary(w): rule make_summary: input: input_make_summary - output: directory("results/summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}") - log: "logs/make_summary/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log", + output: directory("results/summaries/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}") + log: "logs/make_summary/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log", script: "scripts/make_summary.py" rule plot_summary: - input: "results/summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}" - output: "results/plots/summary_{summary}_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}" - log: "logs/plot_summary/{summary}_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log" + input: "results/summaries/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}" + output: "results/plots/{year}/summary_{summary}_elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}" + log: "logs/plot_summary/{summary}_elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log" script: "scripts/plot_summary.py" def input_plot_p_nom_max(w): - return [("networks/elec_s{simpl}{maybe_cluster}.nc" + return [("networks/elec{year}_s{simpl}{maybe_cluster}.nc" .format(maybe_cluster=('' if c == 'full' else ('_' + c)), **w)) for c in w.clusts.split(",")] rule plot_p_nom_max: input: input_plot_p_nom_max - output: "results/plots/elec_s{simpl}_cum_p_nom_max_{clusts}_{techs}_{country}.{ext}" - log: "logs/plot_p_nom_max/elec_s{simpl}_{clusts}_{techs}_{country}_{ext}.log" + output: "results/plots/elec{year}_s{simpl}_cum_p_nom_max_{clusts}_{techs}_{country}.{ext}" + log: "logs/plot_p_nom_max/elec{year}_s{simpl}_{clusts}_{techs}_{country}_{ext}.log" script: "scripts/plot_p_nom_max.py" diff --git a/config.default.yaml b/config.default.yaml index b46432c6..0ba49caf 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -12,6 +12,7 @@ logging: summary_dir: results scenario: + year: [''] # for backwards compatibility simpl: [''] ll: ['copt'] clusters: [37, 128, 256, 512, 1024] diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 174708cb..52b5b866 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -11,6 +11,7 @@ logging: summary_dir: results scenario: + year: [''] simpl: [''] ll: ['copt'] clusters: [5] diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index c11c27f5..a13cdb15 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -212,6 +212,7 @@ def attach_load(n): substation_lv_i = n.buses.index[n.buses['substation_lv']] regions = (gpd.read_file(snakemake.input.regions).set_index('name') .reindex(substation_lv_i)) + # TODO: is there a fallback if year not in timeseries_opsd? opsd_load = (timeseries_opsd(slice(*n.snapshots[[0,-1]].year.astype(str)), snakemake.input.opsd_load) * snakemake.config.get('load', {}).get('scaling_factor', 1.0)) @@ -522,6 +523,11 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) + + year = int(snakemake.wildcards.year) + snapshots = dict(start=str(year), end=str(year+1), closed="left") if year else snakememake.config['snapshots'] + n.set_snapshots(pd.date_range(freq='h', **snapshots)) + n.snapshot_weightings[:] *= 8760. / n.snapshot_weightings.sum() Nyears = n.snapshot_weightings.sum() / 8760. costs = load_costs(Nyears) diff --git a/scripts/base_network.py b/scripts/base_network.py index d2ceccf1..0b535218 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -11,8 +11,6 @@ Relevant Settings .. code:: yaml - snapshots: - countries: electricity: @@ -35,7 +33,7 @@ Relevant Settings .. seealso:: Documentation of the configuration file ``config.yaml`` at - :ref:`snapshots_cf`, :ref:`toplevel_cf`, :ref:`electricity_cf`, :ref:`load_cf`, + :ref:`toplevel_cf`, :ref:`electricity_cf`, :ref:`load_cf`, :ref:`lines_cf`, :ref:`links_cf`, :ref:`transformers_cf` Inputs @@ -554,9 +552,6 @@ def base_network(): n = pypsa.Network() n.name = 'PyPSA-Eur' - n.set_snapshots(pd.date_range(freq='h', **snakemake.config['snapshots'])) - n.snapshot_weightings[:] *= 8760. / n.snapshot_weightings.sum() - n.import_components_from_dataframe(buses, "Bus") n.import_components_from_dataframe(lines, "Line") n.import_components_from_dataframe(transformers, "Transformer") diff --git a/scripts/build_country_flh.py b/scripts/build_country_flh.py index 459b8f38..345d1d09 100644 --- a/scripts/build_country_flh.py +++ b/scripts/build_country_flh.py @@ -165,7 +165,9 @@ if __name__ == '__main__': config = snakemake.config['renewable'][snakemake.wildcards.technology] - time = pd.date_range(freq='m', **snakemake.config['snapshots']) + year = int(snakemake.wildcards.year) + snapshots = dict(start=str(year), end=str(year+1), closed="left") if year else snakememake.config['snapshots'] + time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) cutout = atlite.Cutout(config['cutout'], diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 1e55faf5..dc909c09 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -105,6 +105,7 @@ if __name__ == "__main__": snakemake = mock_snakemake('build_cutout', cutout='europe-2013-era5') configure_logging(snakemake) + # TODO make it accept year independent config cutout_params = snakemake.config['atlite']['cutouts'][snakemake.wildcards.cutout] for p in ('xs', 'ys', 'years', 'months'): if p in cutout_params: diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 71adb66e..9b477625 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -284,7 +284,9 @@ if __name__ == '__main__': config = snakemake.config['renewable'][snakemake.wildcards.technology] - time = pd.date_range(freq='m', **snakemake.config['snapshots']) + year = int(snakemake.wildcards.year) + snapshots = dict(start=str(year), end=str(year+1), closed="left") if year else snakememake.config['snapshots'] + time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) cutout = atlite.Cutout(config['cutout'], diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 84d551e6..a1fc59ba 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -11,6 +11,7 @@ logging: summary_dir: results scenario: + year: [''] simpl: [''] ll: ['copt'] clusters: [5] From ca0ad084381d3ffdda99433cfc4e9663fbc597f0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 18 Oct 2020 14:47:46 +0200 Subject: [PATCH 011/173] reflect filename changes in documentation --- .travis.yml | 2 +- doc/cloudcomputing.rst | 2 +- doc/configuration.rst | 4 ++-- doc/release_notes.rst | 2 +- doc/solving.rst | 2 +- doc/tutorial.rst | 6 +++--- scripts/add_electricity.py | 4 ++-- scripts/add_extra_components.py | 4 ++-- scripts/build_country_flh.py | 4 ++-- scripts/build_renewable_profiles.py | 4 ++-- scripts/cluster_network.py | 16 ++++++++-------- scripts/prepare_network.py | 4 ++-- scripts/simplify_network.py | 8 ++++---- scripts/solve_network.py | 4 ++-- scripts/solve_operations_network.py | 6 +++--- 15 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.travis.yml b/.travis.yml index a6a29a39..b7b8c4d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,6 @@ before_install: script: - cp ./test/config.test1.yaml ./config.yaml - - snakemake -j all solve_all_elec_networks + - snakemake -j all solve_all_networks - rm -rf resources/*.nc resources/*.geojson resources/*.h5 networks results # could repeat for more configurations in future diff --git a/doc/cloudcomputing.rst b/doc/cloudcomputing.rst index f91441a5..95bfa2c7 100644 --- a/doc/cloudcomputing.rst +++ b/doc/cloudcomputing.rst @@ -60,7 +60,7 @@ Now a window with the machine details will open. You have to configure the follo You can edit your machine configuration later. So use a cheap machine type configuration to transfer data and only when everything is ready and tested, your expensive machine type, for instance a custom 8 vCPU with 160 GB memory. Solvers do not parallelise well, so we recommend not to choose more than 8 vCPU. - Check ``snakemake -j -n 1 solve_all_elec_networks`` as a dry run to see how much memory is required. + Check ``snakemake -j -n 1 solve_all_networks`` as a dry run to see how much memory is required. The memory requirements will vary depending on the spatial and temporal resoulution of your optimisation. Example: for an hourly, 181 node full European network, set 8 vCPU and 150 GB memory since the dry-run calculated a 135 GB memory requirement.) - Boot disk: As default, your VM is created with 10 GB. Depending on how much you want to handle on one VM you should increase the disk size. diff --git a/doc/configuration.rst b/doc/configuration.rst index b065290b..95d7af31 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -40,9 +40,9 @@ facilitate running multiple scenarios through a single command .. code:: bash - snakemake -j 1 solve_all_elec_networks + snakemake -j 1 solve_all_networks -For each wildcard, a **list of values** is provided. The rule ``solve_all_elec_networks`` will trigger the rules for creating ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc`` for **all combinations** of the provided wildcard values as defined by Python's `itertools.product(...) `_ function that snakemake's `expand(...) function `_ uses. +For each wildcard, a **list of values** is provided. The rule ``solve_all_networks`` will trigger the rules for creating ``results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc`` for **all combinations** of the provided wildcard values as defined by Python's `itertools.product(...) `_ function that snakemake's `expand(...) function `_ uses. An exemplary dependency graph (starting from the simplification rules) then looks like this: diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 9fc72206..5d2e0f05 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -124,7 +124,7 @@ Release Process * Upload code to `zenodo code repository `_ with `GNU GPL 3.0 `_ license. -* Create pre-built networks for ``config.default.yaml`` by running ``snakemake -j 1 extra_components_all_elec_networks``. +* Create pre-built networks for ``config.default.yaml`` by running ``snakemake -j 1 extra_components_all_networks``. * Upload pre-built networks to `zenodo data repository `_ with `CC BY 4.0 `_ license. diff --git a/doc/solving.rst b/doc/solving.rst index 2cc8eb57..87fdc040 100644 --- a/doc/solving.rst +++ b/doc/solving.rst @@ -7,7 +7,7 @@ Solving Networks ########################################## -After generating and simplifying the networks they can be solved through the rule :mod:`solve_network` by using the collection rule :mod:`solve_all_elec_networks`. Moreover, networks can be solved for another focus with the derivative rules :mod:`solve_network` by using the collection rule :mod:`solve_operations_network` for dispatch-only analyses on an already solved network. +After generating and simplifying the networks they can be solved through the rule :mod:`solve_network` by using the collection rule :mod:`solve_all_networks`. Moreover, networks can be solved for another focus with the derivative rules :mod:`solve_network` by using the collection rule :mod:`solve_operations_network` for dispatch-only analyses on an already solved network. .. toctree:: :caption: Overview diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 3772bc43..53252ff8 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -119,8 +119,8 @@ orders ``snakemake`` to run the script ``solve_network`` that produces the solve .. code:: rule solve_network: - input: "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + input: "networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" [...] script: "scripts/solve_network.py" @@ -129,7 +129,7 @@ orders ``snakemake`` to run the script ``solve_network`` that produces the solve .. warning:: On Windows the previous command may currently cause a ``MissingRuleException`` due to problems with output files in subfolders. This is an `open issue `_ at `snakemake `_. - Windows users should add the option ``--keep-target-files`` to the command or instead run ``snakemake -j 1 solve_all_elec_networks``. + Windows users should add the option ``--keep-target-files`` to the command or instead run ``snakemake -j 1 solve_all_networks``. This triggers a workflow of multiple preceding jobs that depend on each rule's inputs and outputs: diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index a13cdb15..8ac455f7 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -524,8 +524,8 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.base_network) - year = int(snakemake.wildcards.year) - snapshots = dict(start=str(year), end=str(year+1), closed="left") if year else snakememake.config['snapshots'] + year = snakemake.wildcards.year + snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakememake.config['snapshots'] n.set_snapshots(pd.date_range(freq='h', **snapshots)) n.snapshot_weightings[:] *= 8760. / n.snapshot_weightings.sum() Nyears = n.snapshot_weightings.sum() / 8760. diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index a7814ac6..c3af5c68 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -37,13 +37,13 @@ Inputs Outputs ------- -- ``networks/elec_s{simpl}_{clusters}_ec.nc``: +- ``networks/elec{year}_s{simpl}_{clusters}_ec.nc``: Description ----------- -The rule :mod:`add_extra_components` attaches additional extendable components to the clustered and simplified network. These can be configured in the ``config.yaml`` at ``electricity: extendable_carriers: ``. It processes ``networks/elec_s{simpl}_{clusters}.nc`` to build ``networks/elec_s{simpl}_{clusters}_ec.nc``, which in contrast to the former (depending on the configuration) contain with **zero** initial capacity +The rule :mod:`add_extra_components` attaches additional extendable components to the clustered and simplified network. These can be configured in the ``config.yaml`` at ``electricity: extendable_carriers: ``. It processes ``networks/elec{year}_s{simpl}_{clusters}.nc`` to build ``networks/elec{year}_s{simpl}_{clusters}_ec.nc``, which in contrast to the former (depending on the configuration) contain with **zero** initial capacity - ``StorageUnits`` of carrier 'H2' and/or 'battery'. If this option is chosen, every bus is given an extendable ``StorageUnit`` of the corresponding carrier. The energy and power capacities are linked through a parameter that specifies the energy capacity as maximum hours at full dispatch power and is configured in ``electricity: max_hours:``. This linkage leads to one investment variable per storage unit. The default ``max_hours`` lead to long-term hydrogen and short-term battery storage units. diff --git a/scripts/build_country_flh.py b/scripts/build_country_flh.py index 345d1d09..de78749c 100644 --- a/scripts/build_country_flh.py +++ b/scripts/build_country_flh.py @@ -165,8 +165,8 @@ if __name__ == '__main__': config = snakemake.config['renewable'][snakemake.wildcards.technology] - year = int(snakemake.wildcards.year) - snapshots = dict(start=str(year), end=str(year+1), closed="left") if year else snakememake.config['snapshots'] + year = snakemake.wildcards.year + snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakememake.config['snapshots'] time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 9b477625..a02d397b 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -284,8 +284,8 @@ if __name__ == '__main__': config = snakemake.config['renewable'][snakemake.wildcards.technology] - year = int(snakemake.wildcards.year) - snapshots = dict(start=str(year), end=str(year+1), closed="left") if year else snakememake.config['snapshots'] + year = snakemake.wildcards.year + snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakememake.config['snapshots'] time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 47da0b24..e48cb919 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -31,26 +31,26 @@ Relevant Settings Inputs ------ -- ``resources/regions_onshore_elec_s{simpl}.geojson``: confer :ref:`simplify` -- ``resources/regions_offshore_elec_s{simpl}.geojson``: confer :ref:`simplify` -- ``resources/clustermaps_elec_s{simpl}.h5``: confer :ref:`simplify` -- ``networks/elec_s{simpl}.nc``: confer :ref:`simplify` +- ``resources/regions_onshore_elec{year}_s{simpl}.geojson``: confer :ref:`simplify` +- ``resources/regions_offshore_elec{year}_s{simpl}.geojson``: confer :ref:`simplify` +- ``resources/clustermaps_elec{year}_s{simpl}.h5``: confer :ref:`simplify` +- ``networks/elec{year}_s{simpl}.nc``: confer :ref:`simplify` Outputs ------- -- ``resources/regions_onshore_elec_s{simpl}_{clusters}.geojson``: +- ``resources/regions_onshore_elec{year}_s{simpl}_{clusters}.geojson``: .. image:: ../img/regions_onshore_elec_s_X.png :scale: 33 % -- ``resources/regions_offshore_elec_s{simpl}_{clusters}.geojson``: +- ``resources/regions_offshore_elec{year}_s{simpl}_{clusters}.geojson``: .. image:: ../img/regions_offshore_elec_s_X.png :scale: 33 % -- ``resources/clustermaps_elec_s{simpl}_{clusters}.h5``: Mapping of buses and lines from ``networks/elec_s{simpl}.nc`` to ``networks/elec_s{simpl}_{clusters}.nc``; has keys ['/busmap', '/busmap_s', '/linemap', '/linemap_negative', '/linemap_positive'] -- ``networks/elec_s{simpl}_{clusters}.nc``: +- ``resources/clustermaps_elec{year}_s{simpl}_{clusters}.h5``: Mapping of buses and lines from ``networks/elec{year}_s{simpl}.nc`` to ``networks/elec{year}_s{simpl}_{clusters}.nc``; has keys ['/busmap', '/busmap_s', '/linemap', '/linemap_negative', '/linemap_positive'] +- ``networks/elec{year}_s{simpl}_{clusters}.nc``: .. image:: ../img/elec_s_X.png :scale: 40 % diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index a98c63fa..a93f8679 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -37,12 +37,12 @@ Inputs ------ - ``data/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity. -- ``networks/elec_s{simpl}_{clusters}.nc``: confer :ref:`cluster` +- ``networks/elec{year}_s{simpl}_{clusters}.nc``: confer :ref:`cluster` Outputs ------- -- ``networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: Complete PyPSA network that will be handed to the ``solve_network`` rule. +- ``networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: Complete PyPSA network that will be handed to the ``solve_network`` rule. Description ----------- diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index eb6dbe6b..780a794d 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -52,18 +52,18 @@ Inputs Outputs ------- -- ``resources/regions_onshore_elec_s{simpl}.geojson``: +- ``resources/regions_onshore_elec{year}_s{simpl}.geojson``: .. image:: ../img/regions_onshore_elec_s.png :scale: 33 % -- ``resources/regions_offshore_elec_s{simpl}.geojson``: +- ``resources/regions_offshore_elec{year}_s{simpl}.geojson``: .. image:: ../img/regions_offshore_elec_s .png :scale: 33 % -- ``resources/clustermaps_elec_s{simpl}.h5``: Mapping of buses from ``networks/elec.nc`` to ``networks/elec_s{simpl}.nc``; has keys ['/busmap_s'] -- ``networks/elec_s{simpl}.nc``: +- ``resources/clustermaps_elec{year}_s{simpl}.h5``: Mapping of buses from ``networks/elec.nc`` to ``networks/elec{year}_s{simpl}.nc``; has keys ['/busmap_s'] +- ``networks/elec{year}_s{simpl}.nc``: .. image:: ../img/elec_s.png :scale: 33 % diff --git a/scripts/solve_network.py b/scripts/solve_network.py index e6936db6..6e733cd8 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -40,12 +40,12 @@ Relevant Settings Inputs ------ -- ``networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: confer :ref:`prepare` +- ``networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: confer :ref:`prepare` Outputs ------- -- ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: Solved PyPSA network including optimisation results +- ``results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: Solved PyPSA network including optimisation results .. image:: ../img/results.png :scale: 40 % diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index c65e6889..7355bf40 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -32,13 +32,13 @@ Relevant Settings Inputs ------ -- ``networks/elec_s{simpl}_{clusters}.nc``: confer :ref:`cluster` -- ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: confer :ref:`solve` +- ``networks/elec{year}_s{simpl}_{clusters}.nc``: confer :ref:`cluster` +- ``results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: confer :ref:`solve` Outputs ------- -- ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc``: Solved PyPSA network for optimal dispatch including optimisation results +- ``results/networks/elec{year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc``: Solved PyPSA network for optimal dispatch including optimisation results Description ----------- From dacfc8a9a4c3d175297f73bba84ca72c8ad3b87f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 20 Oct 2020 13:53:43 +0200 Subject: [PATCH 012/173] multiyear: fallbacks for hydro and load profiles --- Snakefile | 2 ++ config.default.yaml | 1 + config.tutorial.yaml | 1 + scripts/add_electricity.py | 21 +++++++++++++++++++-- scripts/build_hydro_profile.py | 10 +++++++++- scripts/build_renewable_profiles.py | 8 +++++--- test/config.test1.yaml | 1 + 7 files changed, 38 insertions(+), 6 deletions(-) diff --git a/Snakefile b/Snakefile index fb04256b..b8bc284a 100644 --- a/Snakefile +++ b/Snakefile @@ -166,6 +166,8 @@ if config['enable'].get('retrieve_natura_raster', True): script: 'scripts/retrieve_natura_raster.py' +ruleorder: build_hydro_profile > build_renewable_profiles + rule build_renewable_profiles: input: base_network="networks/base.nc", diff --git a/config.default.yaml b/config.default.yaml index 0ba49caf..9f096fff 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -169,6 +169,7 @@ transformers: load: scaling_factor: 1.0 + fallback_year: 2013 costs: year: 2030 diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 52b5b866..e74a2967 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -147,6 +147,7 @@ transformers: load: scaling_factor: 1.0 + fallback_year: 2013 costs: year: 2030 diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 8ac455f7..98893010 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -212,11 +212,28 @@ def attach_load(n): substation_lv_i = n.buses.index[n.buses['substation_lv']] regions = (gpd.read_file(snakemake.input.regions).set_index('name') .reindex(substation_lv_i)) - # TODO: is there a fallback if year not in timeseries_opsd? - opsd_load = (timeseries_opsd(slice(*n.snapshots[[0,-1]].year.astype(str)), + + available_years = range(2011,2016) + requested_years = n.snapshots.year[[0,-1]] + use_fallback = any(year not in available_years for year in requested_years) + + if use_fallback: + fallback_year = str(snakemake.config["load"]["fallback_year"]) + load_years = [fallback_year, fallback_year] + logger.warning(f"Requested years {list(requested_years.unique().values)} " + f"for load time series not in available years {list(available_years)}. " + f"Falling back to year {fallback_year}.") + else: + load_years = requested_years.astype(str) + + opsd_load = (timeseries_opsd(slice(*load_years), snakemake.input.opsd_load) * snakemake.config.get('load', {}).get('scaling_factor', 1.0)) + if use_fallback: + assert len(requested_years.unique()) == 1, "Fallback for load time series requires single year!" + opsd_load.index = opsd_load.index.map(lambda t: t.replace(year=requested_years[0])) + # Convert to naive UTC (has to be explicit since pandas 0.24) opsd_load.index = opsd_load.index.tz_localize(None) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 339fccaf..46911363 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -75,15 +75,23 @@ if __name__ == "__main__": snakemake = mock_snakemake('build_hydro_profile') configure_logging(snakemake) + year = snakemake.wildcards.year config = snakemake.config['renewable']['hydro'] cutout_dir = os.path.dirname(snakemake.input.cutout) - cutout = atlite.Cutout(config['cutout'], cutout_dir=cutout_dir) + cutout_config = config['cutout'] + if year: cutout_config = cutout_config.format(year=year) + + cutout = atlite.Cutout(cutout_config, cutout_dir=cutout_dir) countries = snakemake.config['countries'] country_shapes = gpd.read_file(snakemake.input.country_shapes).set_index('name')['geometry'].reindex(countries) country_shapes.index.name = 'countries' eia_stats = vhydro.get_eia_annual_hydro_generation(snakemake.input.eia_hydro_generation).reindex(columns=countries) + + if year not in eia_stats.index: + eia_stats.loc[year] = eia_stats.mean() + inflow = cutout.runoff(shapes=country_shapes, smooth=True, lower_threshold_quantile=True, diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index a02d397b..7b838725 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -282,14 +282,16 @@ if __name__ == '__main__': pgb.streams.wrap_stderr() - config = snakemake.config['renewable'][snakemake.wildcards.technology] - year = snakemake.wildcards.year + config = snakemake.config['renewable'][snakemake.wildcards.technology] + cutout_name = config['cutout'] + if year: cutout_name = cutout_name.format(year=year) + snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakememake.config['snapshots'] time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) - cutout = atlite.Cutout(config['cutout'], + cutout = atlite.Cutout(cutout_name, cutout_dir=os.path.dirname(snakemake.input.cutout), **params) diff --git a/test/config.test1.yaml b/test/config.test1.yaml index a1fc59ba..2774be8a 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -147,6 +147,7 @@ transformers: load: scaling_factor: 1.0 + fallback_year: 2013 costs: year: 2030 From 4d2f379ae1dbdff666fa5443a215a4dffad86d68 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 27 Oct 2020 11:30:25 +0100 Subject: [PATCH 013/173] build_hydro: handle empty year wildcard --- scripts/build_hydro_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 46911363..7e1bd212 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -89,7 +89,7 @@ if __name__ == "__main__": eia_stats = vhydro.get_eia_annual_hydro_generation(snakemake.input.eia_hydro_generation).reindex(columns=countries) - if year not in eia_stats.index: + if len(year) > 0 and year not in eia_stats.index: eia_stats.loc[year] = eia_stats.mean() inflow = cutout.runoff(shapes=country_shapes, From 5442d541b4ca5ebd27d67e710c909041c7e72308 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 27 Oct 2020 11:42:18 +0100 Subject: [PATCH 014/173] build_renewables: fix snakemake typo --- scripts/build_renewable_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 7b838725..3201d20b 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -287,7 +287,7 @@ if __name__ == '__main__': cutout_name = config['cutout'] if year: cutout_name = cutout_name.format(year=year) - snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakememake.config['snapshots'] + snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) From eed314ef057dad9baf959d2672e6e41cc5490faf Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 27 Oct 2020 13:40:34 +0100 Subject: [PATCH 015/173] add_electricity: fix snakemake typo --- scripts/add_electricity.py | 2 +- scripts/build_country_flh.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 98893010..a5495585 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -542,7 +542,7 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.base_network) year = snakemake.wildcards.year - snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakememake.config['snapshots'] + snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] n.set_snapshots(pd.date_range(freq='h', **snapshots)) n.snapshot_weightings[:] *= 8760. / n.snapshot_weightings.sum() Nyears = n.snapshot_weightings.sum() / 8760. diff --git a/scripts/build_country_flh.py b/scripts/build_country_flh.py index de78749c..0b65f136 100644 --- a/scripts/build_country_flh.py +++ b/scripts/build_country_flh.py @@ -166,7 +166,7 @@ if __name__ == '__main__': config = snakemake.config['renewable'][snakemake.wildcards.technology] year = snakemake.wildcards.year - snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakememake.config['snapshots'] + snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) From 4fef5616e887c0c9e0cdbee484bf665c763e27bb Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 10 Nov 2020 09:48:13 +0100 Subject: [PATCH 016/173] scripts: change mock snakemake defaults --- scripts/add_brownfield.py | 12 ++++++------ scripts/add_existing_baseyear.py | 14 +++++++------- scripts/prepare_sector_network.py | 24 ++++++++++++------------ scripts/solve_network.py | 8 ++++---- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index a0b8d97b..a3624395 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -86,16 +86,16 @@ if __name__ == "__main__": if 'snakemake' not in globals(): from vresutils.snakemake import MockSnakemake snakemake = MockSnakemake( - wildcards=dict(network='elec', simpl='', clusters='37', lv='1.0', + wildcards=dict(simpl='', clusters='37', lv='1.0', sector_opts='Co2L0-168H-T-H-B-I-solar3-dist1', co2_budget_name='go', planning_horizons='2030'), - input=dict(network='pypsa-eur-sec/results/test/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', - network_p='pypsa-eur-sec/results/test/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_2020.nc', + input=dict(network='pypsa-eur-sec/results/test/prenetworks/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', + network_p='pypsa-eur-sec/results/test/postnetworks/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_2020.nc', costs='pypsa-eur-sec/data/costs/costs_{planning_horizons}.csv', - cop_air_total="pypsa-eur-sec/resources/cop_air_total_{network}_s{simpl}_{clusters}.nc", - cop_soil_total="pypsa-eur-sec/resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc"), - output=['pypsa-eur-sec/results/test/prenetworks_brownfield/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{planning_horizons}.nc'] + cop_air_total="pypsa-eur-sec/resources/cop_air_total_elec_s{simpl}_{clusters}.nc", + cop_soil_total="pypsa-eur-sec/resources/cop_soil_total_elec_s{simpl}_{clusters}.nc"), + output=['pypsa-eur-sec/results/test/prenetworks_brownfield/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{planning_horizons}.nc'] ) import yaml with open('config.yaml', encoding='utf8') as f: diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index cfef2e5e..9eb2ae3a 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -408,18 +408,18 @@ if __name__ == "__main__": if 'snakemake' not in globals(): from vresutils.snakemake import MockSnakemake snakemake = MockSnakemake( - wildcards=dict(network='elec', simpl='', clusters='39', lv='1.0', + wildcards=dict(simpl='', clusters='39', lv='1.0', sector_opts='Co2L0-168H-T-H-B-I-solar3-dist1', co2_budget_name='b30b3', planning_horizons='2020'), - input=dict(network='pypsa-eur-sec/results/test/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', + input=dict(network='pypsa-eur-sec/results/test/prenetworks/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', powerplants='pypsa-eur/resources/powerplants.csv', - busmap_s='pypsa-eur/resources/busmap_{network}_s{simpl}.csv', - busmap='pypsa-eur/resources/busmap_{network}_s{simpl}_{clusters}.csv', + busmap_s='pypsa-eur/resources/busmap_elec_s{simpl}.csv', + busmap='pypsa-eur/resources/busmap_elec_s{simpl}_{clusters}.csv', costs='pypsa-eur-sec/data/costs/costs_{planning_horizons}.csv', - cop_air_total="pypsa-eur-sec/resources/cop_air_total_{network}_s{simpl}_{clusters}.nc", - cop_soil_total="pypsa-eur-sec/resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc"), - output=['pypsa-eur-sec/results/test/prenetworks_brownfield/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{planning_horizons}.nc'], + cop_air_total="pypsa-eur-sec/resources/cop_air_total_elec_s{simpl}_{clusters}.nc", + cop_soil_total="pypsa-eur-sec/resources/cop_soil_total_elec_s{simpl}_{clusters}.nc"), + output=['pypsa-eur-sec/results/test/prenetworks_brownfield/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{planning_horizons}.nc'], ) import yaml with open('config.yaml', encoding='utf8') as f: diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index f6e8a7be..990b6710 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1745,31 +1745,31 @@ if __name__ == "__main__": if 'snakemake' not in globals(): from vresutils.snakemake import MockSnakemake snakemake = MockSnakemake( - wildcards=dict(network='elec', simpl='', clusters='37', lv='1.0', + wildcards=dict(simpl='', clusters='37', lv='1.0', opts='', planning_horizons='2020', co2_budget_name='go', sector_opts='Co2L0-168H-T-H-B-I-solar3-dist1'), - input=dict(network='pypsa-eur/networks/{network}_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc', + input=dict(network='pypsa-eur/networks/elec_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc', timezone_mappings='pypsa-eur-sec/data/timezone_mappings.csv', co2_budget='pypsa-eur-sec/data/co2_budget.csv', - clustered_pop_layout='pypsa-eur-sec/resources/pop_layout_{network}_s{simpl}_{clusters}.csv', + clustered_pop_layout='pypsa-eur-sec/resources/pop_layout_elec_s{simpl}_{clusters}.csv', costs='technology-data/outputs/costs_{planning_horizons}.csv', profile_offwind_ac='pypsa-eur/resources/profile_offwind-ac.nc', profile_offwind_dc='pypsa-eur/resources/profile_offwind-dc.nc', - busmap_s='pypsa-eur/resources/busmap_{network}_s{simpl}.csv', - busmap='pypsa-eur/resources/busmap_{network}_s{simpl}_{clusters}.csv', - cop_air_total='pypsa-eur-sec/resources/cop_air_total_{network}_s{simpl}_{clusters}.nc', - cop_soil_total='pypsa-eur-sec/resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc', - solar_thermal_total='pypsa-eur-sec/resources/solar_thermal_total_{network}_s{simpl}_{clusters}.nc', + busmap_s='pypsa-eur/resources/busmap_elec_s{simpl}.csv', + busmap='pypsa-eur/resources/busmap_elec_s{simpl}_{clusters}.csv', + cop_air_total='pypsa-eur-sec/resources/cop_air_total_elec_s{simpl}_{clusters}.nc', + cop_soil_total='pypsa-eur-sec/resources/cop_soil_total_elec_s{simpl}_{clusters}.nc', + solar_thermal_total='pypsa-eur-sec/resources/solar_thermal_total_elec_s{simpl}_{clusters}.nc', energy_totals_name='pypsa-eur-sec/data/energy_totals.csv', - heat_demand_total='pypsa-eur-sec/resources/heat_demand_total_{network}_s{simpl}_{clusters}.nc', + heat_demand_total='pypsa-eur-sec/resources/heat_demand_total_elec_s{simpl}_{clusters}.nc', heat_profile='pypsa-eur-sec/data/heat_load_profile_BDEW.csv', transport_name='pypsa-eur-sec/data/transport_data.csv', - temp_air_total='pypsa-eur-sec/resources/temp_air_total_{network}_s{simpl}_{clusters}.nc', + temp_air_total='pypsa-eur-sec/resources/temp_air_total_elec_s{simpl}_{clusters}.nc', co2_totals_name='pypsa-eur-sec/data/co2_totals.csv', biomass_potentials='pypsa-eur-sec/data/biomass_potentials.csv', - industrial_demand='pypsa-eur-sec/resources/industrial_demand_{network}_s{simpl}_{clusters}.csv',), - output=['pypsa-eur-sec/results/test/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc'] + industrial_demand='pypsa-eur-sec/resources/industrial_demand_elec_s{simpl}_{clusters}.csv',), + output=['pypsa-eur-sec/results/test/prenetworks/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc'] ) import yaml with open('config.yaml', encoding='utf8') as f: diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a7078433..b2c56d54 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -369,13 +369,13 @@ if __name__ == "__main__": if 'snakemake' not in globals(): from vresutils.snakemake import MockSnakemake, Dict snakemake = MockSnakemake( - wildcards=dict(network='elec', simpl='', clusters='39', lv='1.0', + wildcards=dict(simpl='', clusters='39', lv='1.0', sector_opts='Co2L0-168H-T-H-B-I-solar3-dist1', co2_budget_name='b30b3', planning_horizons='2050'), - input=dict(network="pypsa-eur-sec/results/test/prenetworks_brownfield/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc"), + input=dict(network="pypsa-eur-sec/results/test/prenetworks_brownfield/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc"), output=["results/networks/s{simpl}_{clusters}_lv{lv}_{sector_opts}_{co2_budget_name}_{planning_horizons}-test.nc"], - log=dict(gurobi="logs/{network}_s{simpl}_{clusters}_lv{lv}_{sector_opts}_{co2_budget_name}_{planning_horizons}_gurobi-test.log", - python="logs/{network}_s{simpl}_{clusters}_lv{lv}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python-test.log") + log=dict(gurobi="logs/elec_s{simpl}_{clusters}_lv{lv}_{sector_opts}_{co2_budget_name}_{planning_horizons}_gurobi-test.log", + python="logs/elec_s{simpl}_{clusters}_lv{lv}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python-test.log") ) import yaml with open('config.yaml', encoding='utf8') as f: From 723afc32e9b292f5cc543c04c28bd2302f17176c Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 10 Nov 2020 09:51:02 +0100 Subject: [PATCH 017/173] scripts: step towards support for subset of countries --- config.default.yaml | 3 +++ config.myopic.yaml | 3 +++ scripts/build_industrial_energy_demand_per_country_today.py | 2 ++ scripts/build_industrial_production_per_country.py | 1 + 4 files changed, 9 insertions(+) diff --git a/config.default.yaml b/config.default.yaml index 7aa23901..ead33e2b 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -11,6 +11,7 @@ foresight: 'overnight' #options are overnight, myopic, perfect (perfect is not y scenario: sectors: [E] # ignore this legacy setting + year: [''] # weather year simpl: [''] # only relevant for PyPSA-Eur lv: [1.0,1.5] # allowed transmission line volume expansion, can be any float >= 1.0 (today) or "opt" clusters: [45,50] # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred @@ -34,6 +35,8 @@ snapshots: end: "2014-01-01" closed: 'left' # end is not inclusive +countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] + atlite: cutout_dir: '../pypsa-eur/cutouts' cutout_name: "europe-2013-era5" diff --git a/config.myopic.yaml b/config.myopic.yaml index c7f5ac1e..0dc4cdf3 100644 --- a/config.myopic.yaml +++ b/config.myopic.yaml @@ -11,6 +11,7 @@ foresight: 'myopic' #options are overnight, myopic, perfect (perfect is not yet scenario: sectors: [E] # ignore this legacy setting + year: [''] # weather year simpl: [''] # only relevant for PyPSA-Eur lv: [1.0,1.5] # allowed transmission line volume expansion, can be any float >= 1.0 (today) or "opt" clusters: [45,50] # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred @@ -34,6 +35,8 @@ snapshots: end: "2014-01-01" closed: 'left' # end is not inclusive +countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] + atlite: cutout_dir: '../pypsa-eur/cutouts' cutout_name: "europe-2013-era5" diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 7593477b..00611318 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -61,6 +61,8 @@ eu28 = ['FR', 'DE', 'GB', 'IT', 'ES', 'PL', 'SE', 'NL', 'BE', 'FI', 'DK', 'PT', 'RO', 'AT', 'BG', 'EE', 'GR', 'LV', 'CZ', 'HU', 'IE', 'SK', 'LT', 'HR', 'LU', 'SI', 'CY', 'MT'] +eu28 = list(set(eu28).intersection(snakemake.config["countries"])) + jrc_names = {"GR" : "EL", "GB" : "UK"} diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index c8fa6910..4a993a91 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -39,6 +39,7 @@ eu28 = ['FR', 'DE', 'GB', 'IT', 'ES', 'PL', 'SE', 'NL', 'BE', 'FI', countries = non_EU + eu28 +countries = list(set(countries).intersection(snakemake.config["countries"])) sectors = ['Iron and steel','Chemicals Industry','Non-metallic mineral products', 'Pulp, paper and printing', 'Food, beverages and tobacco', 'Non Ferrous Metals', From 59cb0361222dbd457307baf68b62d58075b46537 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 10 Nov 2020 09:51:32 +0100 Subject: [PATCH 018/173] energy_totals: only fix 'BA' if in list of countries --- scripts/build_energy_totals.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 1682ac40..0cfa709b 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -378,12 +378,12 @@ def build_energy_totals(): clean_df.loc[missing,"total aviation passenger"] = clean_df.loc[missing,["total domestic aviation passenger","total international aviation passenger"]].sum(axis=1) clean_df.loc[missing,"total aviation freight"] = clean_df.loc[missing,["total domestic aviation freight","total international aviation freight"]].sum(axis=1) + if "BA" in clean_df.index: + #fix missing data for BA (services and road energy data) + missing = (clean_df.loc["BA"] == 0.) - #fix missing data for BA (services and road energy data) - missing = (clean_df.loc["BA"] == 0.) - - #add back in proportional to RS with ratio of total residential demand - clean_df.loc["BA",missing] = clean_df.loc["BA","total residential"]/clean_df.loc["RS","total residential"]*clean_df.loc["RS",missing] + #add back in proportional to RS with ratio of total residential demand + clean_df.loc["BA",missing] = clean_df.loc["BA","total residential"]/clean_df.loc["RS","total residential"]*clean_df.loc["RS",missing] clean_df.to_csv(snakemake.output.energy_name) From df94ea1c8bce3f97d696a08ae202585a32f26dee Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 10 Nov 2020 09:52:06 +0100 Subject: [PATCH 019/173] add multiyear support --- Snakefile | 244 +++++++++--------- scripts/build_clustered_population_layouts.py | 5 +- scripts/build_heat_demand.py | 10 +- scripts/build_population_layouts.py | 6 +- scripts/build_solar_thermal_profiles.py | 9 +- scripts/build_temperature_profiles.py | 9 +- 6 files changed, 153 insertions(+), 130 deletions(-) diff --git a/Snakefile b/Snakefile index f003bd5e..2d5564a8 100644 --- a/Snakefile +++ b/Snakefile @@ -2,8 +2,8 @@ configfile: "config.yaml" wildcard_constraints: + year="[0-9]*", lv="[a-z0-9\.]+", - network="[a-zA-Z0-9]*", simpl="[a-zA-Z0-9]*", clusters="[0-9]+m?", sectors="[+a-zA-Z0-9]+", @@ -25,17 +25,17 @@ rule all: rule solve_all_networks: input: - expand(config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", + expand(config['results_dir'] + config['run'] + "/postnetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", **config['scenario']) rule test_script: input: - expand("resources/heat_demand_urban_elec_s_{clusters}.nc", + expand("resources/heat_demand_urban_elec{year}_s_{clusters}.nc", **config['scenario']) rule prepare_sector_networks: input: - expand(config['results_dir'] + config['run'] + "/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", + expand(config['results_dir'] + config['run'] + "/prenetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", **config['scenario']) @@ -44,96 +44,96 @@ rule build_population_layouts: nuts3_shapes=pypsaeur('resources/nuts3_shapes.geojson'), urban_percent="data/urban_percent.csv" output: - pop_layout_total="resources/pop_layout_total.nc", - pop_layout_urban="resources/pop_layout_urban.nc", - pop_layout_rural="resources/pop_layout_rural.nc" + pop_layout_total="resources/pop_layout{year}_total.nc", + pop_layout_urban="resources/pop_layout{year}_urban.nc", + pop_layout_rural="resources/pop_layout{year}_rural.nc" resources: mem_mb=20000 script: "scripts/build_population_layouts.py" rule build_clustered_population_layouts: input: - pop_layout_total="resources/pop_layout_total.nc", - pop_layout_urban="resources/pop_layout_urban.nc", - pop_layout_rural="resources/pop_layout_rural.nc", - regions_onshore=pypsaeur('resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson') + pop_layout_total="resources/pop_layout{year}_total.nc", + pop_layout_urban="resources/pop_layout{year}_urban.nc", + pop_layout_rural="resources/pop_layout{year}_rural.nc", + regions_onshore=pypsaeur('resources/regions_onshore_elec{year}_s{simpl}_{clusters}.geojson') output: - clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv" + clustered_pop_layout="resources/pop_layout_elec{year}_s{simpl}_{clusters}.csv" resources: mem_mb=10000 script: "scripts/build_clustered_population_layouts.py" rule build_simplified_population_layouts: input: - pop_layout_total="resources/pop_layout_total.nc", - pop_layout_urban="resources/pop_layout_urban.nc", - pop_layout_rural="resources/pop_layout_rural.nc", - regions_onshore=pypsaeur('resources/regions_onshore_{network}_s{simpl}.geojson') + pop_layout_total="resources/pop_layout{year}_total.nc", + pop_layout_urban="resources/pop_layout{year}_urban.nc", + pop_layout_rural="resources/pop_layout{year}_rural.nc", + regions_onshore=pypsaeur('resources/regions_onshore_elec{year}_s{simpl}.geojson') output: - clustered_pop_layout="resources/pop_layout_{network}_s{simpl}.csv" + clustered_pop_layout="resources/pop_layout_elec{year}_s{simpl}.csv" resources: mem_mb=10000 script: "scripts/build_clustered_population_layouts.py" rule build_heat_demands: input: - pop_layout_total="resources/pop_layout_total.nc", - pop_layout_urban="resources/pop_layout_urban.nc", - pop_layout_rural="resources/pop_layout_rural.nc", - regions_onshore=pypsaeur("resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson") + pop_layout_total="resources/pop_layout{year}_total.nc", + pop_layout_urban="resources/pop_layout{year}_urban.nc", + pop_layout_rural="resources/pop_layout{year}_rural.nc", + regions_onshore=pypsaeur("resources/regions_onshore_elec{year}_s{simpl}_{clusters}.geojson") output: - heat_demand_urban="resources/heat_demand_urban_{network}_s{simpl}_{clusters}.nc", - heat_demand_rural="resources/heat_demand_rural_{network}_s{simpl}_{clusters}.nc", - heat_demand_total="resources/heat_demand_total_{network}_s{simpl}_{clusters}.nc" + heat_demand_urban="resources/heat_demand_urban_elec{year}_s{simpl}_{clusters}.nc", + heat_demand_rural="resources/heat_demand_rural_elec{year}_s{simpl}_{clusters}.nc", + heat_demand_total="resources/heat_demand_total_elec{year}_s{simpl}_{clusters}.nc" resources: mem_mb=20000 script: "scripts/build_heat_demand.py" rule build_temperature_profiles: input: - pop_layout_total="resources/pop_layout_total.nc", - pop_layout_urban="resources/pop_layout_urban.nc", - pop_layout_rural="resources/pop_layout_rural.nc", - regions_onshore=pypsaeur("resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson") + pop_layout_total="resources/pop_layout{year}_total.nc", + pop_layout_urban="resources/pop_layout{year}_urban.nc", + pop_layout_rural="resources/pop_layout{year}_rural.nc", + regions_onshore=pypsaeur("resources/regions_onshore_elec{year}_s{simpl}_{clusters}.geojson") output: - temp_soil_total="resources/temp_soil_total_{network}_s{simpl}_{clusters}.nc", - temp_soil_rural="resources/temp_soil_rural_{network}_s{simpl}_{clusters}.nc", - temp_soil_urban="resources/temp_soil_urban_{network}_s{simpl}_{clusters}.nc", - temp_air_total="resources/temp_air_total_{network}_s{simpl}_{clusters}.nc", - temp_air_rural="resources/temp_air_rural_{network}_s{simpl}_{clusters}.nc", - temp_air_urban="resources/temp_air_urban_{network}_s{simpl}_{clusters}.nc" + temp_soil_total="resources/temp_soil_total_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_rural="resources/temp_soil_rural_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_urban="resources/temp_soil_urban_elec{year}_s{simpl}_{clusters}.nc", + temp_air_total="resources/temp_air_total_elec{year}_s{simpl}_{clusters}.nc", + temp_air_rural="resources/temp_air_rural_elec{year}_s{simpl}_{clusters}.nc", + temp_air_urban="resources/temp_air_urban_elec{year}_s{simpl}_{clusters}.nc" resources: mem_mb=20000 script: "scripts/build_temperature_profiles.py" rule build_cop_profiles: input: - temp_soil_total="resources/temp_soil_total_{network}_s{simpl}_{clusters}.nc", - temp_soil_rural="resources/temp_soil_rural_{network}_s{simpl}_{clusters}.nc", - temp_soil_urban="resources/temp_soil_urban_{network}_s{simpl}_{clusters}.nc", - temp_air_total="resources/temp_air_total_{network}_s{simpl}_{clusters}.nc", - temp_air_rural="resources/temp_air_rural_{network}_s{simpl}_{clusters}.nc", - temp_air_urban="resources/temp_air_urban_{network}_s{simpl}_{clusters}.nc" + temp_soil_total="resources/temp_soil_total_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_rural="resources/temp_soil_rural_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_urban="resources/temp_soil_urban_elec{year}_s{simpl}_{clusters}.nc", + temp_air_total="resources/temp_air_total_elec{year}_s{simpl}_{clusters}.nc", + temp_air_rural="resources/temp_air_rural_elec{year}_s{simpl}_{clusters}.nc", + temp_air_urban="resources/temp_air_urban_elec{year}_s{simpl}_{clusters}.nc" output: - cop_soil_total="resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc", - cop_soil_rural="resources/cop_soil_rural_{network}_s{simpl}_{clusters}.nc", - cop_soil_urban="resources/cop_soil_urban_{network}_s{simpl}_{clusters}.nc", - cop_air_total="resources/cop_air_total_{network}_s{simpl}_{clusters}.nc", - cop_air_rural="resources/cop_air_rural_{network}_s{simpl}_{clusters}.nc", - cop_air_urban="resources/cop_air_urban_{network}_s{simpl}_{clusters}.nc" + cop_soil_total="resources/cop_soil_total_elec{year}_s{simpl}_{clusters}.nc", + cop_soil_rural="resources/cop_soil_rural_elec{year}_s{simpl}_{clusters}.nc", + cop_soil_urban="resources/cop_soil_urban_elec{year}_s{simpl}_{clusters}.nc", + cop_air_total="resources/cop_air_total_elec{year}_s{simpl}_{clusters}.nc", + cop_air_rural="resources/cop_air_rural_elec{year}_s{simpl}_{clusters}.nc", + cop_air_urban="resources/cop_air_urban_elec{year}_s{simpl}_{clusters}.nc" resources: mem_mb=20000 script: "scripts/build_cop_profiles.py" rule build_solar_thermal_profiles: input: - pop_layout_total="resources/pop_layout_total.nc", - pop_layout_urban="resources/pop_layout_urban.nc", - pop_layout_rural="resources/pop_layout_rural.nc", - regions_onshore=pypsaeur("resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson") + pop_layout_total="resources/pop_layout{year}_total.nc", + pop_layout_urban="resources/pop_layout{year}_urban.nc", + pop_layout_rural="resources/pop_layout{year}_rural.nc", + regions_onshore=pypsaeur("resources/regions_onshore_elec{year}_s{simpl}_{clusters}.geojson") output: - solar_thermal_total="resources/solar_thermal_total_{network}_s{simpl}_{clusters}.nc", - solar_thermal_urban="resources/solar_thermal_urban_{network}_s{simpl}_{clusters}.nc", - solar_thermal_rural="resources/solar_thermal_rural_{network}_s{simpl}_{clusters}.nc" + solar_thermal_total="resources/solar_thermal_total_elec{year}_s{simpl}_{clusters}.nc", + solar_thermal_urban="resources/solar_thermal_urban_elec{year}_s{simpl}_{clusters}.nc", + solar_thermal_rural="resources/solar_thermal_rural_elec{year}_s{simpl}_{clusters}.nc" resources: mem_mb=20000 script: "scripts/build_solar_thermal_profiles.py" @@ -204,12 +204,12 @@ rule build_industrial_production_per_country_tomorrow: rule build_industrial_distribution_key: input: - clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv", + clustered_pop_layout="resources/pop_layout_elec{year}_s{simpl}_{clusters}.csv", europe_shape=pypsaeur('resources/europe_shape.geojson'), hotmaps_industrial_database="data/Industrial_Database.csv", - network=pypsaeur('networks/{network}_s{simpl}_{clusters}.nc') + network=pypsaeur('networks/elec{year}_s{simpl}_{clusters}.nc') output: - industrial_distribution_key="resources/industrial_distribution_key_{network}_s{simpl}_{clusters}.csv" + industrial_distribution_key="resources/industrial_distribution_key_elec{year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=1000 script: 'scripts/build_industrial_distribution_key.py' @@ -218,10 +218,10 @@ rule build_industrial_distribution_key: rule build_industrial_production_per_node: input: - industrial_distribution_key="resources/industrial_distribution_key_{network}_s{simpl}_{clusters}.csv", + industrial_distribution_key="resources/industrial_distribution_key_elec{year}_s{simpl}_{clusters}.csv", industrial_production_per_country_tomorrow="resources/industrial_production_per_country_tomorrow.csv" output: - industrial_production_per_node="resources/industrial_production_{network}_s{simpl}_{clusters}.csv" + industrial_production_per_node="resources/industrial_production_elec{year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=1000 script: 'scripts/build_industrial_production_per_node.py' @@ -230,10 +230,10 @@ rule build_industrial_production_per_node: rule build_industrial_energy_demand_per_node: input: industry_sector_ratios="resources/industry_sector_ratios.csv", - industrial_production_per_node="resources/industrial_production_{network}_s{simpl}_{clusters}.csv", - industrial_energy_demand_per_node_today="resources/industrial_energy_demand_today_{network}_s{simpl}_{clusters}.csv" + industrial_production_per_node="resources/industrial_production_elec{year}_s{simpl}_{clusters}.csv", + industrial_energy_demand_per_node_today="resources/industrial_energy_demand_today_elec{year}_s{simpl}_{clusters}.csv" output: - industrial_energy_demand_per_node="resources/industrial_energy_demand_{network}_s{simpl}_{clusters}.csv" + industrial_energy_demand_per_node="resources/industrial_energy_demand_elec{year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=1000 script: 'scripts/build_industrial_energy_demand_per_node.py' @@ -252,10 +252,10 @@ rule build_industrial_energy_demand_per_country_today: rule build_industrial_energy_demand_per_node_today: input: - industrial_distribution_key="resources/industrial_distribution_key_{network}_s{simpl}_{clusters}.csv", + industrial_distribution_key="resources/industrial_distribution_key_elec{year}_s{simpl}_{clusters}.csv", industrial_energy_demand_per_country_today="resources/industrial_energy_demand_per_country_today.csv" output: - industrial_energy_demand_per_node_today="resources/industrial_energy_demand_today_{network}_s{simpl}_{clusters}.csv" + industrial_energy_demand_per_node_today="resources/industrial_energy_demand_today_elec{year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=1000 script: 'scripts/build_industrial_energy_demand_per_node_today.py' @@ -275,10 +275,10 @@ rule build_industrial_energy_demand_per_country: rule build_industrial_demand: input: - clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv", + clustered_pop_layout="resources/pop_layout_elec{year}_s{simpl}_{clusters}.csv", industrial_demand_per_country="resources/industrial_energy_demand_per_country.csv" output: - industrial_demand="resources/industrial_demand_{network}_s{simpl}_{clusters}.csv" + industrial_demand="resources/industrial_demand_elec{year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=1000 script: 'scripts/build_industrial_demand.py' @@ -286,7 +286,7 @@ rule build_industrial_demand: rule prepare_sector_network: input: - network=pypsaeur('networks/{network}_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc'), + network=pypsaeur('networks/elec{year}_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc'), energy_totals_name='resources/energy_totals.csv', co2_totals_name='resources/co2_totals.csv', transport_name='resources/transport_data.csv', @@ -296,45 +296,45 @@ rule prepare_sector_network: costs=config['costs_dir'] + "costs_{planning_horizons}.csv", h2_cavern = "data/hydrogen_salt_cavern_potentials.csv", co2_budget="data/co2_budget.csv", - profile_offwind_ac=pypsaeur("resources/profile_offwind-ac.nc"), - profile_offwind_dc=pypsaeur("resources/profile_offwind-dc.nc"), - busmap_s=pypsaeur("resources/busmap_{network}_s{simpl}.csv"), - busmap=pypsaeur("resources/busmap_{network}_s{simpl}_{clusters}.csv"), - clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv", - simplified_pop_layout="resources/pop_layout_{network}_s{simpl}.csv", - industrial_demand="resources/industrial_energy_demand_{network}_s{simpl}_{clusters}.csv", - heat_demand_urban="resources/heat_demand_urban_{network}_s{simpl}_{clusters}.nc", - heat_demand_rural="resources/heat_demand_rural_{network}_s{simpl}_{clusters}.nc", - heat_demand_total="resources/heat_demand_total_{network}_s{simpl}_{clusters}.nc", - temp_soil_total="resources/temp_soil_total_{network}_s{simpl}_{clusters}.nc", - temp_soil_rural="resources/temp_soil_rural_{network}_s{simpl}_{clusters}.nc", - temp_soil_urban="resources/temp_soil_urban_{network}_s{simpl}_{clusters}.nc", - temp_air_total="resources/temp_air_total_{network}_s{simpl}_{clusters}.nc", - temp_air_rural="resources/temp_air_rural_{network}_s{simpl}_{clusters}.nc", - temp_air_urban="resources/temp_air_urban_{network}_s{simpl}_{clusters}.nc", - cop_soil_total="resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc", - cop_soil_rural="resources/cop_soil_rural_{network}_s{simpl}_{clusters}.nc", - cop_soil_urban="resources/cop_soil_urban_{network}_s{simpl}_{clusters}.nc", - cop_air_total="resources/cop_air_total_{network}_s{simpl}_{clusters}.nc", - cop_air_rural="resources/cop_air_rural_{network}_s{simpl}_{clusters}.nc", - cop_air_urban="resources/cop_air_urban_{network}_s{simpl}_{clusters}.nc", - solar_thermal_total="resources/solar_thermal_total_{network}_s{simpl}_{clusters}.nc", - solar_thermal_urban="resources/solar_thermal_urban_{network}_s{simpl}_{clusters}.nc", - solar_thermal_rural="resources/solar_thermal_rural_{network}_s{simpl}_{clusters}.nc" - output: config['results_dir'] + config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc' + profile_offwind_ac=pypsaeur("resources/profile{year}_offwind-ac.nc"), + profile_offwind_dc=pypsaeur("resources/profile{year}_offwind-dc.nc"), + busmap_s=pypsaeur("resources/busmap_elec{year}_s{simpl}.csv"), + busmap=pypsaeur("resources/busmap_elec{year}_s{simpl}_{clusters}.csv"), + clustered_pop_layout="resources/pop_layout_elec{year}_s{simpl}_{clusters}.csv", + simplified_pop_layout="resources/pop_layout_elec{year}_s{simpl}.csv", + industrial_demand="resources/industrial_energy_demand_elec{year}_s{simpl}_{clusters}.csv", + heat_demand_urban="resources/heat_demand_urban_elec{year}_s{simpl}_{clusters}.nc", + heat_demand_rural="resources/heat_demand_rural_elec{year}_s{simpl}_{clusters}.nc", + heat_demand_total="resources/heat_demand_total_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_total="resources/temp_soil_total_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_rural="resources/temp_soil_rural_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_urban="resources/temp_soil_urban_elec{year}_s{simpl}_{clusters}.nc", + temp_air_total="resources/temp_air_total_elec{year}_s{simpl}_{clusters}.nc", + temp_air_rural="resources/temp_air_rural_elec{year}_s{simpl}_{clusters}.nc", + temp_air_urban="resources/temp_air_urban_elec{year}_s{simpl}_{clusters}.nc", + cop_soil_total="resources/cop_soil_total_elec{year}_s{simpl}_{clusters}.nc", + cop_soil_rural="resources/cop_soil_rural_elec{year}_s{simpl}_{clusters}.nc", + cop_soil_urban="resources/cop_soil_urban_elec{year}_s{simpl}_{clusters}.nc", + cop_air_total="resources/cop_air_total_elec{year}_s{simpl}_{clusters}.nc", + cop_air_rural="resources/cop_air_rural_elec{year}_s{simpl}_{clusters}.nc", + cop_air_urban="resources/cop_air_urban_elec{year}_s{simpl}_{clusters}.nc", + solar_thermal_total="resources/solar_thermal_total_elec{year}_s{simpl}_{clusters}.nc", + solar_thermal_urban="resources/solar_thermal_urban_elec{year}_s{simpl}_{clusters}.nc", + solar_thermal_rural="resources/solar_thermal_rural_elec{year}_s{simpl}_{clusters}.nc" + output: config['results_dir'] + config['run'] + '/prenetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc' threads: 1 resources: mem_mb=2000 - benchmark: config['results_dir'] + config['run'] + "/benchmarks/prepare_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}" + benchmark: config['results_dir'] + config['run'] + "/benchmarks/prepare_network/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}" script: "scripts/prepare_sector_network.py" rule plot_network: input: - network=config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" + network=config['results_dir'] + config['run'] + "/postnetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" output: - map=config['results_dir'] + config['run'] + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{co2_budget_name}_{planning_horizons}.pdf", - today=config['results_dir'] + config['run'] + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}-today.pdf" + map=config['results_dir'] + config['run'] + "/maps/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{co2_budget_name}_{planning_horizons}.pdf", + today=config['results_dir'] + config['run'] + "/maps/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}-today.pdf" threads: 2 resources: mem_mb=10000 script: "scripts/plot_network.py" @@ -351,10 +351,10 @@ rule copy_config: rule make_summary: input: - networks=expand(config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", + networks=expand(config['results_dir'] + config['run'] + "/postnetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", **config['scenario']), costs=config['costs_dir'] + "costs_{}.csv".format(config['scenario']['planning_horizons'][0]), - plots=expand(config['results_dir'] + config['run'] + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{co2_budget_name}_{planning_horizons}.pdf", + plots=expand(config['results_dir'] + config['run'] + "/maps/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{co2_budget_name}_{planning_horizons}.pdf", **config['scenario']) #heat_demand_name='data/heating/daily_heat_demand.h5' output: @@ -397,16 +397,16 @@ if config["foresight"] == "overnight": rule solve_network: input: - network=config['results_dir'] + config['run'] + "/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", + network=config['results_dir'] + config['run'] + "/prenetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", costs=config['costs_dir'] + "costs_{planning_horizons}.csv", config=config['summary_dir'] + '/' + config['run'] + '/configs/config.yaml' - output: config['results_dir'] + config['run'] + "/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" + output: config['results_dir'] + config['run'] + "/postnetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" shadow: "shallow" log: - solver=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_solver.log", - python=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python.log", - memory=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_memory.log" - benchmark: config['results_dir'] + config['run'] + "/benchmarks/solve_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}" + solver=config['results_dir'] + config['run'] + "/logs/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_solver.log", + python=config['results_dir'] + config['run'] + "/logs/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python.log", + memory=config['results_dir'] + config['run'] + "/logs/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_memory.log" + benchmark: config['results_dir'] + config['run'] + "/benchmarks/solve_network/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}" threads: 4 resources: mem_mb=config['solving']['mem'] # group: "solve" # with group, threads is ignored https://bitbucket.org/snakemake/snakemake/issues/971/group-job-description-does-not-contain @@ -417,15 +417,15 @@ if config["foresight"] == "myopic": rule add_existing_baseyear: input: - network=config['results_dir'] + config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', + network=config['results_dir'] + config['run'] + '/prenetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', powerplants=pypsaeur('resources/powerplants.csv'), - busmap_s=pypsaeur("resources/busmap_{network}_s{simpl}.csv"), - busmap=pypsaeur("resources/busmap_{network}_s{simpl}_{clusters}.csv"), - clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv", + busmap_s=pypsaeur("resources/busmap_elec{year}_s{simpl}.csv"), + busmap=pypsaeur("resources/busmap_elec{year}_s{simpl}_{clusters}.csv"), + clustered_pop_layout="resources/pop_layout_elec{year}_s{simpl}_{clusters}.csv", costs=config['costs_dir'] + "costs_{}.csv".format(config['scenario']['planning_horizons'][0]), - cop_soil_total="resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc", - cop_air_total="resources/cop_air_total_{network}_s{simpl}_{clusters}.nc" - output: config['results_dir'] + config['run'] + '/prenetworks-brownfield/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc' + cop_soil_total="resources/cop_soil_total_elec{year}_s{simpl}_{clusters}.nc", + cop_air_total="resources/cop_air_total_elec{year}_s{simpl}_{clusters}.nc" + output: config['results_dir'] + config['run'] + '/prenetworks-brownfield/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc' wildcard_constraints: planning_horizons=config['scenario']['planning_horizons'][0] #only applies to baseyear threads: 1 @@ -434,18 +434,18 @@ if config["foresight"] == "myopic": def process_input(wildcards): i = config["scenario"]["planning_horizons"].index(int(wildcards.planning_horizons)) - return config['results_dir'] + config['run'] + "/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_" + str(config["scenario"]["planning_horizons"][i-1]) + ".nc" + return config['results_dir'] + config['run'] + "/postnetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_" + str(config["scenario"]["planning_horizons"][i-1]) + ".nc" rule add_brownfield: input: - network=config['results_dir'] + config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', + network=config['results_dir'] + config['run'] + '/prenetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', network_p=process_input, #solved network at previous time step costs=config['costs_dir'] + "costs_{planning_horizons}.csv", - cop_soil_total="resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc", - cop_air_total="resources/cop_air_total_{network}_s{simpl}_{clusters}.nc" + cop_soil_total="resources/cop_soil_total_elec{year}_s{simpl}_{clusters}.nc", + cop_air_total="resources/cop_air_total_elec{year}_s{simpl}_{clusters}.nc" - output: config['results_dir'] + config['run'] + "/prenetworks-brownfield/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" + output: config['results_dir'] + config['run'] + "/prenetworks-brownfield/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" threads: 4 resources: mem_mb=2000 script: "scripts/add_brownfield.py" @@ -454,16 +454,16 @@ if config["foresight"] == "myopic": rule solve_network_myopic: input: - network=config['results_dir'] + config['run'] + "/prenetworks-brownfield/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", + network=config['results_dir'] + config['run'] + "/prenetworks-brownfield/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", costs=config['costs_dir'] + "costs_{planning_horizons}.csv", config=config['summary_dir'] + '/' + config['run'] + '/configs/config.yaml' - output: config['results_dir'] + config['run'] + "/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" + output: config['results_dir'] + config['run'] + "/postnetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" shadow: "shallow" log: - solver=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_solver.log", - python=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python.log", - memory=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_memory.log" - benchmark: config['results_dir'] + config['run'] + "/benchmarks/solve_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}" + solver=config['results_dir'] + config['run'] + "/logs/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_solver.log", + python=config['results_dir'] + config['run'] + "/logs/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python.log", + memory=config['results_dir'] + config['run'] + "/logs/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_memory.log" + benchmark: config['results_dir'] + config['run'] + "/benchmarks/solve_network/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}" threads: 4 resources: mem_mb=config['solving']['mem'] script: "scripts/solve_network.py" diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index 94cbd133..85543afc 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -5,8 +5,11 @@ import pandas as pd import atlite import helper +year = snakemake.wildcards.year +cutout_name = snakemake.config['atlite']['cutout_name'] +if year: cutout_name = cutout_name.format(year=year) -cutout = atlite.Cutout(snakemake.config['atlite']['cutout_name'], +cutout = atlite.Cutout(cutout_name, cutout_dir=snakemake.config['atlite']['cutout_dir']) diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 865f12bd..0e6cdbd7 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -15,10 +15,16 @@ if 'snakemake' not in globals(): snakemake.input = Dict() snakemake.output = Dict() -time = pd.date_range(freq='m', **snakemake.config['snapshots']) +year = snakemake.wildcards.year + +snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] +time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) -cutout = atlite.Cutout(snakemake.config['atlite']['cutout_name'], +cutout_name = snakemake.config['atlite']['cutout_name'] +if year: cutout_name = cutout_name.format(year=year) + +cutout = atlite.Cutout(cutout_name, cutout_dir=snakemake.config['atlite']['cutout_dir'], **params) diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index 5093e1b4..15804576 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -21,7 +21,11 @@ if 'snakemake' not in globals(): snakemake.input["urban_percent"] = "data/urban_percent.csv" -cutout = atlite.Cutout(snakemake.config['atlite']['cutout_name'], +year = snakemake.wildcards.year +cutout_name = snakemake.config['atlite']['cutout_name'] +if year: cutout_name = cutout_name.format(year=year) + +cutout = atlite.Cutout(cutout_name, cutout_dir=snakemake.config['atlite']['cutout_dir']) grid_cells = cutout.grid_cells() diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index c26266aa..7d4199d5 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -15,12 +15,17 @@ if 'snakemake' not in globals(): snakemake.input = Dict() snakemake.output = Dict() -time = pd.date_range(freq='m', **snakemake.config['snapshots']) +year = snakemake.wildcards.year + +snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] +time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) +cutout_name = snakemake.config['atlite']['cutout_name'] +if year: cutout_name = cutout_name.format(year=year) -cutout = atlite.Cutout(snakemake.config['atlite']['cutout_name'], +cutout = atlite.Cutout(cutout_name, cutout_dir=snakemake.config['atlite']['cutout_dir'], **params) diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index a55bd606..2cad501c 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -15,11 +15,16 @@ if 'snakemake' not in globals(): snakemake.input = Dict() snakemake.output = Dict() -time = pd.date_range(freq='m', **snakemake.config['snapshots']) +year = snakemake.wildcards.year + +snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] +time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) +cutout_name = snakemake.config['atlite']['cutout_name'] +if year: cutout_name = cutout_name.format(year=year) -cutout = atlite.Cutout(snakemake.config['atlite']['cutout_name'], +cutout = atlite.Cutout(cutout_name, cutout_dir=snakemake.config['atlite']['cutout_dir'], **params) From ce03353ac3fbd66c95156e1cd241baf01681f89d Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 18 Feb 2022 10:27:10 +0100 Subject: [PATCH 020/173] change year to weather_year --- config.default.yaml | 2 +- config.tutorial.yaml | 2 +- test/config.test1.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 2c2e151f..9c986494 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -12,7 +12,7 @@ logging: summary_dir: results scenario: - year: [''] # for backwards compatibility + weather_year: [''] # for backwards compatibility simpl: [''] ll: ['copt'] clusters: [37, 128, 256, 512, 1024] diff --git a/config.tutorial.yaml b/config.tutorial.yaml index feee6366..6ff1237d 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -12,7 +12,7 @@ logging: summary_dir: results scenario: - year: [''] + weather_year: [''] simpl: [''] ll: ['copt'] clusters: [5] diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 9de59e7c..c73e59fb 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -11,7 +11,7 @@ logging: summary_dir: results scenario: - year: [''] + weather_year: [''] simpl: [''] ll: ['copt'] clusters: [5] From f26cdc132b9bf20c31fdabe3a815c828415cce5c Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 18 Feb 2022 11:21:05 +0100 Subject: [PATCH 021/173] add wildcard constraint for weather year --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index b4b5b98f..fe6242a1 100644 --- a/Snakefile +++ b/Snakefile @@ -18,7 +18,7 @@ ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4) wildcard_constraints: - year="[0-9]*", + weather_year="[0-9]*", simpl="[a-zA-Z0-9]*|all", clusters="[0-9]+m?|all", ll="(v|c)([0-9\.]+|opt|all)|all", From 8d1047380a406d0724e964cc8cdec7f0030e89e6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 18 Feb 2022 11:27:07 +0100 Subject: [PATCH 022/173] remove myopic config file --- config.myopic.yaml | 346 --------------------------------------------- 1 file changed, 346 deletions(-) delete mode 100644 config.myopic.yaml diff --git a/config.myopic.yaml b/config.myopic.yaml deleted file mode 100644 index 0dc4cdf3..00000000 --- a/config.myopic.yaml +++ /dev/null @@ -1,346 +0,0 @@ -version: 0.3.0 - -logging_level: INFO - -results_dir: 'results/' -summary_dir: results -costs_dir: '../technology-data/outputs/' -run: 'your-run-name' # use this to keep track of runs with different settings -foresight: 'myopic' #options are overnight, myopic, perfect (perfect is not yet implemented) - - -scenario: - sectors: [E] # ignore this legacy setting - year: [''] # weather year - simpl: [''] # only relevant for PyPSA-Eur - lv: [1.0,1.5] # allowed transmission line volume expansion, can be any float >= 1.0 (today) or "opt" - clusters: [45,50] # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred - opts: [''] # only relevant for PyPSA-Eur - sector_opts: [Co2L0-3H-H-B-solar3-dist1] # this is where the main scenario settings are - # to really understand the options here, look in scripts/prepare_sector_network.py - # Co2Lx specifies the CO2 target in x% of the 1990 values; default will give default (5%); - # Co2L0p25 will give 25% CO2 emissions; Co2Lm0p05 will give 5% negative emissions - # xH is the temporal resolution; 3H is 3-hourly, i.e. one snapshot every 3 hours - # single letters are sectors: T for land transport, H for building heating, - # B for biomass supply, I for industry, shipping and aviation - # solarx or onwindx changes the available installable potential by factor x - # dist{n} includes distribution grids with investment cost of n times cost in data/costs.csv - planning_horizons : [2020, 2030, 2040, 2050] #investment years for myopic and perfect; or costs year for overnight - co2_budget_name: ['go'] #gives shape of CO2 budgets over planning horizon - -# snapshots are originally set in PyPSA-Eur/config.yaml but used again by PyPSA-Eur-Sec -snapshots: - # arguments to pd.date_range - start: "2013-01-01" - end: "2014-01-01" - closed: 'left' # end is not inclusive - -countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] - -atlite: - cutout_dir: '../pypsa-eur/cutouts' - cutout_name: "europe-2013-era5" - -# this information is NOT used but needed as an argument for -# pypsa-eur/scripts/add_electricity.py/load_costs in make_summary.py -electricity: - max_hours: - battery: 6 - H2: 168 - -biomass: - year: 2030 - scenario: "Med" - classes: - solid biomass: ['Primary agricultural residues', 'Forestry energy residue', 'Secondary forestry residues', 'Secondary Forestry residues – sawdust', 'Forestry residues from landscape care biomass', 'Municipal waste'] - not included: ['Bioethanol sugar beet biomass', 'Rapeseeds for biodiesel', 'sunflower and soya for Biodiesel', 'Starchy crops biomass', 'Grassy crops biomass', 'Willow biomass', 'Poplar biomass potential', 'Roundwood fuelwood', 'Roundwood Chips & Pellets'] - biogas: ['Manure biomass potential', 'Sludge biomass'] - -# only relevant for foresight = myopic or perfect -existing_capacities: - grouping_years: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] - threshold_capacity: 10 - conventional_carriers: ['lignite', 'coal', 'oil', 'uranium'] - -sector: - 'central' : True - 'central_fraction' : 0.6 - 'dsm_restriction_value' : 0.75 #Set to 0 for no restriction on BEV DSM - 'dsm_restriction_time' : 7 #Time at which SOC of BEV has to be dsm_restriction_value - 'transport_heating_deadband_upper' : 20. - 'transport_heating_deadband_lower' : 15. - 'ICE_lower_degree_factor' : 0.375 #in per cent increase in fuel consumption per degree above deadband - 'ICE_upper_degree_factor' : 1.6 - 'EV_lower_degree_factor' : 0.98 - 'EV_upper_degree_factor' : 0.63 - 'district_heating_loss' : 0.15 - 'bev' : True #turns on EV battery - 'bev_availability' : 0.5 #How many cars do smart charging - 'v2g' : True #allows feed-in to grid from EV battery - 'transport_fuel_cell_share' : 0. #0 means all EVs, 1 means all FCs - 'shipping_average_efficiency' : 0.4 #For conversion of fuel oil to propulsion in 2011 - 'time_dep_hp_cop' : True - 'space_heating_fraction' : 1.0 #fraction of space heating active - 'retrofitting' : False - 'retroI-fraction' : 0.25 - 'retroII-fraction' : 0.55 - 'retrofitting-cost_factor' : 1.0 - 'tes' : True - 'tes_tau' : 3. - 'boilers' : True - 'oil_boilers': False - 'chp' : True - 'micro_chp' : False - 'solar_thermal' : True - 'solar_cf_correction': 0.788457 # = >>> 1/1.2683 - 'marginal_cost_storage' : 0. #1e-4 - 'methanation' : True - 'helmeth' : True - 'dac' : True - 'co2_vent' : True - 'SMR' : True - 'ccs_fraction' : 0.9 - 'hydrogen_underground_storage' : True - 'use_fischer_tropsch_waste_heat' : True - 'use_fuel_cell_waste_heat' : True - 'electricity_distribution_grid' : False - 'electricity_distribution_grid_cost_factor' : 1.0 #multiplies cost in data/costs.csv - 'electricity_grid_connection' : True # only applies to onshore wind and utility PV - 'gas_distribution_grid' : True - 'gas_distribution_grid_cost_factor' : 1.0 #multiplies cost in data/costs.csv - -costs: - year: 2030 - lifetime: 25 #default lifetime - # From a Lion Hirth paper, also reflects average of Noothout et al 2016 - discountrate: 0.07 - # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html # noqa: E501 - USD2013_to_EUR2013: 0.7532 - - # Marginal and capital costs can be overwritten - # capital_cost: - # Wind: Bla - marginal_cost: # - solar: 0.01 - onwind: 0.015 - offwind: 0.015 - hydro: 0. - H2: 0. - battery: 0. - - emission_prices: # only used with the option Ep (emission prices) - co2: 0. - - lines: - length_factor: 1.25 #to estimate offwind connection costs - - -solving: - #tmpdir: "path/to/tmp" - options: - formulation: kirchhoff - clip_p_max_pu: 1.e-2 - load_shedding: false - noisy_costs: true - - min_iterations: 1 - max_iterations: 1 - # nhours: 1 - - solver: - name: gurobi - threads: 4 - method: 2 # barrier - crossover: 0 - BarConvTol: 1.e-5 - Seed: 123 - AggFill: 0 - PreDual: 0 - GURO_PAR_BARDENSETHRESH: 200 - #FeasibilityTol: 1.e-6 - - #name: cplex - #threads: 4 - #lpmethod: 4 # barrier - #solutiontype: 2 # non basic solution, ie no crossover - #barrier_convergetol: 1.e-5 - #feasopt_tolerance: 1.e-6 - mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 - -industry: - 'St_primary_fraction' : 0.3 # fraction of steel produced via primary route (DRI + EAF) versus secondary route (EAF); today fraction is 0.6 - 'H2_DRI' : 1.7 #H2 consumption in Direct Reduced Iron (DRI), MWh_H2,LHV/ton_Steel from Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279 - 'elec_DRI' : 0.322 #electricity consumption in Direct Reduced Iron (DRI) shaft, MWh/tSt HYBRIT brochure https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf - 'Al_primary_fraction' : 0.2 # fraction of aluminium produced via the primary route versus scrap; today fraction is 0.4 - 'MWh_CH4_per_tNH3_SMR' : 10.8 # 2012's demand from https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf - 'MWh_elec_per_tNH3_SMR' : 0.7 # same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3 - 'MWh_H2_per_tNH3_electrolysis' : 6.5 # from https://doi.org/10.1016/j.joule.2018.04.017, around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy) - 'MWh_elec_per_tNH3_electrolysis' : 1.17 # from https://doi.org/10.1016/j.joule.2018.04.017 Table 13 (air separation and HB) - 'NH3_process_emissions' : 24.5 # in MtCO2/a from SMR for H2 production for NH3 from UNFCCC for 2015 for EU28 - 'petrochemical_process_emissions' : 25.5 # in MtCO2/a for petrochemical and other from UNFCCC for 2015 for EU28 - 'HVC_primary_fraction' : 1.0 #fraction of current non-ammonia basic chemicals produced via primary route - -plotting: - map: - figsize: [7, 7] - boundaries: [-10.2, 29, 35, 72] - p_nom: - bus_size_factor: 5.e+4 - linewidth_factor: 3.e+3 # 1.e+3 #3.e+3 - - costs_max: 1200 - costs_threshold: 1 - - - energy_max: 20000. - energy_min: -15000. - energy_threshold: 50. - - - vre_techs: ["onwind", "offwind-ac", "offwind-dc", "solar", "ror"] - renewable_storage_techs: ["PHS","hydro"] - conv_techs: ["OCGT", "CCGT", "Nuclear", "Coal"] - storage_techs: ["hydro+PHS", "battery", "H2"] - # store_techs: ["Li ion", "water tanks"] - load_carriers: ["AC load"] #, "heat load", "Li ion load"] - AC_carriers: ["AC line", "AC transformer"] - link_carriers: ["DC line", "Converter AC-DC"] - heat_links: ["heat pump", "resistive heater", "CHP heat", "CHP electric", - "gas boiler", "central heat pump", "central resistive heater", "central CHP heat", - "central CHP electric", "central gas boiler"] - heat_generators: ["gas boiler", "central gas boiler", "solar thermal collector", "central solar thermal collector"] - tech_colors: - "onwind" : "b" - "onshore wind" : "b" - 'offwind' : "c" - 'offshore wind' : "c" - 'offwind-ac' : "c" - 'offshore wind (AC)' : "c" - 'offwind-dc' : "#009999" - 'offshore wind (DC)' : "#009999" - 'wave' : "#004444" - "hydro" : "#3B5323" - "hydro reservoir" : "#3B5323" - "ror" : "#78AB46" - "run of river" : "#78AB46" - 'hydroelectricity' : '#006400' - 'solar' : "y" - 'solar PV' : "y" - 'solar thermal' : 'coral' - 'solar rooftop' : '#e6b800' - "OCGT" : "wheat" - "OCGT marginal" : "sandybrown" - "OCGT-heat" : "orange" - "gas boiler" : "orange" - "gas boilers" : "orange" - "gas boiler marginal" : "orange" - "gas-to-power/heat" : "orange" - "gas" : "brown" - "natural gas" : "brown" - "SMR" : "#4F4F2F" - "oil" : "#B5A642" - "oil boiler" : "#B5A677" - "lines" : "k" - "transmission lines" : "k" - "H2" : "m" - "hydrogen storage" : "m" - "battery" : "slategray" - "battery storage" : "slategray" - "home battery" : "#614700" - "home battery storage" : "#614700" - "Nuclear" : "r" - "Nuclear marginal" : "r" - "nuclear" : "r" - "uranium" : "r" - "Coal" : "k" - "coal" : "k" - "Coal marginal" : "k" - "Lignite" : "grey" - "lignite" : "grey" - "Lignite marginal" : "grey" - "CCGT" : "orange" - "CCGT marginal" : "orange" - "heat pumps" : "#76EE00" - "heat pump" : "#76EE00" - "air heat pump" : "#76EE00" - "ground heat pump" : "#40AA00" - "power-to-heat" : "#40AA00" - "resistive heater" : "pink" - "Sabatier" : "#FF1493" - "methanation" : "#FF1493" - "power-to-gas" : "#FF1493" - "power-to-liquid" : "#FFAAE9" - "helmeth" : "#7D0552" - "helmeth" : "#7D0552" - "DAC" : "#E74C3C" - "co2 stored" : "#123456" - "CO2 sequestration" : "#123456" - "CCS" : "k" - "co2" : "#123456" - "co2 vent" : "#654321" - "solid biomass for industry co2 from atmosphere" : "#654321" - "solid biomass for industry co2 to stored": "#654321" - "gas for industry co2 to atmosphere": "#654321" - "gas for industry co2 to stored": "#654321" - "Fischer-Tropsch" : "#44DD33" - "kerosene for aviation": "#44BB11" - "naphtha for industry" : "#44FF55" - "water tanks" : "#BBBBBB" - "hot water storage" : "#BBBBBB" - "hot water charging" : "#BBBBBB" - "hot water discharging" : "#999999" - "CHP" : "r" - "CHP heat" : "r" - "CHP electric" : "r" - "PHS" : "g" - "Ambient" : "k" - "Electric load" : "b" - "Heat load" : "r" - "Transport load" : "grey" - "heat" : "darkred" - "rural heat" : "#880000" - "central heat" : "#b22222" - "decentral heat" : "#800000" - "low-temperature heat for industry" : "#991111" - "process heat" : "#FF3333" - "heat demand" : "darkred" - "electric demand" : "k" - "Li ion" : "grey" - "district heating" : "#CC4E5C" - "retrofitting" : "purple" - "building retrofitting" : "purple" - "BEV charger" : "grey" - "V2G" : "grey" - "transport" : "grey" - "electricity" : "k" - "gas for industry" : "#333333" - "solid biomass for industry" : "#555555" - "industry electricity" : "#222222" - "industry new electricity" : "#222222" - "process emissions to stored" : "#444444" - "process emissions to atmosphere" : "#888888" - "process emissions" : "#222222" - "transport fuel cell" : "#AAAAAA" - "biogas" : "#800000" - "solid biomass" : "#DAA520" - "today" : "#D2691E" - "shipping" : "#6495ED" - "electricity distribution grid" : "#333333" - nice_names: - # OCGT: "Gas" - # OCGT marginal: "Gas (marginal)" - offwind: "offshore wind" - onwind: "onshore wind" - battery: "Battery storage" - lines: "Transmission lines" - AC line: "AC lines" - AC-AC: "DC lines" - ror: "Run of river" - nice_names_n: - offwind: "offshore\nwind" - onwind: "onshore\nwind" - # OCGT: "Gas" - H2: "Hydrogen\nstorage" - # OCGT marginal: "Gas (marginal)" - lines: "transmission\nlines" - ror: "run of river" From 4397f1fd0c52cc4454b231720d92cfdfeb29c1f3 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 18 Feb 2022 11:32:06 +0100 Subject: [PATCH 023/173] minor fixes --- Snakefile | 2 +- config.default.yaml | 2 -- scripts/build_population_layouts.py | 5 ++++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Snakefile b/Snakefile index c0c1c7fa..4efb15f1 100644 --- a/Snakefile +++ b/Snakefile @@ -12,7 +12,7 @@ configfile: "config.yaml" wildcard_constraints: - year="[0-9]*", + weather_year="[0-9]*", lv="[a-z0-9\.]+", simpl="[a-zA-Z0-9]*", clusters="[0-9]+m?", diff --git a/config.default.yaml b/config.default.yaml index 010c31d3..63dcab51 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -65,8 +65,6 @@ snapshots: end: "2014-01-01" closed: left # end is not inclusive -countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] - atlite: cutout: ../pypsa-eur/cutouts/europe-2013-era5.nc diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index e83dd5f3..a5ae2ebf 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -12,7 +12,10 @@ from vresutils import shapes as vshapes if __name__ == '__main__': if 'snakemake' not in globals(): from helper import mock_snakemake - snakemake = mock_snakemake('build_population_layouts', year='') + snakemake = mock_snakemake( + 'build_population_layouts', + weather_year='', + ) year = snakemake.wildcards.weather_year cutout_config = snakemake.config['atlite']['cutout'] From cd2078849a45aca70eb43257df57e1fbda99709d Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 18 Feb 2022 11:33:30 +0100 Subject: [PATCH 024/173] remove misplaced code snippet --- scripts/build_clustered_population_layouts.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index e9ef7398..7a8de88b 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -5,10 +5,6 @@ import xarray as xr import pandas as pd import atlite -year = snakemake.wildcards.weather_year -cutout_name = snakemake.config['atlite']['cutout_name'] -if year: cutout_name = cutout_name.format(year=year) - if __name__ == '__main__': if 'snakemake' not in globals(): from helper import mock_snakemake From a71552eb1b1ecf20eb4c2cb54b5b25b51a6968d0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 09:30:52 +0200 Subject: [PATCH 025/173] fix parameter correcitons --- data/parameter_corrections.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/data/parameter_corrections.yaml b/data/parameter_corrections.yaml index d16d5d24..b50fc03a 100644 --- a/data/parameter_corrections.yaml +++ b/data/parameter_corrections.yaml @@ -38,22 +38,6 @@ Link: "T23": "6355" # bus0 == bus1 to remove link in remove_unconnected_components (NordBalt) bus1: index: -<<<<<<< HEAD - "11511": "6240" # fix wrong bus allocation from 6241 - "14559": "6240" # fix wrong bus allocation from 6241 - "12998": "1333" # combine link 12998 + 12997 in 12998 - "5627": '2309' # combine link 5627 + 5628 in 5627 - "8068": "5819" # fix GB location of Anglo-Scottish interconnector - length: - index: - "12998": 409.0 - "5627": 26.39 - bus0: - index: - "14552": "5819" # fix GB location of GB-IE interconnector - "5628": "7276" # bus0 == bus1 to remove link in remove_unconnected_components - "12997": "7276" # bus0 == bus1 to remove link in remove_unconnected_components -======= "12931": "8152" # BorWin3 "5582": "2382" # combine link 5583 + 5582 in 5582 (Sardinia) "13589": "1349" # combine link 13589 + 13588 in 13589 (Sardinia) @@ -72,4 +56,3 @@ Line: v_nom: index: "14573": 220 # 220/380kV issue of substation in PT ->>>>>>> master From 6d260435a633d18e049715b741ae1e0e4a4e7c65 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 13:49:37 +0200 Subject: [PATCH 026/173] allow fixed load data year deviating from snapshots --- Snakefile | 4 ++-- config.default.yaml | 2 +- scripts/build_load_data.py | 20 ++++++++++++++++++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Snakefile b/Snakefile index 518f5619..94ad7926 100644 --- a/Snakefile +++ b/Snakefile @@ -77,8 +77,8 @@ rule retrieve_load_data: rule build_load_data: input: "data/load_raw.csv" - output: "resources/load.csv" - log: "logs/build_load_data.log" + output: "resources/load{weather_year}.csv" + log: "logs/build_load_data{weather_year}.log" script: 'scripts/build_load_data.py' diff --git a/config.default.yaml b/config.default.yaml index 9c986494..c1dbd8b7 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -196,7 +196,7 @@ load: time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from manual_adjustments: true # false scaling_factor: 1.0 - fallback_year: 2013 + fixed_year: false # false or year costs: year: 2030 diff --git a/scripts/build_load_data.py b/scripts/build_load_data.py index 10921782..0cad8ddf 100755 --- a/scripts/build_load_data.py +++ b/scripts/build_load_data.py @@ -196,11 +196,23 @@ if __name__ == "__main__": configure_logging(snakemake) + weather_year = snakemake.wildcard.weather_year + if weather_year: + snapshots = dict( + start=weather_year, + end=str(int(weather_year)+1), + closed="left" + ) + else: + snapshots = snakemake.config['snapshots'] + snapshots = pd.date_range(freq='h', **snapshots) + + fixed_year = snakemake.config["load"].get("fixed_year", False) + years = slice(fixed_year, fixed_year) if fixed_year else slice(snapshots[0], snapshots[-1]) + powerstatistics = snakemake.config['load']['power_statistics'] interpolate_limit = snakemake.config['load']['interpolate_limit'] countries = snakemake.config['countries'] - snapshots = pd.date_range(freq='h', **snakemake.config['snapshots']) - years = slice(snapshots[0], snapshots[-1]) time_shift = snakemake.config['load']['time_shift_for_large_gaps'] load = load_timeseries(snakemake.input[0], years, countries, powerstatistics) @@ -220,5 +232,9 @@ if __name__ == "__main__": '`time_shift_for_large_gaps` or modify the `manual_adjustment` function ' 'for implementing the needed load data modifications.') + # need to reindex load time series to target year + if fixed_year: + load.index = load.index.map(lambda t: t.replace(year=snapshots.year[0])) + load.to_csv(snakemake.output[0]) From 2b43f43006a026bd6ee70233a9844b5c5013533f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 13:51:08 +0200 Subject: [PATCH 027/173] add more versatile cutout building with weather years --- Snakefile | 13 ++++++++++++- scripts/build_cutout.py | 16 +++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Snakefile b/Snakefile index 94ad7926..9a65d5df 100644 --- a/Snakefile +++ b/Snakefile @@ -18,7 +18,7 @@ ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4) wildcard_constraints: - weather_year="[0-9]*", + weather_year="[0-9]{4}|$", simpl="[a-zA-Z0-9]*|all", clusters="[0-9]+m?|all", ll="(v|c)([0-9\.]+|opt|all)|all", @@ -148,6 +148,9 @@ rule build_bus_regions: script: "scripts/build_bus_regions.py" if config['enable'].get('build_cutout', False): + + ruleorder: build_cutout_year > build_cutout + rule build_cutout: input: regions_onshore="resources/regions_onshore.geojson", @@ -159,6 +162,14 @@ if config['enable'].get('build_cutout', False): resources: mem_mb=ATLITE_NPROCESSES * 1000 script: "scripts/build_cutout.py" + rule build_cutout_year: + input: rules.build_cutout.input + output: "cutouts/{cutout}-{weather_year}.nc" + log: "logs/build_cutout/{cutout}-{weather_year}.log" + benchmark: "benchmarks/build_cutout_{cutout}-{weather_year}" + threads: ATLITE_NPROCESSES + resources: mem_mb=ATLITE_NPROCESSES * 1000 + script: "scripts/build_cutout.py" if config['enable'].get('retrieve_cutout', True): rule retrieve_cutout: diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 2722a62b..a5681b00 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -103,15 +103,21 @@ logger = logging.getLogger(__name__) if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('build_cutout', cutout='europe-2013-era5') + # snakemake = mock_snakemake('build_cutout_year', cutout='europe-era5', weather_year=2011) + snakemake = mock_snakemake('build_cutout', cutout='europe-era5') configure_logging(snakemake) - # TODO make it accept year independent config cutout_params = snakemake.config['atlite']['cutouts'][snakemake.wildcards.cutout] - snapshots = pd.date_range(freq='h', **snakemake.config['snapshots']) - time = [snapshots[0], snapshots[-1]] - cutout_params['time'] = slice(*cutout_params.get('time', time)) + if hasattr(snakemake.wildcards, 'weather_year'): + time = snakemake.wildcards.weather_year + cutout_params["time"] = [time, time] + + if "time" not in cutout_params: + snapshots = pd.date_range(freq='h', **snakemake.config['snapshots']) + cutout_params["time"] = [snapshots[0], snapshots[-1]] + + cutout_params['time'] = slice(*cutout_params['time']) if {'x', 'y', 'bounds'}.isdisjoint(cutout_params): # Determine the bounds from bus regions with a buffer of two grid cells From 4a44c45ace3732c3723d13d3dceabdd31c97237a Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 13:51:56 +0200 Subject: [PATCH 028/173] add option to cut Feb 29 in leap years --- config.default.yaml | 1 + scripts/add_electricity.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/config.default.yaml b/config.default.yaml index c1dbd8b7..9e7841e8 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -37,6 +37,7 @@ enable: build_natura_raster: false retrieve_natura_raster: true custom_busmap: false + drop_leap_days: true electricity: voltages: [220., 300., 380.] diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index bd6a5e46..a035f437 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -625,4 +625,8 @@ if __name__ == "__main__": add_nice_carrier_names(n, snakemake.config) + if snakemake.config['enable'].get('drop_leap_days', True): + leap_days = (n.snapshots.day == 29) & (n.snapshots.month == 2) + n.set_snapshots(n.snapshots[~leap_days]) + n.export_to_netcdf(snakemake.output[0]) From 91ca07d72f66b2796ef948928883db269505119d Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 14:00:47 +0200 Subject: [PATCH 029/173] add option to specify deviating norm year for hydro --- config.default.yaml | 1 + scripts/build_hydro_profile.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 9e7841e8..010631e4 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -169,6 +169,7 @@ renewable: PHS_max_hours: 6 hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float clip_min_inflow: 1.0 + norm_year: 2013 lines: types: diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 70c8380e..fdd028e9 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -84,8 +84,12 @@ if __name__ == "__main__": eia_stats = vhydro.get_eia_annual_hydro_generation(snakemake.input.eia_hydro_generation).reindex(columns=countries) - if len(year) > 0 and year not in eia_stats.index: - eia_stats.loc[year] = eia_stats.mean() + weather_year = snakemake.wildcards.weather_year + norm_year = snakemake.config['renewable']['hydro'].get('norm_year') + if norm_year: + eia_stats.loc[weather_year] = eia_stats.loc[norm_year] + elif weather_year and weather_year not in eia_stats.index: + eia_stats.loc[weather_year] = eia_stats.median() inflow = cutout.runoff(shapes=country_shapes, smooth=True, From ff2de7066f477f8af13f8eea29906d038ed0d8f4 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 14:01:27 +0200 Subject: [PATCH 030/173] remove upstream deleted build_country_flh --- scripts/build_country_flh.py | 245 ----------------------------------- 1 file changed, 245 deletions(-) delete mode 100644 scripts/build_country_flh.py diff --git a/scripts/build_country_flh.py b/scripts/build_country_flh.py deleted file mode 100644 index 0b65f136..00000000 --- a/scripts/build_country_flh.py +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env python - -# SPDX-FileCopyrightText: : 2017-2020 The PyPSA-Eur Authors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -""" -Create ``.csv`` files and plots for comparing per country full load hours of renewable time series. - -Relevant Settings ------------------ - -.. code:: yaml - - snapshots: - - renewable: - {technology}: - cutout: - resource: - correction_factor: - -.. seealso:: - Documentation of the configuration file ``config.yaml`` at - :ref:`snapshots_cf`, :ref:`renewable_cf` - -Inputs ------- - -- ``data/bundle/corine/g250_clc06_V18_5.tif``: `CORINE Land Cover (CLC) `_ inventory on `44 classes `_ of land use (e.g. forests, arable land, industrial, urban areas). - - .. image:: img/corine.png - :scale: 33 % - -- ``data/bundle/GEBCO_2014_2D.nc``: A `bathymetric `_ data set with a global terrain model for ocean and land at 15 arc-second intervals by the `General Bathymetric Chart of the Oceans (GEBCO) `_. - - .. image:: img/gebco_2019_grid_image.jpg - :scale: 50 % - - **Source:** `GEBCO `_ - -- ``data/pietzker2014.xlsx``: `Supplementary material 2 `_ from `Pietzcker et al. `_; not part of the data bundle; download and place here yourself. -- ``resources/natura.tiff``: confer :ref:`natura` -- ``resources/country_shapes.geojson``: confer :ref:`shapes` -- ``resources/offshore_shapes.geojson``: confer :ref:`shapes` -- ``resources/regions_onshore.geojson``: (if not offshore wind), confer :ref:`busregions` -- ``resources/regions_offshore.geojson``: (if offshore wind), :ref:`busregions` -- ``"cutouts/" + config["renewable"][{technology}]['cutout']``: :ref:`cutout` -- ``networks/base.nc``: :ref:`base` - -Outputs -------- - -- ``resources/country_flh_area_{technology}.csv``: -- ``resources/country_flh_aggregated_{technology}.csv``: -- ``resources/country_flh_uncorrected_{technology}.csv``: -- ``resources/country_flh_{technology}.pdf``: -- ``resources/country_exclusion_{technology}``: - -Description ------------ - -""" - -import logging -from _helpers import configure_logging - -import os -import atlite -import numpy as np -import xarray as xr -import pandas as pd - -import geokit as gk -from scipy.sparse import vstack -import pycountry as pyc -import matplotlib.pyplot as plt - -from vresutils import landuse as vlanduse -from vresutils.array import spdiag - -import progressbar as pgb - -from build_renewable_profiles import init_globals, calculate_potential - -logger = logging.getLogger(__name__) - - -def build_area(flh, countries, areamatrix, breaks, fn): - area_unbinned = xr.DataArray(areamatrix.todense(), [countries, capacity_factor.coords['spatial']]) - bins = xr.DataArray(pd.cut(flh.to_series(), bins=breaks), flh.coords, name="bins") - area = area_unbinned.groupby(bins).sum(dim="spatial").to_pandas() - area.loc[:,slice(*area.sum()[lambda s: s > 0].index[[0,-1]])].to_csv(fn) - area.columns = area.columns.map(lambda s: s.left) - return area - - -def plot_area_not_solar(area, countries): - # onshore wind/offshore wind - a = area.T - - fig, axes = plt.subplots(nrows=len(countries), sharex=True) - for c, ax in zip(countries, axes): - d = a[[c]] / 1e3 - d.plot.bar(ax=ax, legend=False, align='edge', width=1.) - ax.set_ylabel(f"Potential {c} / GW") - ax.set_title(c) - ax.legend() - ax.set_xlabel("Full-load hours") - fig.savefig(snakemake.output.plot, transparent=True, bbox_inches='tight') - -def plot_area_solar(area, p_area, countries): - # onshore wind/offshore wind - p = p_area.T - a = area.T - - fig, axes = plt.subplots(nrows=len(countries), sharex=True, squeeze=False) - for c, ax in zip(countries, axes.flat): - d = pd.concat([a[c], p[c]], keys=['PyPSA-Eur', 'Pietzker'], axis=1) / 1e3 - d.plot.bar(ax=ax, legend=False, align='edge', width=1.) - # ax.set_ylabel(f"Potential {c} / GW") - ax.set_title(c) - ax.legend() - ax.set_xlabel("Full-load hours") - - fig.savefig(snakemake.output.plot, transparent=True, bbox_inches='tight') - - -def build_aggregate(flh, countries, areamatrix, breaks, p_area, fn): - agg_a = pd.Series(np.ravel((areamatrix / areamatrix.sum(axis=1)).dot(flh.values)), - countries, name="PyPSA-Eur") - - if p_area is None: - agg_a['Overall'] = float((np.asarray((areamatrix.sum(axis=0) / areamatrix.sum()) - .dot(flh.values)).squeeze())) - - agg = pd.DataFrame({'PyPSA-Eur': agg_a}) - else: - # Determine indices of countries which are also in Pietzcker - inds = pd.Index(countries).get_indexer(p_area.index) - areamatrix = areamatrix[inds] - - agg_a['Overall'] = float((np.asarray((areamatrix.sum(axis=0) / areamatrix.sum()) - .dot(flh.values)).squeeze())) - - midpoints = (breaks[1:] + breaks[:-1])/2. - p = p_area.T - - # Per-country FLH comparison - agg_p = pd.Series((p / p.sum()).multiply(midpoints, axis=0).sum(), name="Pietzker") - agg_p['Overall'] = float((p.sum(axis=1) / p.sum().sum()).multiply(midpoints, axis=0).sum()) - - agg = pd.DataFrame({'PyPSA-Eur': agg_a, 'Pietzcker': agg_p, 'Ratio': agg_p / agg_a}) - - agg.to_csv(fn) - -if __name__ == '__main__': - if 'snakemake' not in globals(): - from _helpers import mock_snakemake - snakemake = mock_snakemake('build_country_flh', technology='solar') - configure_logging(snakemake) - - pgb.streams.wrap_stderr() - - - config = snakemake.config['renewable'][snakemake.wildcards.technology] - - year = snakemake.wildcards.year - snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] - time = pd.date_range(freq='m', **snapshots) - params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) - - cutout = atlite.Cutout(config['cutout'], - cutout_dir=os.path.dirname(snakemake.input.cutout), - **params) - - minx, maxx, miny, maxy = cutout.extent - dx = (maxx - minx) / (cutout.shape[1] - 1) - dy = (maxy - miny) / (cutout.shape[0] - 1) - bounds = gk.Extent.from_xXyY((minx - dx/2., maxx + dx/2., - miny - dy/2., maxy + dy/2.)) - - # Use GLAES to compute available potentials and the transition matrix - paths = dict(snakemake.input) - - init_globals(bounds.xXyY, dx, dy, config, paths) - regions = gk.vector.extractFeatures(paths["regions"], onlyAttr=True) - countries = pd.Index(regions["name"], name="country") - - widgets = [ - pgb.widgets.Percentage(), - ' ', pgb.widgets.SimpleProgress(format='(%s)' % pgb.widgets.SimpleProgress.DEFAULT_FORMAT), - ' ', pgb.widgets.Bar(), - ' ', pgb.widgets.Timer(), - ' ', pgb.widgets.ETA() - ] - progressbar = pgb.ProgressBar(prefix='Compute GIS potentials: ', widgets=widgets, max_value=len(countries)) - - if not os.path.isdir(snakemake.output.exclusion): - os.makedirs(snakemake.output.exclusion) - - matrix = vstack([calculate_potential(i, save_map=os.path.join(snakemake.output.exclusion, countries[i])) - for i in progressbar(regions.index)]) - - areamatrix = matrix * spdiag(vlanduse._cutout_cell_areas(cutout).ravel()) - areamatrix.data[areamatrix.data < 1.] = 0 # ignore weather cells where only less than 1 km^2 can be installed - areamatrix.eliminate_zeros() - - resource = config['resource'] - func = getattr(cutout, resource.pop('method')) - correction_factor = config.get('correction_factor', 1.) - - capacity_factor = func(capacity_factor=True, show_progress='Compute capacity factors: ', **resource).stack(spatial=('y', 'x')) - flh_uncorr = capacity_factor * 8760 - flh_corr = correction_factor * flh_uncorr - - if snakemake.wildcards.technology == 'solar': - pietzcker = pd.read_excel(snakemake.input.pietzker, sheet_name="PV on all area", skiprows=2, header=[0,1]).iloc[1:177] - p_area1_50 = pietzcker['Usable Area at given FLh in 1-50km distance to settlement '].dropna(axis=1) - p_area1_50.columns = p_area1_50.columns.str.split(' ').str[0] - - p_area50_100 = pietzcker['Usable Area at given FLh in 50-100km distance to settlement '] - - p_area = p_area1_50 + p_area50_100 - cols = p_area.columns - breaks = cols.str.split('-').str[0].append(pd.Index([cols[-1].split('-')[1]])).astype(int) - p_area.columns = breaks[:-1] - - p_area = p_area.reindex(countries.map(lambda c: pyc.countries.get(alpha_2=c).name)) - p_area.index = countries - p_area = p_area.dropna() # Pietzcker does not have data for CZ and MK - else: - breaks = np.r_[0:8000:50] - p_area = None - - - area = build_area(flh_corr, countries, areamatrix, breaks, snakemake.output.area) - - if snakemake.wildcards.technology == 'solar': - plot_area_solar(area, p_area, p_area.index) - else: - plot_area_not_solar(area, countries) - - build_aggregate(flh_uncorr, countries, areamatrix, breaks, p_area, snakemake.output.uncorrected) - build_aggregate(flh_corr, countries, areamatrix, breaks, p_area, snakemake.output.aggregated) From e9b3cb7558b42c3e70e8518e3447820aa716dbae Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 14:02:19 +0200 Subject: [PATCH 031/173] add_electricity: remove unused load data handling --- scripts/add_electricity.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index a035f437..76aa7fa1 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -207,37 +207,11 @@ def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.): regions = (gpd.read_file(regions).set_index('name') .reindex(substation_lv_i)) -# ------------- TO MERGE -------------- - available_years = range(2011,2016) - requested_years = n.snapshots.year[[0,-1]] - use_fallback = any(year not in available_years for year in requested_years) - - if use_fallback: - fallback_year = str(snakemake.config["load"]["fallback_year"]) - load_years = [fallback_year, fallback_year] - logger.warning(f"Requested years {list(requested_years.unique().values)} " - f"for load time series not in available years {list(available_years)}. " - f"Falling back to year {fallback_year}.") - else: - load_years = requested_years.astype(str) - - opsd_load = (timeseries_opsd(slice(*load_years), - snakemake.input.opsd_load) * - snakemake.config.get('load', {}).get('scaling_factor', 1.0)) - - if use_fallback: - assert len(requested_years.unique()) == 1, "Fallback for load time series requires single year!" - opsd_load.index = opsd_load.index.map(lambda t: t.replace(year=requested_years[0])) - - # Convert to naive UTC (has to be explicit since pandas 0.24) - opsd_load.index = opsd_load.index.tz_localize(None) - opsd_load = (pd.read_csv(load, index_col=0, parse_dates=True) .filter(items=countries)) logger.info(f"Load data scaled with scalling factor {scaling}.") opsd_load *= scaling -# ------------- TO MERGE -------------- nuts3 = gpd.read_file(nuts3_shapes).set_index('index') From 500890b6af5c1068a9f514d430723a42e33ddf17 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 14:02:40 +0200 Subject: [PATCH 032/173] Snakefile: remove duplicate use of weather year wildcard --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 9a65d5df..8e138228 100644 --- a/Snakefile +++ b/Snakefile @@ -398,7 +398,7 @@ rule make_summary: rule plot_summary: input: "results/summaries/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}" - output: "results/plots/{weather_year}/summary_{summary}_elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}" + output: "results/plots/summary_{summary}_elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}" log: "logs/plot_summary/{summary}_elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log" script: "scripts/plot_summary.py" From 5cc6e9e18a71098d3a4c20978a641fca13c71c4f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 14:07:48 +0200 Subject: [PATCH 033/173] doc: add reference to new config settings --- doc/configtables/hydro.csv | 1 + doc/configtables/load.csv | 1 + doc/configtables/toplevel.csv | 1 + 3 files changed, 3 insertions(+) diff --git a/doc/configtables/hydro.csv b/doc/configtables/hydro.csv index fc53334e..8148b256 100644 --- a/doc/configtables/hydro.csv +++ b/doc/configtables/hydro.csv @@ -4,3 +4,4 @@ carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydr PHS_max_hours,h,float,"Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_." hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}","Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation `_." clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero." +norm_year,--,"Year in EIA hydro generation dataset; or False to disable","To specify a specific year by which hydro inflow is normed that deviates from the snapshots' year" \ No newline at end of file diff --git a/doc/configtables/load.csv b/doc/configtables/load.csv index d6cfa686..6162940a 100644 --- a/doc/configtables/load.csv +++ b/doc/configtables/load.csv @@ -5,3 +5,4 @@ interpolate_limit,hours,integer,"Maximum gap size (consecutive nans) which inter time_shift_for_large_gaps,string,string,"Periods which are used for copying time-slices in order to fill large gaps of nans. Have to be valid ``pandas`` period strings." manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`." scaling_factor,--,float,"Global correction factor for the load time series." +fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year" \ No newline at end of file diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv index 0270c0bd..842f72fd 100644 --- a/doc/configtables/toplevel.csv +++ b/doc/configtables/toplevel.csv @@ -15,3 +15,4 @@ enable,,, -- build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`." -- retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`." -- custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec{weather_year}_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec{weather_year}_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``." +-- drop_leap_days,bool,"{true, false}","Switch to drop February 29 from all time-dependent data in leap years" \ No newline at end of file From 95b01735c83049abad8090ddd320b148b4132fa5 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 14:14:36 +0200 Subject: [PATCH 034/173] Snakefile: fix regex for weather year to allow empty --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 8e138228..1c161bb6 100644 --- a/Snakefile +++ b/Snakefile @@ -18,7 +18,7 @@ ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4) wildcard_constraints: - weather_year="[0-9]{4}|$", + weather_year="[0-9]{4}|", simpl="[a-zA-Z0-9]*|all", clusters="[0-9]+m?|all", ll="(v|c)([0-9\.]+|opt|all)|all", From 0b27e407104b0deb0fed8609a4270ac722dcc639 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 14:15:00 +0200 Subject: [PATCH 035/173] Snakefile: load data is weather-year dependent --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 1c161bb6..ee824eda 100644 --- a/Snakefile +++ b/Snakefile @@ -240,7 +240,7 @@ rule add_electricity: powerplants='resources/powerplants.csv', hydro_capacities='data/bundle/hydro_capacities.csv', geth_hydro_capacities='data/geth2015_hydro_capacities.csv', - load='resources/load.csv', + load='resources/load{weather_year}.csv', nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": "resources/profile{weather_year}_" + f"{tech}.nc" for tech in config['renewable']} From 8055e45b6807160f957b41cb212619e672e378b6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 14:56:46 +0200 Subject: [PATCH 036/173] combine time selection with weather year --- Snakefile | 7 ++++- config.default.yaml | 3 ++- scripts/build_clustered_population_layouts.py | 6 ++--- scripts/build_heat_demand.py | 27 +++++++++---------- scripts/build_population_layouts.py | 6 ++--- scripts/build_solar_thermal_profiles.py | 27 +++++++++---------- scripts/build_temperature_profiles.py | 18 +++++++++---- 7 files changed, 53 insertions(+), 41 deletions(-) diff --git a/Snakefile b/Snakefile index e2c1a881..0ca4e7c8 100644 --- a/Snakefile +++ b/Snakefile @@ -12,7 +12,7 @@ configfile: "config.yaml" wildcard_constraints: - weather_year="[0-9]*", + weather_year="[0-9]{4}|", lv="[a-z0-9\.]+", simpl="[a-zA-Z0-9]*", clusters="[0-9]+m?", @@ -68,6 +68,7 @@ if config.get('retrieve_sector_databundle', True): rule build_population_layouts: input: + cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), nuts3_shapes=pypsaeur('resources/nuts3_shapes.geojson'), urban_percent="data/urban_percent.csv" output: @@ -82,6 +83,7 @@ rule build_population_layouts: rule build_clustered_population_layouts: input: + cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -163,6 +165,7 @@ else: rule build_heat_demands: input: + cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -178,6 +181,7 @@ rule build_heat_demands: rule build_temperature_profiles: input: + cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -216,6 +220,7 @@ rule build_cop_profiles: rule build_solar_thermal_profiles: input: + cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", diff --git a/config.default.yaml b/config.default.yaml index 3ed14371..7f137515 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -66,7 +66,8 @@ snapshots: closed: left # end is not inclusive atlite: - cutout: ../pypsa-eur/cutouts/europe-2013-era5.nc + cutout: europe-era5-{weather_year} + drop_leap_day: false # this information is NOT used but needed as an argument for # pypsa-eur/scripts/add_electricity.py/load_costs in make_summary.py diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index 7a8de88b..ca50db51 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -15,10 +15,10 @@ if __name__ == '__main__': clusters=48, ) + cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - cutout_config = snakemake.config['atlite']['cutout'] - if year: cutout_name = cutout_config.format(weather_year=year) - cutout = atlite.Cutout(cutout_config) + if year: cutout_name = cutout_name.format(weather_year=year) + cutout = atlite.Cutout(cutout_name) clustered_regions = gpd.read_file( snakemake.input.regions_onshore).set_index('name').buffer(0).squeeze() diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 1640379c..23eeae27 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -16,22 +16,21 @@ if __name__ == '__main__': clusters=48, ) - if 'snakemake' not in globals(): - from vresutils import Dict - import yaml - snakemake = Dict() - with open('config.yaml') as f: - snakemake.config = yaml.safe_load(f) - snakemake.input = Dict() - snakemake.output = Dict() - + cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] - time = pd.date_range(freq='m', **snapshots) + drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) - cutout_config = snakemake.config['atlite']['cutout'] - if year: cutout_name = cutout_config.format(weather_year=year) - cutout = atlite.Cutout(cutout_config).sel(time=time) + if year: + snapshots = dict(start=year, end=str(int(year)+1), closed="left") + cutout_name = cutout_name.format(weather_year=year) + else: + snapshots = snakemake.config['snapshots'] + + time = pd.date_range(freq='m', **snapshots) + if drop_leap_day: + time = time[~((time.month == 2) & (time.day == 29))] + + cutout = atlite.Cutout(cutout_name).sel(time=time) clustered_regions = gpd.read_file( snakemake.input.regions_onshore).set_index('name').buffer(0).squeeze() diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index e6dc767d..01727777 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -17,10 +17,10 @@ if __name__ == '__main__': weather_year='', ) + cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - cutout_config = snakemake.config['atlite']['cutout'] - if year: cutout_name = cutout_config.format(weather_year=year) - cutout = atlite.Cutout(cutout_config) + if year: cutout_name = cutout_name.format(weather_year=year) + cutout = atlite.Cutout(cutout_name) grid_cells = cutout.grid_cells() diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index 6ae522eb..e87c9a1e 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -16,24 +16,23 @@ if __name__ == '__main__': clusters=48, ) - if 'snakemake' not in globals(): - from vresutils import Dict - import yaml - snakemake = Dict() - with open('config.yaml') as f: - snakemake.config = yaml.safe_load(f) - snakemake.input = Dict() - snakemake.output = Dict() - config = snakemake.config['solar_thermal'] + cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] - time = pd.date_range(freq='m', **snapshots) + drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) - cutout_config = snakemake.config['atlite']['cutout'] - if year: cutout_name = cutout_config.format(weather_year=year) - cutout = atlite.Cutout(cutout_config).sel(time=time) + if year: + snapshots = dict(start=year, end=str(int(year)+1), closed="left") + cutout_name = cutout_name.format(weather_year=year) + else: + snapshots = snakemake.config['snapshots'] + + time = pd.date_range(freq='m', **snapshots) + if drop_leap_day: + time = time[~((time.month == 2) & (time.day == 29))] + + cutout = atlite.Cutout(cutout_name).sel(time=time) clustered_regions = gpd.read_file( snakemake.input.regions_onshore).set_index('name').buffer(0).squeeze() diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index 75fc9ca1..02cae24c 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -16,13 +16,21 @@ if __name__ == '__main__': clusters=48, ) + cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] - time = pd.date_range(freq='m', **snapshots) + drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) - cutout_config = snakemake.config['atlite']['cutout'] - if year: cutout_name = cutout_config.format(weather_year=year) - cutout = atlite.Cutout(cutout_config).sel(time=time) + if year: + snapshots = dict(start=year, end=str(int(year)+1), closed="left") + cutout_name = cutout_name.format(weather_year=year) + else: + snapshots = snakemake.config['snapshots'] + + time = pd.date_range(freq='m', **snapshots) + if drop_leap_day: + time = time[~((time.month == 2) & (time.day == 29))] + + cutout = atlite.Cutout(cutout_name).sel(time=time) clustered_regions = gpd.read_file( snakemake.input.regions_onshore).set_index('name').buffer(0).squeeze() From ec4a45b46bcbc5943266f9fd0e7b2f7f60ebc081 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 15:03:17 +0200 Subject: [PATCH 037/173] add weather_year wildcard to mock_snakemake objects --- scripts/add_electricity.py | 2 +- scripts/add_extra_components.py | 2 +- scripts/build_hydro_profile.py | 2 +- scripts/build_load_data.py | 4 ++-- scripts/build_renewable_profiles.py | 2 +- scripts/cluster_network.py | 2 +- scripts/make_summary.py | 4 ++-- scripts/plot_network.py | 4 ++-- scripts/plot_p_nom_max.py | 4 ++-- scripts/plot_summary.py | 4 ++-- scripts/prepare_links_p_nom.py | 2 +- scripts/prepare_network.py | 2 +- scripts/simplify_network.py | 2 +- scripts/solve_network.py | 2 +- scripts/solve_operations_network.py | 2 +- 15 files changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 76aa7fa1..f092c563 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -550,7 +550,7 @@ def add_nice_carrier_names(n, config): if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('add_electricity') + snakemake = mock_snakemake('add_electricity', weather_year='') configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 9ab413d3..44f94ff7 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -189,7 +189,7 @@ def attach_hydrogen_pipelines(n, costs, elec_opts): if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('add_extra_components', network='elec', + snakemake = mock_snakemake('add_extra_components', weather_year='', simpl='', clusters=5) configure_logging(snakemake) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index fdd028e9..a6676228 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -71,7 +71,7 @@ logger = logging.getLogger(__name__) if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('build_hydro_profile') + snakemake = mock_snakemake('build_hydro_profile', weather_year='') configure_logging(snakemake) config_hydro = snakemake.config['renewable']['hydro'] diff --git a/scripts/build_load_data.py b/scripts/build_load_data.py index 0cad8ddf..e1a20b14 100755 --- a/scripts/build_load_data.py +++ b/scripts/build_load_data.py @@ -192,11 +192,11 @@ if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('build_load_data') + snakemake = mock_snakemake('build_load_data', weather_year='') configure_logging(snakemake) - weather_year = snakemake.wildcard.weather_year + weather_year = snakemake.wildcards.weather_year if weather_year: snapshots = dict( start=weather_year, diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 70cadab4..87db7d0e 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -198,7 +198,7 @@ logger = logging.getLogger(__name__) if __name__ == '__main__': if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('build_renewable_profiles', technology='solar') + snakemake = mock_snakemake('build_renewable_profiles', technology='solar', weather_year='') configure_logging(snakemake) pgb.streams.wrap_stderr() diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index ea24c356..d4da9833 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -329,7 +329,7 @@ def plot_busmap_for_n_clusters(n, n_clusters, fn=None): if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('cluster_network', network='elec', simpl='', clusters='5') + snakemake = mock_snakemake('cluster_network', simpl='', clusters='5', weather_year='') configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index c070d33f..8e0debb1 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -426,8 +426,8 @@ def to_csv(dfs, dir): if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('make_summary', network='elec', simpl='', - clusters='5', ll='copt', opts='Co2L-24H', country='all') + snakemake = mock_snakemake('make_summary', simpl='', + clusters='5', ll='copt', opts='Co2L-24H', country='all', weather_year='') network_dir = os.path.join('..', 'results', 'networks') else: network_dir = os.path.join('results', 'networks') diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 71a6e627..25a81a22 100755 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -251,9 +251,9 @@ def plot_total_cost_bar(n, opts, ax=None): if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('plot_network', network='elec', simpl='', + snakemake = mock_snakemake('plot_network', simpl='', clusters='5', ll='copt', opts='Co2L-24H', - attr='p_nom', ext="pdf") + attr='p_nom', ext="pdf", weather_year='') configure_logging(snakemake) set_plot_style() diff --git a/scripts/plot_p_nom_max.py b/scripts/plot_p_nom_max.py index e79ad274..2a8b1bbd 100644 --- a/scripts/plot_p_nom_max.py +++ b/scripts/plot_p_nom_max.py @@ -48,9 +48,9 @@ def cum_p_nom_max(net, tech, country=None): if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('plot_p_nom_max', network='elec', simpl='', + snakemake = mock_snakemake('plot_p_nom_max', simpl='', techs='solar,onwind,offwind-dc', ext='png', - clusts= '5,full', country= 'all') + clusts= '5,full', country= 'all', weather_year='') configure_logging(snakemake) plot_kwds = dict(drawstyle="steps-post") diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index bc2bd30c..6c7f3926 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -165,9 +165,9 @@ def plot_energy(infn, config, fn=None): if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('plot_summary', summary='energy', network='elec', + snakemake = mock_snakemake('plot_summary', summary='energy', simpl='', clusters=5, ll='copt', opts='Co2L-24H', - attr='', ext='png', country='all') + attr='', ext='png', country='all', weather_year='') configure_logging(snakemake) config = snakemake.config diff --git a/scripts/prepare_links_p_nom.py b/scripts/prepare_links_p_nom.py index b83089d6..417e04d7 100644 --- a/scripts/prepare_links_p_nom.py +++ b/scripts/prepare_links_p_nom.py @@ -60,7 +60,7 @@ def extract_coordinates(s): if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake #rule must be enabled in config - snakemake = mock_snakemake('prepare_links_p_nom', simpl='', network='elec') + snakemake = mock_snakemake('prepare_links_p_nom', simpl='', weather_year='') configure_logging(snakemake) links_p_nom = pd.read_html('https://en.wikipedia.org/wiki/List_of_HVDC_projects', header=0, match="SwePol")[0] diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index ffe48fd0..6f982986 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -202,7 +202,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('prepare_network', network='elec', simpl='', + snakemake = mock_snakemake('prepare_network', weather_year='', simpl='', clusters='40', ll='v0.3', opts='Co2L-24H') configure_logging(snakemake) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 8c3bdad3..b5c1a1d8 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -384,7 +384,7 @@ def cluster(n, n_clusters, config): if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('simplify_network', simpl='', network='elec') + snakemake = mock_snakemake('simplify_network', weather_year='', simpl='') configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 39f2119e..ad7da9c5 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -274,7 +274,7 @@ def solve_network(n, config, opts='', **kwargs): if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('solve_network', network='elec', simpl='', + snakemake = mock_snakemake('solve_network', weather_year='', simpl='', clusters='5', ll='copt', opts='Co2L-BAU-CCL-24H') configure_logging(snakemake) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 717cba46..bac866c2 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -96,7 +96,7 @@ def set_parameters_from_optimized(n, n_optim): if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('solve_operations_network', network='elec', + snakemake = mock_snakemake('solve_operations_network', weather_year='', simpl='', clusters='5', ll='copt', opts='Co2L-BAU-24H') configure_logging(snakemake) From 9f9e65afb1433d35809bb9e431ef33ab10dbe5e3 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 15:18:07 +0200 Subject: [PATCH 038/173] add missing handling of weather_year wildcard --- Snakefile | 28 ++++++++++++------------- scripts/build_heat_demand.py | 3 +-- scripts/build_solar_thermal_profiles.py | 3 +-- scripts/build_temperature_profiles.py | 3 +-- scripts/build_transport_demand.py | 8 ++++++- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Snakefile b/Snakefile index 0ca4e7c8..fa78641b 100644 --- a/Snakefile +++ b/Snakefile @@ -440,8 +440,8 @@ else: rule build_population_weighted_energy_totals: input: energy_totals='resources/energy_totals.csv', - clustered_pop_layout="resources/pop_layout_elec_s{simpl}_{clusters}.csv" - output: "resources/pop_weighted_energy_totals_s{simpl}_{clusters}.csv" + clustered_pop_layout="resources/pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + output: "resources/pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=2000 script: "scripts/build_population_weighted_energy_totals.py" @@ -449,17 +449,17 @@ rule build_population_weighted_energy_totals: rule build_transport_demand: input: - clustered_pop_layout="resources/pop_layout_elec_s{simpl}_{clusters}.csv", - pop_weighted_energy_totals="resources/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", + clustered_pop_layout="resources/pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", + pop_weighted_energy_totals="resources/pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv", transport_data='resources/transport_data.csv', traffic_data_KFZ="data/emobility/KFZ__count", traffic_data_Pkw="data/emobility/Pkw__count", - temp_air_total="resources/temp_air_total_elec_s{simpl}_{clusters}.nc", + temp_air_total="resources/temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", output: - transport_demand="resources/transport_demand_s{simpl}_{clusters}.csv", - transport_data="resources/transport_data_s{simpl}_{clusters}.csv", - avail_profile="resources/avail_profile_s{simpl}_{clusters}.csv", - dsm_profile="resources/dsm_profile_s{simpl}_{clusters}.csv" + transport_demand="resources/transport_demand{weather_year}_s{simpl}_{clusters}.csv", + transport_data="resources/transport_data{weather_year}_s{simpl}_{clusters}.csv", + avail_profile="resources/avail_profile{weather_year}_s{simpl}_{clusters}.csv", + dsm_profile="resources/dsm_profile{weather_year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=2000 script: "scripts/build_transport_demand.py" @@ -470,11 +470,11 @@ rule prepare_sector_network: overrides="data/override_component_attrs", network=pypsaeur('networks/elec{weather_year}_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc'), energy_totals_name='resources/energy_totals.csv', - pop_weighted_energy_totals="resources/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", - transport_demand="resources/transport_demand_s{simpl}_{clusters}.csv", - transport_data="resources/transport_data_s{simpl}_{clusters}.csv", - avail_profile="resources/avail_profile_s{simpl}_{clusters}.csv", - dsm_profile="resources/dsm_profile_s{simpl}_{clusters}.csv", + pop_weighted_energy_totals="resources/pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv", + transport_demand="resources/transport_demand{weather_year}_s{simpl}_{clusters}.csv", + transport_data="resources/transport_data{weather_year}_s{simpl}_{clusters}.csv", + avail_profile="resources/avail_profile{weather_year}_s{simpl}_{clusters}.csv", + dsm_profile="resources/dsm_profile{weather_year}_s{simpl}_{clusters}.csv", co2_totals_name='resources/co2_totals.csv', biomass_potentials='resources/biomass_potentials_s{simpl}_{clusters}.csv', heat_profile="data/heat_load_profile_BDEW.csv", diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 23eeae27..fe44de16 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -18,7 +18,6 @@ if __name__ == '__main__': cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) if year: snapshots = dict(start=year, end=str(int(year)+1), closed="left") @@ -27,7 +26,7 @@ if __name__ == '__main__': snapshots = snakemake.config['snapshots'] time = pd.date_range(freq='m', **snapshots) - if drop_leap_day: + if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] cutout = atlite.Cutout(cutout_name).sel(time=time) diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index e87c9a1e..8bdc07c5 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -20,7 +20,6 @@ if __name__ == '__main__': cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) if year: snapshots = dict(start=year, end=str(int(year)+1), closed="left") @@ -29,7 +28,7 @@ if __name__ == '__main__': snapshots = snakemake.config['snapshots'] time = pd.date_range(freq='m', **snapshots) - if drop_leap_day: + if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] cutout = atlite.Cutout(cutout_name).sel(time=time) diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index 02cae24c..ab006409 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -18,7 +18,6 @@ if __name__ == '__main__': cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) if year: snapshots = dict(start=year, end=str(int(year)+1), closed="left") @@ -27,7 +26,7 @@ if __name__ == '__main__': snapshots = snakemake.config['snapshots'] time = pd.date_range(freq='m', **snapshots) - if drop_leap_day: + if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] cutout = atlite.Cutout(cutout_name).sel(time=time) diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index a5aabc31..a9d6e440 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -159,6 +159,7 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_transport_demand", + weather_year='', simpl="", clusters=48, ) @@ -173,7 +174,12 @@ if __name__ == "__main__": options = snakemake.config["sector"] - snapshots = pd.date_range(freq='h', **snakemake.config["snapshots"], tz="UTC") + year = snakemake.wildcards.weather_year + snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] + snapshots = pd.date_range(freq='h', **snapshots, tz="UTC") + if snakemake.config["atlite"].get("drop_leap_day", False): + leap_day = (snapshots.month == 2) & (snapshots.day == 29) + snapshots = snapshots[~leap_day] Nyears = 1 From a2e26004d7b2026a61cc17683323b2673ad40649 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 15:36:37 +0200 Subject: [PATCH 039/173] add further missing handling of weather_year wildcard --- Snakefile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Snakefile b/Snakefile index fa78641b..852e924f 100644 --- a/Snakefile +++ b/Snakefile @@ -68,7 +68,7 @@ if config.get('retrieve_sector_databundle', True): rule build_population_layouts: input: - cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), + cutout=pypsaeur("cutouts/" + config['atlite']['cutout'] + ".nc"), nuts3_shapes=pypsaeur('resources/nuts3_shapes.geojson'), urban_percent="data/urban_percent.csv" output: @@ -83,7 +83,7 @@ rule build_population_layouts: rule build_clustered_population_layouts: input: - cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), + cutout=pypsaeur("cutouts/" + config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -138,8 +138,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson", production="data/gas_network/scigrid-gas/data/IGGIELGN_Productions.geojson", planned_lng="data/gas_network/planned_LNGs.csv", - regions_onshore=pypsaeur("resources/regions_onshore_elec_s{simpl}_{clusters}.geojson"), - regions_offshore=pypsaeur('resources/regions_offshore_elec_s{simpl}_{clusters}.geojson') + regions_onshore=pypsaeur("resources/regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_offshore=pypsaeur('resources/regions_offshore{weather_year}_elec_s{simpl}_{clusters}.geojson') output: gas_input_nodes="resources/gas_input_locations_s{simpl}_{clusters}.geojson", gas_input_nodes_simplified="resources/gas_input_locations_s{simpl}_{clusters}_simplified.csv" @@ -150,8 +150,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule cluster_gas_network: input: cleaned_gas_network="resources/gas_network.csv", - regions_onshore=pypsaeur("resources/regions_onshore_elec_s{simpl}_{clusters}.geojson"), - regions_offshore=pypsaeur("resources/regions_offshore_elec_s{simpl}_{clusters}.geojson") + regions_onshore=pypsaeur("resources/regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_offshore=pypsaeur("resources/regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson") output: clustered_gas_network="resources/gas_network_elec_s{simpl}_{clusters}.csv" resources: mem_mb=4000 @@ -165,7 +165,7 @@ else: rule build_heat_demands: input: - cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), + cutout=pypsaeur("cutouts/" + config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -181,7 +181,7 @@ rule build_heat_demands: rule build_temperature_profiles: input: - cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), + cutout=pypsaeur("cutouts/" + config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -220,7 +220,7 @@ rule build_cop_profiles: rule build_solar_thermal_profiles: input: - cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), + cutout=pypsaeur("cutouts/" + config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -476,7 +476,7 @@ rule prepare_sector_network: avail_profile="resources/avail_profile{weather_year}_s{simpl}_{clusters}.csv", dsm_profile="resources/dsm_profile{weather_year}_s{simpl}_{clusters}.csv", co2_totals_name='resources/co2_totals.csv', - biomass_potentials='resources/biomass_potentials_s{simpl}_{clusters}.csv', + biomass_potentials='resources/biomass_potentials{weather_year}_s{simpl}_{clusters}.csv', heat_profile="data/heat_load_profile_BDEW.csv", costs=CDIR + "costs_{planning_horizons}.csv", profile_offwind_ac=pypsaeur("resources/profile{weather_year}_offwind-ac.nc"), From 6713317bbea94156d3e7166115ab709e54530f70 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 16 Jun 2022 08:46:47 +0200 Subject: [PATCH 040/173] add_electricity: fix missing variable assignment --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index f092c563..b86e8a9e 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -563,7 +563,7 @@ if __name__ == "__main__": closed="left" ) else: - snakemake.config['snapshots'] + snapshots = snakemake.config['snapshots'] n.set_snapshots(pd.date_range(freq='h', **snapshots)) Nyears = n.snapshot_weightings.objective.sum() / 8760. From cb0554f609f5e2318542e73eb7b9dfda2f3a84e2 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 16 Jun 2022 08:51:06 +0200 Subject: [PATCH 041/173] config: add fixed_year parameter for load --- config.tutorial.yaml | 2 +- test/config.test1.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index c9638260..7e161ef2 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -157,7 +157,7 @@ load: time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from manual_adjustments: true # false scaling_factor: 1.0 - fallback_year: 2013 + fixed_year: 2013 costs: year: 2030 diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 95a8f85a..86451646 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -155,7 +155,7 @@ load: time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from manual_adjustments: true # false scaling_factor: 1.0 - fallback_year: 2013 + fixed_year: false costs: year: 2030 From b4c819c7573a08e46e46ec99cf68163491105738 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 16 Jun 2022 08:51:06 +0200 Subject: [PATCH 042/173] config: add fixed_year parameter for load --- config.tutorial.yaml | 2 +- test/config.test1.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index c9638260..f0e24822 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -157,7 +157,7 @@ load: time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from manual_adjustments: true # false scaling_factor: 1.0 - fallback_year: 2013 + fixed_year: false costs: year: 2030 diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 95a8f85a..86451646 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -155,7 +155,7 @@ load: time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from manual_adjustments: true # false scaling_factor: 1.0 - fallback_year: 2013 + fixed_year: false costs: year: 2030 From 173ab7b318e1d0e38bbab96b6bc3f103ccbf2abf Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 16 Jun 2022 09:00:33 +0200 Subject: [PATCH 043/173] add_electricity: make drop_leap_year own function --- scripts/add_electricity.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index b86e8a9e..9158ce33 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -547,6 +547,15 @@ def add_nice_carrier_names(n, config): logger.warning(f'tech_colors for carriers {missing_i} not defined in config.') n.carriers['color'] = colors + +def drop_leap_day(n): + if not n.snapshots.is_leap_year.any(): return + leap_days = (n.snapshots.day == 29) & (n.snapshots.month == 2) + n.set_snapshots(n.snapshots[~leap_days]) + n.snapshot_weightings[:] = 8760/len(n.snapshots) + logger.info("Dropped February 29 from leap year.") + + if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake @@ -600,7 +609,6 @@ if __name__ == "__main__": add_nice_carrier_names(n, snakemake.config) if snakemake.config['enable'].get('drop_leap_days', True): - leap_days = (n.snapshots.day == 29) & (n.snapshots.month == 2) - n.set_snapshots(n.snapshots[~leap_days]) + drop_leap_day(n) n.export_to_netcdf(snakemake.output[0]) From b171614feaa87b15e4d4d25b3d6f95f3e76f97c7 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 20 Jul 2022 16:37:22 +0200 Subject: [PATCH 044/173] must parse fixed_year to string --- scripts/build_load_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_load_data.py b/scripts/build_load_data.py index c1419c5d..d7cd91a4 100755 --- a/scripts/build_load_data.py +++ b/scripts/build_load_data.py @@ -220,7 +220,7 @@ if __name__ == "__main__": snapshots = pd.date_range(freq='h', **snapshots) fixed_year = snakemake.config["load"].get("fixed_year", False) - years = slice(fixed_year, fixed_year) if fixed_year else slice(snapshots[0], snapshots[-1]) + years = slice(str(fixed_year), str(fixed_year)) if fixed_year else slice(snapshots[0], snapshots[-1]) powerstatistics = snakemake.config['load']['power_statistics'] interpolate_limit = snakemake.config['load']['interpolate_limit'] From af96af1329ea046cb384cf9b69ba27fbd2dd4524 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 22 Jul 2022 15:03:54 +0200 Subject: [PATCH 045/173] hydro: approximation routine for missing eia years and correction by capacity --- Snakefile | 6 ++- config.default.yaml | 4 +- data/eia_hydro_annual_capacity.csv | 50 +++++++++++++++++++++ data/era5-annual-runoff-per-country.csv | 34 ++++++++++++++ envs/environment.yaml | 1 + scripts/build_hydro_profile.py | 59 +++++++++++++++++++++++-- 6 files changed, 147 insertions(+), 7 deletions(-) create mode 100644 data/eia_hydro_annual_capacity.csv create mode 100644 data/era5-annual-runoff-per-country.csv diff --git a/Snakefile b/Snakefile index 691566c2..e02b911b 100644 --- a/Snakefile +++ b/Snakefile @@ -154,7 +154,7 @@ if config['enable'].get('build_cutout', False): input: regions_onshore="resources/regions_onshore.geojson", regions_offshore="resources/regions_offshore.geojson" - output: "cutouts/{cutout}.nc" + output: protected("cutouts/{cutout}.nc") log: "logs/build_cutout/{cutout}.log" benchmark: "benchmarks/build_cutout_{cutout}" threads: ATLITE_NPROCESSES @@ -163,7 +163,7 @@ if config['enable'].get('build_cutout', False): rule build_cutout_year: input: rules.build_cutout.input - output: "cutouts/{cutout}-{weather_year}.nc" + output: protected("cutouts/{cutout}-{weather_year}.nc") log: "logs/build_cutout/{cutout}-{weather_year}.log" benchmark: "benchmarks/build_cutout_{cutout}-{weather_year}" threads: ATLITE_NPROCESSES @@ -226,6 +226,8 @@ rule build_hydro_profile: input: country_shapes='resources/country_shapes.geojson', eia_hydro_generation='data/eia_hydro_annual_generation.csv', + eia_hydro_capacity='data/eia_hydro_annual_capacity.csv', + era5_runoff='data/era5-annual-runoff-per-country.csv', cutout=f"cutouts/{config['renewable']['hydro']['cutout']}.nc" if "hydro" in config["renewable"] else "config['renewable']['hydro']['cutout'] not configured", output: 'resources/profile{weather_year}_hydro.nc' log: "logs/build_hydro_profile{weather_year}.log" diff --git a/config.default.yaml b/config.default.yaml index 1e1b7dfc..aa2d64d0 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -184,7 +184,9 @@ renewable: PHS_max_hours: 6 hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float clip_min_inflow: 1.0 - norm_year: 2013 + eia_norm_year: 2013 + eia_correct_by_capacity: false + eia_approximate_missing: false conventional: nuclear: diff --git a/data/eia_hydro_annual_capacity.csv b/data/eia_hydro_annual_capacity.csv new file mode 100644 index 00000000..a95d71a4 --- /dev/null +++ b/data/eia_hydro_annual_capacity.csv @@ -0,0 +1,50 @@ +# https://www.eia.gov/international/data/world/electricity/electricity-generation?pd=2&p=00000000000000000000008&u=1&f=A&v=mapbubble&a=-&i=none&vo=value&t=R&g=000000000000002&l=73-1028i008017kg6368g80a4k000e0ag00gg0004g8g0ho00g000400008&s=315532800000&e=1577836800000&ev=false& +Report generated on: 07-21-2022 16:15:18 +"API","","1980","1981","1982","1983","1984","1985","1986","1987","1988","1989","1990","1991","1992","1993","1994","1995","1996","1997","1998","1999","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2011","2012","2013","2014","2015","2016","2017","2018","2019","2020" +"","hydroelectricity installed capacity (million kW)","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","" +"INTL.33-7-EURO-MK.A"," Europe","136.143","137.425","141.734","143.763","142.894","144.738","147.631","150.428","153.428","153.345","139.346","134.524","137.463","138.338","139.688","141.47","142.121","143.595","143.957","146.4204","147.3512","147.736","152.173","152.938","150.4894","151.424","152.276","154.198","155.405","156.988","159.495","162.112","165.452","170.146","171.588","174.906","176.998","178.221","180.846","181.68","184.909" +"INTL.33-7-ALB-MK.A"," Albania","0.5","0.525","0.55","0.6","0.625","0.65","0.675","0.68","0.69","0.69","1.668","1.668","1.668","1.668","1.668","1.445","1.445","1.445","1.445","1.445","1.445","1.445","1.445","1.445","1.445","1.432","1.432","1.432","1.45","1.45","1.461","1.508","1.628","1.781","1.725","1.798","1.913","2.047","2.105","2.193","2.39" +"INTL.33-7-AUT-MK.A"," Austria","8.206","9.157","9.51","9.582","10.034","10.171","10.423","10.691","10.762","10.858","7.028","7.129","7.204","7.202","7.245","7.323","7.385","7.54","7.685","7.669","7.676","7.703","7.567","7.607","7.613","7.667","7.684","7.845","7.848","7.827","7.913","7.947","7.97","8.272","8.321","8.457","8.493","8.506","8.591","8.63","9.001" +"INTL.33-7-BEL-MK.A"," Belgium","0.073","0.08","0.086","0.086","0.086","0.087","0.089","0.09","0.093","0.095","0.094","0.094","0.094","0.095","0.095","0.096","0.096","0.096","0.097","0.103","0.103","0.111","0.111","0.11","0.115","0.105","0.107","0.11","0.111","0.11","0.118","0.119","0.12","0.119","0.121","0.112","0.115","0.107","0.108","0.108","0.12" +"INTL.33-7-BIH-MK.A"," Bosnia and Herzegovina","--","--","--","--","--","--","--","--","--","--","--","--","1.2","1.2","1.139","1.219","1.219","1.219","1.624","1.983","1.983","1.993","2.38","2.38","2.38","2.38","2.411","2.411","2.117","2.117","2.117","2.117","2.12","2.12","2.049","2.055","2.084","2.084","2.09","2.09","2.093" +"INTL.33-7-BGR-MK.A"," Bulgaria","1.895","1.895","1.895","1.975","1.975","1.975","1.975","1.975","1.975","1.973","1.973","1.401","1.401","1.401","1.401","1.401","1.401","1.803","1.803","1.803","1.881","1.706","1.948","1.984","1.984","1.984","1.984","2.012","2.12","2.137","2.184","2.035","2.095","2.165","2.19","2.206","2.206","2.21","2.359","2.366","1.725" +"INTL.33-7-HRV-MK.A"," Croatia","--","--","--","--","--","--","--","--","--","--","--","--","1.769","1.77","1.77","1.781","1.785","1.785","1.785","1.785","1.785","1.785","1.775","1.783","1.79","1.804","1.804","1.782","1.782","1.799","1.848","1.848","1.848","1.897","1.9","1.915","1.912","1.912","1.913","1.913","1.848" +"INTL.33-7-CYP-MK.A"," Cyprus","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-7-CZE-MK.A"," Czech Republic","--","--","--","--","--","--","--","--","--","--","--","--","--","0.911","0.906","0.908","0.868","0.905","0.888","1.008","0.952","1","1","1.004","1.015","1.02","1.016","1.024","1.029","1.037","1.049","1.05","1.065","1.08","1.08","1.088","1.09","1.093","1.094","1.096","1.097" +"INTL.33-7-DNK-MK.A"," Denmark","0.008","0.008","0.01","0.009","0.009","0.009","0.009","0.009","0.009","0.011","0.01","0.01","0.01","0.01","0.008","0.01","0.01","0.01","0.011","0.011","0.01","0.011","0.011","0.011","0.011","0.011","0.009","0.009","0.009","0.009","0.009","0.009","0.009","0.009","0.009","0.007","0.009","0.009","0.009","0.009","0.009" +"INTL.33-7-EST-MK.A"," Estonia","--","--","--","--","--","--","--","--","--","--","--","--","0.001","0.001","0.001","0.001","0.001","0.001","0.001","0.0012","0.0012","0.003","0.003","0.004","0.004","0.005","0.005","0.005","0.005","0.007","0.006","0.005","0.008","0.008","0.005","0.006","0.006","0.007","0.007","0.007","0.008" +"INTL.33-7-FRO-MK.A"," Faroe Islands","0.018","0.018","0.018","0.018","0.018","0.018","0.018","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.0314","0.032","0.032","0.032","0.032","0.0314","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.039","0.039","0.04","0.04","0.04","0.04","0.04","0.04","0.039" +"INTL.33-7-FIN-MK.A"," Finland","2.42","2.467","2.474","2.503","2.497","2.505","2.555","2.586","2.597","2.586","2.621","2.648","2.679","2.731","2.736","2.777","2.785","2.861","2.881","2.881","2.882","2.926","2.964","2.966","2.999","3.035","3.062","3.102","3.122","3.145","3.155","3.196","3.196","3.224","3.248","3.249","3.249","3.272","3.287","3.287","3.263" +"INTL.33-7-CSK-MK.A"," Former Czechoslovakia","2.578","2.832","2.84","2.84","2.875","2.897","2.89","2.975","2.988","3.042","3.036","3.061","3.061","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-7-SCG-MK.A"," Former Serbia and Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","2.25","2.25","2.25","2.25","2.25","2.25","2.25","2.296","2.296","2.296","2.296","2.296","2.206","2.206","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-7-YUG-MK.A"," Former Yugoslavia","6.2","6.25","5.886","5.886","6.386","6.736","7.086","7.386","7.625","7.686","7.386","7.386","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-7-FRA-MK.A"," France","17.431","17.63","18.247","18.4","18.661","19.034","19.786","19.991","20.174","20.338","17.717","17.655","17.767","17.837","17.902","17.898","18","18.018","18.024","17.947","17.646","17.674","17.775","17.927","17.812","17.808","17.82","17.832","17.922","18.009","18.156","18.373","18.388","18.379","18.392","18.415","18.486","18.561","18.857","18.88","19.671" +"INTL.33-7-DEU-MK.A"," Germany","--","--","--","--","--","--","--","--","--","--","--","3.31","3.317","3.385","3.471","3.624","3.563","3.569","3.642","3.802","4.086","4.101","4.193","4.088","4.209","4.134","4.117","4.083","4.104","4.283","4.252","4.469","4.451","4.433","4.424","4.433","4.442","4.449","4.456","4.456","4.658" +"INTL.33-7-DDR-MK.A"," Germany, East","1.852","1.845","1.852","1.851","1.845","1.844","1.844","1.844","1.844","1.844","1.844","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-7-DEUW-MK.A"," Germany, West","6.45","6.509","6.531","6.631","6.668","6.71","6.71","6.71","6.85","6.86","6.86","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-7-GIB-MK.A"," Gibraltar","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-7-GRC-MK.A"," Greece","1.415","1.714","1.714","1.714","1.714","1.822","1.822","1.822","1.836","1.986","2.093","2.197","2.208","2.208","2.208","2.208","2.207","2.412","2.241","2.344","2.373","2.377","2.379","2.38","2.4","2.407","2.435","2.451","2.477","2.502","2.516","2.525","2.537","2.539","2.69","2.693","2.693","2.693","2.71","2.71","2.697" +"INTL.33-7-HUN-MK.A"," Hungary","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.054","0.049","0.049","0.049","0.049","0.051","0.053","0.053","0.055","0.056","0.057","0.057","0.057","0.057","0.057","0.057","0.057","0.056" +"INTL.33-7-ISL-MK.A"," Iceland","0.545","0.615","0.755","0.755","0.755","0.755","0.756","0.756","0.756","0.756","0.756","0.779","0.879","0.879","0.884","0.884","0.884","0.923","0.956","1.016","1.064","1.109","1.155","1.155","1.163","1.163","1.163","1.758","1.879","1.875","1.883","1.884","1.877","1.984","1.984","1.987","1.987","1.995","2.099","2.099","2.086" +"INTL.33-7-IRL-MK.A"," Ireland","0.224","0.224","0.225","0.225","0.226","0.226","0.221","0.222","0.222","0.222","0.223","0.226","0.226","0.226","0.227","0.227","0.232","0.233","0.233","0.236","0.236","0.238","0.24","0.24","0.24","0.234","0.234","0.234","0.234","0.234","0.237","0.237","0.237","0.237","0.237","0.237","0.237","0.237","0.237","0.237","0.237" +"INTL.33-7-ITA-MK.A"," Italy","15.826","15.766","16.877","17.125","12.166","12.16","12.419","12.435","12.495","12.547","12.582","12.692","12.718","12.788","12.864","12.964","12.999","13.06","13.058","13.417","13.389","13.456","13.557","13.703","13.789","13.89","13.528","13.573","13.732","13.827","13.976","14.193","14.325","14.454","14.506","14.628","14.991","15.109","15.182","15.583","14.908" +"INTL.33-7-XKS-MK.A"," Kosovo","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","0.043","0.043","0.045","0.046","0.046","0.046","0.046","0.046","0.078","0.08","0.096","0.096","0.092" +"INTL.33-7-LVA-MK.A"," Latvia","--","--","--","--","--","--","--","--","--","--","--","--","1.499","1.504","1.506","1.521","1.521","1.487","1.517","1.523","1.523","1.565","1.565","1.537","1.536","1.536","1.536","1.536","1.536","1.536","1.576","1.576","1.576","1.587","1.588","1.588","1.564","1.564","1.565","1.565","1.576" +"INTL.33-7-LTU-MK.A"," Lithuania","--","--","--","--","--","--","--","--","--","--","--","--","0.106","0.106","0.108","0.108","0.108","0.108","0.108","0.112","0.112","0.113","0.103","0.109","0.11","0.117","0.117","0.115","0.115","0.116","0.116","0.116","0.116","0.116","0.117","0.117","0.117","0.117","0.117","0.117","0.116" +"INTL.33-7-LUX-MK.A"," Luxembourg","0.029","0.029","0.029","0.032","0.032","0.032","0.032","0.032","0.032","0.032","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.039","0.039","0.039","0.039","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.035","0.034","0.034","0.034" +"INTL.33-7-MLT-MK.A"," Malta","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-7-MNE-MK.A"," Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","0.658","0.658","0.658","0.658","0.658","0.658","0.658","0.651","0.651","0.651","0.651","0.652","0.652","0.652","0.658" +"INTL.33-7-NLD-MK.A"," Netherlands","0","0","0","0","0","0.002","0.002","0.002","0.002","0.025","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.038" +"INTL.33-7-MKD-MK.A"," North Macedonia","--","--","--","--","--","--","--","--","--","--","--","--","0.426","0.426","0.413","0.423","0.423","0.434","0.434","0.4338","0.434","0.436","0.436","0.448","0.516","0.54","0.545","0.546","0.552","0.553","0.555","0.556","0.595","0.603","0.617","0.632","0.66","0.674","0.674","0.674","0.674" +"INTL.33-7-NOR-MK.A"," Norway","19.443","20.963","21.789","22.37","22.628","23.076","23.555","24.358","24.453","25.728","25.817","25.823","25.963","25.983","26.215","27.379","27.494","27.327","26.982","27.54","26.766","26.319","26.604","26.947","26.721","27.222","27.398","27.647","28.062","28.188","28.367","28.618","29.158","29.682","29.889","29.939","30.281","30.382","31.12","31.182","31.556" +"INTL.33-7-POL-MK.A"," Poland","0.647","0.647","0.647","0.647","0.645","0.646","0.646","0.646","0.772","0.647","0.467","0.467","0.468","0.475","0.489","0.482","0.492","0.495","0.501","0.505","0.509","0.517","0.517","0.524","0.535","0.542","0.549","0.546","0.553","0.556","0.56","0.564","0.569","0.573","0.582","0.588","0.597","0.591","0.592","0.592","0.605" +"INTL.33-7-PRT-MK.A"," Portugal","2.516","2.615","2.854","2.944","3.016","2.721","2.818","2.82","2.722","2.799","2.783","2.772","3.146","3.613","3.697","3.848","3.867","3.877","3.94","3.93","3.918","3.943","3.966","3.966","3.974","3.968","4.004","4.012","4.009","4.042","4.057","4.49","4.414","4.363","4.368","4.446","4.458","4.462","4.484","4.484","4.373" +"INTL.33-7-ROU-MK.A"," Romania","3.455","3.533","3.734","3.885","4.062","4.42","4.706","5.057","5.421","5.583","5.666","5.723","5.687","5.872","5.938","6.011","6.038","6.074","6.081","6.082","6.12","6.122","6.242","6.248","6.279","6.289","6.282","6.331","6.362","6.358","6.382","6.391","6.456","6.249","6.256","6.359","6.377","6.328","6.328","6.328","6.221" +"INTL.33-7-SRB-MK.A"," Serbia","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","2.206","2.206","2.206","2.221","2.293","2.322","2.352","2.317","2.403","2.408","2.416","2.424","2.424","2.424","2.484" +"INTL.33-7-SVK-MK.A"," Slovakia","--","--","--","--","--","--","--","--","--","--","--","--","--","0.925","1.208","1.525","1.655","1.664","1.682","1.684","1.685","1.586","1.586","1.588","1.602","1.596","1.597","1.599","1.632","1.571","1.6","1.607","1.606","1.607","1.607","1.606","1.608","1.607","1.612","1.612","1.505" +"INTL.33-7-SVN-MK.A"," Slovenia","--","--","--","--","--","--","--","--","--","--","--","--","0.755","0.755","0.756","0.757","0.734","0.734","0.861","0.846","0.846","0.839","0.983","0.974","0.974","0.979","1.009","1.018","1.027","1.07","1.074","1.073","1.074","1.119","1.116","1.115","1.113","1.167","1.163","1.163","1.344" +"INTL.33-7-ESP-MK.A"," Spain","13.473","10.869","10.945","10.917","10.935","10.959","11.153","10.556","10.984","11.597","11.32","11.429","11.484","11.484","11.545","11.689","11.793","11.596","11.537","11.802","12.672","12.744","15.55","15.525","12.82","12.808","12.907","12.961","13.04","13.069","13.275","13.283","13.293","14.076","14.081","14.086","14.053","14.052","14.053","14.053","14.292" +"INTL.33-7-SWE-MK.A"," Sweden","14.859","14.919","15.215","15.29","15.445","15.69","15.813","15.996","16.112","15.759","15.904","15.891","16.021","15.867","16.072","15.725","15.776","16.371","16.169","16.432","16.506","16.523","16.187","16.098","16.302","16.302","16.234","16.592","16.352","16.544","16.624","16.478","16.315","16.395","15.897","16.23","16.367","16.403","16.332","16.332","16.379" +"INTL.33-7-CHE-MK.A"," Switzerland","11.45","11.46","11.47","11.47","11.48","11.48","11.51","11.51","11.52","11.58","3.474","3.484","3.504","3.509","3.526","3.541","3.55","3.553","3.584","3.614","3.636","3.642","3.653","3.669","3.65","3.682","3.694","3.7","3.709","3.749","3.81","3.852","3.882","3.896","3.948","3.996","4.06","4.112","4.193","4.193","4.193" +"INTL.33-7-TUR-MK.A"," Turkey","2.131","2.356","3.082","3.239","3.875","3.875","3.878","5.003","6.219","6.598","6.764","7.114","8.379","9.682","9.865","9.863","9.935","10.102","10.307","10.537","11.175","11.673","12.241","12.579","12.645","12.906","13.063","13.395","13.829","14.553","15.831","17.137","19.609","22.289","23.643","25.868","26.681","27.273","28.291","28.503","30.984" +"INTL.33-7-GBR-MK.A"," United Kingdom","2.451","2.451","2.451","2.721","4.188","4.19","4.192","4.197","4.196","1.424","1.11","1.415","1.423","1.425","1.425","1.432","1.455","1.488","1.475","1.477","1.485","1.629","1.59","1.486","1.499","1.501","1.515","1.522","1.626","1.638","1.637","1.673","1.693","1.709","1.73","1.777","1.836","1.873","1.878","1.878","1.879" \ No newline at end of file diff --git a/data/era5-annual-runoff-per-country.csv b/data/era5-annual-runoff-per-country.csv new file mode 100644 index 00000000..44055ee7 --- /dev/null +++ b/data/era5-annual-runoff-per-country.csv @@ -0,0 +1,34 @@ +name,1951,1952,1953,1954,1955,1956,1957,1958,1959,1960,1961,1962,1963,1964,1965,1966,1967,1968,1969,1970,1971,1972,1973,1974,1975,1976,1977,1978,1979,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021 +AL,9921.845743062182,13900.722955927391,9710.931748556879,10574.663320350839,19341.246363906943,17818.135264440367,11442.713887022777,18086.801216289034,15090.128262650946,21363.26299103702,9091.698545327283,15645.252953546536,26744.08978638467,15193.032691775385,15984.127347228396,21350.165383112253,12855.252167834218,13420.929958145927,18491.92173822311,23127.97840031047,15647.322475185589,18621.138651647332,15899.845988546795,17394.46706381624,9941.517004312835,15056.517227865626,13546.145796746823,19078.487245825123,21734.889469128113,18681.018783118114,17330.7404590047,11854.58505075189,8621.019687739607,10958.633158319248,11928.114867207425,14053.437518158384,11478.547049310142,9794.162917024925,9309.704198107493,7531.383319836307,13417.032891488734,9396.693750356952,9409.269914411027,9602.245346872442,13721.093186096236,18546.848247572612,10075.274288997707,13160.130634381218,14979.3178626011,11173.442130939595,10656.167120597063,11333.102860367679,12218.649211273396,16968.134523176956,13531.444805168734,12351.485474236735,7777.709342414358,9395.578415692386,13162.74078695902,21279.522933687273,7131.262505063414,9979.8596278608,14852.708775826433,10550.911120633964,9976.081734060937,14930.01135394502,8322.107101243482,14871.511785547276,8735.240828081134,9201.521447196284,15965.927842545672 +AT,82083.36248472283,77811.80736180999,72623.28291775129,92135.9214601233,96990.37023646115,85097.60090834703,90540.2512560922,87926.80279582532,87840.33332785149,95771.51050456028,80100.0597367668,89961.83970292915,94250.95465501286,83836.27750897587,127502.64206362939,114305.16098508416,109841.9915507258,94950.56912780282,78607.44609072198,109558.29937241938,70075.51296406337,91798.91825728992,87743.39136454638,99172.3185471974,112079.92282492586,77403.35720896415,141634.05110901658,127491.44958971704,103513.31397365611,94072.4808516721,90719.06004331182,74178.70850428617,71954.43763473642,75430.93523721115,83080.51606488624,68680.54577957527,88997.37641681913,78365.83036558417,88108.35920540475,70055.7892534604,83590.790849829,80943.05621285089,88054.73669568839,64323.72991620271,76516.31784700113,88537.52869871816,74980.7275243927,76703.23963260197,90699.65433828124,82618.46892777775,70279.06284438398,76864.90473420777,38883.771847486016,64333.37459685728,71147.24783545699,61897.64030624322,56722.92181769331,64364.47029441406,80300.64217575881,73731.45518976511,60840.309523971024,77575.9681355992,74925.98195655535,87076.27014563217,55659.14157544586,71826.27744178762,65790.17399828041,62758.61938425757,63451.07652587882,63443.71962749899,60531.78695214822 +BE,2265.3933295931465,2299.7187477495568,1601.6287456097295,903.9162052526016,1681.6717933948885,1553.0826978024484,2040.8513201770277,2596.453742857211,1572.1256345899858,1777.5813929080084,2626.96550179367,2109.9419394561014,1095.552430877252,1153.3090754765356,2926.2047527605378,3273.3444811813465,2455.1462895729287,2709.4254348359373,2044.223080485547,3206.6821776003253,1403.4368531879622,1739.01009962226,1509.0861879549796,2489.4865224203572,2440.1073430603583,1152.6261514941527,2218.937185944037,2759.156597708309,2657.6330859359005,2904.764428136616,3657.83503348807,2493.2486286738635,2917.270148862828,2660.3113890881846,1897.0141168954808,2027.0116832611525,2822.5902787959144,2964.8457470488684,1943.761175974426,1192.1713008070133,1419.0119225208746,1475.2557281855488,1769.9742186132778,2647.8931490311224,2842.9882619071104,880.5015070232031,1197.5019163599245,2093.4616667083587,2614.387699318041,2455.605567526133,2964.8707630509653,2512.265466519676,1585.2842393930487,929.2255532772314,1281.2863306004592,1121.936801130382,2183.018089275124,2209.175694512598,1603.8823576630593,1544.7801250586733,1495.8450265096008,1698.4561860883377,1826.3476500921734,1803.4120415623293,1828.4613155494794,2424.4009518987536,1091.1596112517386,1814.2183897620703,1171.4609814371709,2056.8674333110275,2031.8962335362714 +BG,7806.165358407907,7617.879584469116,9313.915986883927,12665.103843530671,18165.703005314856,20925.111917073817,12315.95137614649,14307.215196342033,11652.743586791588,16042.778215050708,15000.011504505326,15411.367697482856,23959.284980030086,13749.537818599603,14477.782616320155,19301.913180173906,18779.64649297375,14471.754083245778,21905.976678210158,20337.934359162857,20009.368056844236,22642.8256575709,23291.544303799303,14596.266505770593,16672.574320967968,18465.93977853061,15123.0740003407,16401.953223276385,17509.31598177905,19228.19892146255,14599.539868168713,16192.551398587811,14129.882713144674,15395.87601647586,11853.583066311567,12273.885630367266,13296.879487675578,12962.17913948413,10178.141531948864,10101.748861881337,17833.95419276747,11290.360773903032,7050.797194092144,7109.831367133397,12879.727662239184,15849.231621819281,14036.330428384765,14600.701492135015,14377.532715491632,7705.7385359122845,8414.14109970378,12768.14503989922,11390.56673556146,10492.153643864705,15665.526881990423,13518.231074072748,9896.670696858979,8514.724851169687,10495.74229946015,14331.982881842356,7228.669024198985,6779.132502990354,8418.829006301234,16447.848686603495,15740.35987338571,11179.883037182273,8789.581276892988,11591.504961624405,6993.71096793164,8046.946588923506,10797.966046345515 +BA,23467.52324346034,27732.028396263388,21355.39127202247,21518.57483169428,38198.02048713121,26864.382517947488,20815.045478598513,30820.77172304343,26041.454718573805,32434.848569531026,20613.68138756547,29427.6067241259,33216.04695823398,35126.00882914745,35434.02869793576,33198.240125653894,27292.13451964922,28870.288475787063,34855.527471322755,40158.11721181478,26000.45986387781,30818.815452611776,25386.059210704487,29741.547122355205,28211.934365516674,30030.474291377286,28262.88980319508,36551.088477619196,29006.131913946814,27659.528119372964,23600.138376555304,18313.888493525537,16009.416242480263,24722.92263318226,19189.890721764787,24683.705818314953,20989.732351804592,20569.866983400992,18470.90664070533,14851.755901997987,25891.954823722892,19819.186826010166,18921.820235243023,20522.576037546467,27525.961934821684,30713.034955646464,21412.918614981576,21344.870107122344,26543.94285666588,19942.69654945001,23180.52553450498,25225.421682786062,17217.841128593398,25342.758536033023,22647.27040540609,20311.586220306715,13360.715771642963,18725.9052067595,21814.111731615874,31110.03238839381,11170.209355889527,11624.972537489217,20618.808518471225,26321.87815471064,18023.46871599901,20293.60245322545,15351.515268991054,23732.8659965382,19641.494693119246,14018.69083089969,18476.061736021245 +CH,96134.70662377033,78280.50065735803,75567.82823680741,75624.52378636891,84437.3353706146,82858.22758683441,70824.92066589095,89311.09471805945,65693.73271567532,95736.9646979805,81823.38979168108,71941.93730231738,93291.72439376163,67130.87226350856,103459.16116112942,100678.84780651607,97724.23622167954,103707.47043615916,84031.35407618036,103162.60292359442,67550.72288370335,80436.85045155323,87045.62560332651,91431.67032188186,103676.61542793321,68303.04781353423,112322.8911282786,90003.61666202937,91028.65800738563,79435.56403729375,84362.27602442491,71967.17653868764,71356.17066607098,70327.74412176231,72852.24726422041,60501.17532436925,83372.98013161327,74234.0349385613,61890.14596593614,54336.52856303606,58271.018596959824,62644.46734396249,75790.31562737533,69637.43067630196,63290.99710001813,60360.96649160057,57476.7610131337,62343.013781750116,78693.52307555292,73667.98513148604,75122.7031133837,70327.33051388402,33043.59766466136,48968.55545142022,47632.62274536815,48588.59714178414,58398.342285916435,57742.38347363473,42559.51792407362,59408.23894223828,57129.80243604273,71163.63652517287,70858.4268509181,119263.66699527185,55000.17388464577,60563.73895679769,51249.64855691976,45607.547672961526,60542.786248766475,50488.945849740405,62587.04259357905 +CZ,5741.811363153907,6430.8701452824125,6608.002211449021,5547.98412657507,11139.03676570244,9975.869136026628,10340.80877548634,13532.930674695574,7705.735677184478,7812.801872929193,10964.355351867012,10646.504003233118,9719.560436989883,9722.51782291833,19015.37334991238,15549.667939948402,15927.255735311868,12882.876953226223,10353.403919517015,11853.576269226234,10677.61771918624,10413.95103805089,6895.283030051313,9424.903567027312,12922.392855988994,9137.874616036986,17532.348544152504,18533.830228515635,15250.542039808313,10190.043701741004,11537.335951365127,9452.574932230946,5833.228549596612,5329.206957903766,7580.6976645678615,7069.4638232187235,11641.01844987317,9790.145937530902,7320.767578365769,5406.873869997462,5625.637231646826,7239.835546851702,6440.471526964078,10089.617439184036,12468.55796361433,12603.058166229213,11520.381274473184,9364.18293733304,10411.740148030329,8903.525346614548,9434.353552398077,14463.707947162726,8297.335864348554,6305.0428212307515,9383.102792968806,9420.905095490103,7053.530866655102,7934.504156979587,9131.766698685136,11027.709419784474,7973.011990884055,6406.065926850977,10728.184944007047,6645.433895571247,6345.82145823084,6499.675988954033,7016.463805946017,5619.135413529,4807.786670051716,5785.96568493957,7986.497102061061 +DE,37514.09293381361,41854.44001450301,38958.042582957896,35908.26729908446,54302.825387264325,53900.0627595872,49537.22949846474,58655.764340031456,37627.8257805545,42967.11131060519,58392.899587658314,47246.42956098116,36543.949785170116,36184.44504543948,76454.75505815547,77563.28089874369,65706.46393994696,62713.75591115003,50646.052126013274,74501.93068438585,33965.80961685558,36936.195615288314,42537.597915180464,56616.68921207296,56744.68927682596,34807.98202654087,56642.02192030585,72249.78534694383,61866.53633607989,55155.47864420242,60392.13262306467,54016.67652252674,53772.61604029787,44957.681101902934,45340.54136937088,48213.458592237475,66703.44582319073,60862.17019199549,41014.24992391146,32890.656129618095,33561.7114818598,35496.398391557115,43529.12222860253,55750.09930602895,59293.24549493131,37708.06634825865,37073.71374151417,42227.76427477898,55642.269571505705,50261.32889407123,50099.1653624443,63663.80803911536,33932.84222448403,24148.0930626651,39327.588161226195,34424.75266875371,37774.7468462978,39904.36386097259,38761.37664135111,43371.39725264248,35879.97041420226,37163.99533546333,50391.241894760955,37073.97568394232,36139.75677566728,42435.96482981362,34097.15586833773,36767.50341472353,28228.762854078665,31841.550922412007,39703.58319343355 +DK,540.6494487174206,225.87222027671936,240.57818634617777,370.37587945377,297.48979685029366,176.69208006156848,208.79213770008658,418.4554011794129,292.8102234525168,354.1220188699609,405.97396851309696,497.6042583966885,394.062076725445,291.3295691524654,304.5438623304028,591.4756757289098,629.1912050409122,479.19182164984767,353.75286712188796,454.0799499391422,338.48620130918835,357.676218196066,271.5200039604648,288.04880490856954,389.1757124784286,200.03398963977796,391.555913715114,397.29527898061724,379.8043799580585,514.7861201159348,505.7304478209306,381.57816134966373,489.3522161731293,301.717309458533,340.1024114463131,377.00536197863613,349.05617800662674,489.4227950883326,223.23589869898433,269.17110101837187,288.12441629799764,229.5796362727797,293.635637901186,464.89056739838395,386.583686695923,100.62743648672041,155.50324274468716,357.89273417380645,451.04208007822723,403.3432275318107,376.8995908622044,395.3896113505339,169.30459202493293,211.16687292699712,238.87912856891634,225.9981737610454,469.5229598902774,334.5021223766927,213.88993390188736,267.6283833775911,319.04746406617295,376.09256801086406,317.617725512616,353.26516627601364,404.2285931109035,408.5235836651241,351.86517079981496,305.15203377590285,361.8170659996248,441.45504335438466,312.14518786073575 +ES,49381.7181466273,42506.42487287232,29823.22966835847,33520.89863652133,42189.87541544434,45018.59980204108,27387.4917761516,45602.58407624375,62626.425244916936,94141.56249288519,66884.65469167147,67115.06386526441,82553.38984370838,62483.82642486853,54633.78478265801,97009.71651609142,47073.863176177474,55800.125131342014,85746.55555964066,72028.42027117872,75395.28981871007,97165.39237066667,52196.13272216457,55183.75674414718,44434.367388636274,45688.29479796517,107175.1885102772,376352.4882722023,122222.92956502433,46652.32256081546,42283.296882354574,45745.627884904505,49745.69531403085,62249.891346371536,58920.93920695715,48705.91089393512,49506.55303300167,60104.870297070986,41417.79510072781,40561.22111509541,48584.70074175896,41980.59062088254,40901.640882458836,43899.80095828514,42846.07384845135,68844.74333042726,58684.98840324564,44556.941393617584,38670.78308215774,46815.51322723987,68212.19494117319,35011.91030341564,50475.76713743041,37227.78564631605,21874.579431867896,35771.95151620915,29462.81813629621,28957.890173503936,32351.242924601538,48787.03990347552,31409.59113067243,24367.630996980257,54440.48961905899,50212.19464108009,33563.81874118821,47043.00929278407,21662.04001863739,49011.344970588914,40719.843457258816,40971.905822060144,45241.676643808205 +EE,1013.5168183220601,680.8524541483096,896.074617741102,657.0814978199126,1138.8891299468455,1033.2360728791764,1360.240006093535,1004.6950457068739,934.8774758153143,610.9977141696028,1048.0576885162222,1477.5046721309996,819.8347754595931,595.9919701101757,664.1717649435486,1178.672290642306,1112.7122616720162,1003.3658496852601,739.8499843054013,970.7396668771637,1156.5986195943308,988.0643633892423,963.0153554345143,973.0118510683724,980.7669857762866,637.7376743732589,1121.0664931638019,1485.3090062847518,1014.4287893275732,701.3843644468006,1196.8127434491068,1136.1215677919786,956.0911286121622,761.6620263988196,1109.6559596693944,945.8137139982223,1246.4435124663337,1081.0063689141978,1180.9951778617858,1433.4526339800395,1044.4962572791317,875.9289554452553,699.3911738148247,770.5276589427534,923.6083617038047,353.16555295606884,572.4961222214871,770.5595680360855,811.3520528619258,480.6204379373417,526.2783934002991,624.0252970776874,317.7449195254128,502.6073570248072,523.2149143909173,201.10844695565802,516.5064948459781,787.1638506300568,929.6905631193786,943.6792182991617,682.831069893961,933.1072250631543,692.097322772721,449.72254117947216,392.4173074585239,635.9676582877272,753.2897252561945,692.2577760060705,496.359994444626,685.8431112730673,512.1207984775516 +FI,37605.30609105331,44212.92045299994,43354.55443946037,40909.90249435943,45502.4817028443,31224.476747778986,41198.39417754888,37433.6293362709,28205.152814527217,22733.614552593714,36790.55908786393,45364.155643266684,28818.806516416316,35920.468538637644,43898.27049057533,41064.81684006838,48740.383821508905,37939.33952866352,30918.72594548001,32676.177851908644,33754.05667642144,33605.2810320407,40213.26757169686,46999.65997165162,40988.91471061698,26205.15827640759,41167.74001571865,25866.031354509407,34538.39918373158,28474.82496101674,52752.27216474974,31970.598763267328,35752.18054352296,34628.515223969094,27958.317625583102,31764.196982878522,30015.16030263079,26346.04561795855,37548.80632649389,27745.021972655202,27337.35846234453,42346.93155636462,32896.698992597754,22520.879762748657,23208.41547315833,19884.31515309925,22475.195958848508,33947.011695157635,26939.65172877202,32198.274008146243,28319.0878193112,25068.601753935876,16735.56947336933,29510.104024439464,31518.153863523567,22455.334677451585,28461.18834411878,38706.997170480194,29619.48059832548,29700.763598820195,30127.95453034573,45678.82538666528,31580.034602099753,29256.819253905658,43076.06538538077,49489.04536337738,37281.852766328535,33818.560580958125,35465.53242057822,41410.545080415,41567.72109759498 +FR,135094.49763612094,106772.95460342853,80280.31347296045,74490.3783289433,106687.24245156738,78827.6552216503,82482.11027901788,110101.24506868717,102231.57173723538,134019.59611286796,107848.78158188045,101612.59295931137,120079.53146371644,89568.66448178052,139585.30725052828,153772.55334552698,108799.90779570332,132530.25240249874,145056.24077647954,157344.40874499045,121182.68180087653,135378.15307040678,100408.75538533325,132164.7891142495,122382.6612084835,103706.50937621434,174240.52624537304,149853.10658224527,139992.58441391244,124892.14391178945,138893.07653615897,136102.17909683278,136298.85434619425,112484.75227834118,103842.75366698352,105453.40277549413,121935.76619549586,138379.45234918175,71924.60006455882,68830.14474115185,82603.77634528138,113556.04015089742,104293.46882473862,133822.28230897718,106263.7222156586,102481.34621013676,87080.48957235571,78895.08319488679,105735.9820525235,105626.7734987112,124313.3006359069,92144.11932119,73537.29646152578,78137.60525599915,58286.814348802,68928.05802270533,75743.7796112372,83489.37862547796,61518.93775410048,71311.23184506799,58810.65089438103,62581.66694545332,108566.1756738168,106810.3035898108,64981.51870529863,80829.27869961815,55203.39988738071,99034.47679069567,69318.43286028007,80169.78942387344,91015.86505778233 +GB,30949.55733300973,26701.9428167181,24834.423901258084,36292.29933386541,21223.79048684261,23695.358277919488,25987.57619398849,29907.138025215598,20532.262685247148,29689.355990879158,29755.7349738537,26456.609914217806,26827.642866860373,23873.145060634775,28779.074240844395,32368.78110647752,35977.394307138864,31071.326823875082,26446.212227329706,32888.07880400224,26015.735528737063,30161.596277105604,23017.874991984525,33287.290870143,28286.810808036254,25581.658649545065,33234.01349421005,31886.616851943854,36147.13303532815,34133.94123938026,36553.158784009356,34512.41082508442,31536.090754720717,28572.314934249887,33275.60304522956,33665.001595891976,29575.879486696827,34513.91395990196,28917.71257427291,33609.176553253485,26240.381135352633,32589.192149993025,28862.78621634025,32807.4074928576,28488.125666145854,20034.93309228305,23116.389736102017,36396.90134863421,35532.192401610075,39288.48780281492,29384.81519197965,36157.931892943554,21622.405196644995,32040.743170655453,30471.91526631006,28610.855281883465,33870.42864117695,37281.10394972126,33227.13302814586,22275.90424618414,30588.672790028984,34121.358436958675,27706.739428180073,37230.134388487466,33699.78058075289,31403.25713954576,26791.643973463473,25928.83232662774,30294.933954035565,36884.703709615125,29006.326905891798 +GR,10056.853784361187,12820.948661755581,12880.09713386794,18009.367077161973,21378.182658177288,29473.438101212807,11770.791493072751,17561.986308683798,14068.336209442465,21036.22984593777,10600.651816726047,19253.94027167335,33903.71393281602,18899.558983910043,17444.093386496537,23383.013428670623,15695.74724949063,19981.88173635776,22837.72155515553,20477.278624856677,21882.21250746782,25630.65513791745,23207.036995448034,22044.998521881807,12010.599831751206,19432.667576547075,10483.216379681078,17709.119022396855,23856.615736538963,24868.152661457774,21212.948012018598,19349.21918097175,12988.230031417628,16765.76278511076,15711.921661553248,14768.388537620414,15953.652684601051,12434.694106124754,8265.22849298033,8609.897588471382,14667.4570613014,8511.641075667938,10022.119539267658,13526.966234241496,13788.166042309897,20514.65575034389,13890.655920759986,13637.35506363307,17713.539092926785,10603.937587040999,9085.188935046803,13431.534902143163,18237.093734480102,15007.784707319466,15322.74478671021,15189.508916238075,9627.363919501071,7964.610286111404,17730.814440852053,21625.83796282357,10692.331430067703,11630.957871879938,16037.311332863352,13770.795305947142,17211.823293162783,14042.921372422215,9179.230226001768,15781.789228215457,11036.807907632012,8148.5700551254695,17837.056638784255 +HR,11579.119396857217,11616.785819262946,7841.411279590863,7071.274416413621,12992.083400898755,8164.444623202054,7347.788717816701,10332.483465428491,10537.238741811467,14479.677080073134,8479.661662510043,11787.072656273116,12862.758623825672,13371.003032039336,15990.668161905827,14470.61427957732,10644.752476691014,11929.91211438134,13611.162472968235,14159.968181810045,8748.9998436583,15752.17883640428,8576.387915317433,13121.48137750286,9754.915357083264,10574.934113880938,12339.192872828058,13526.82683401755,12407.125261525374,13314.467203516846,10589.407510915647,8240.264291214513,7438.667886688274,10422.72691638502,8167.94266416981,9179.047407118169,8517.079324039581,8574.526688740114,7000.526114186264,6044.501432950472,8424.638032244015,7640.079353180473,8493.38568955205,7673.466816227631,10113.96973112034,10475.583854629525,8161.447547783003,7730.598307694147,9688.392563588315,7721.732973546704,8547.91442119075,9110.630144082634,5654.409914846469,10449.908940883637,8544.776426500835,7703.363550389989,5356.181480736791,7888.76450496884,7981.496673656482,12159.302077747214,4232.3935344941665,4890.509308451722,11184.320641158047,12936.9950624162,6984.475921997467,9245.055929521772,6446.204647314787,9728.03403894923,7468.897556955247,6117.536838412799,7342.634549696792 +HU,455.56009955160323,446.6314758325084,756.8984035642842,450.9800559529234,596.2398492658431,618.0772176598714,433.5306920564666,626.6699838105968,736.6623236995524,901.6628598550868,955.0588428000881,1010.9577650474141,1739.1969813800204,989.7527773708749,1794.409065735425,2141.779921355298,1587.224990709883,671.8419282109866,1725.8355118081338,2716.2211930211206,1024.7764676942138,1478.4297579371664,1174.856199808959,1380.70450266654,1412.5023358393119,740.4722294240396,2077.892875467242,1382.2122922975911,1644.3319455706533,1323.8387691522125,1268.4319727220777,1050.1561979992803,938.7640185853782,701.18673897032,1176.2218406698482,1296.5341770126697,1360.4975641937087,1113.61620590335,686.0888933448485,340.9577501900905,585.7771480398817,710.4671894364953,683.5190908209308,901.0768559915952,882.6855336512624,1696.8667744778322,1072.391017559335,1091.9628754375356,1986.255314199332,1224.1891103332646,723.3105004079888,525.4091579835733,484.84230756145246,965.526984284363,1182.1048527111184,1188.8833398485922,413.588170351592,480.2721461089437,695.3047160171476,1875.1966820116616,1424.9690036241302,335.4105482008909,1504.9089491398734,1485.372787096725,1238.0161031995012,1379.3214547239281,769.0933902263423,1164.4795232335282,401.1056623458921,586.967649321741,686.5594518266915 +IE,4646.742323576806,3498.3840852321937,2544.6578209214726,5419.077105100732,3195.826410242899,2135.0862825425293,3896.6031468108818,5166.0575828547335,3059.5430902455864,4420.523688067632,3920.565970045664,2677.658691610785,3238.0778129090736,2814.899041865502,4169.640537659651,4975.659864222032,4131.237385306254,4051.6063579500055,3501.9810380506974,4162.222738923252,3277.97066925379,4149.907266073347,3535.126191101705,5127.397347805918,3672.8392481603883,3233.4968306810615,4670.955668853978,4602.662496185422,5486.794576286672,5974.860239294209,4958.279351735843,5539.220771177308,4814.939874910363,4704.080605250908,5214.372862088697,5579.052373684467,4424.669817520398,5645.170219782959,4099.107666436778,4291.534915722901,4394.7348113611615,4474.842169491055,4574.431496810677,5809.6920444413845,4586.044007664658,4500.049573188483,4158.002042556449,5864.151573833777,5389.317421868625,5602.682945402753,3910.7950042057882,5664.313605042361,3809.963053559503,4119.424688220842,4183.3641097691625,4756.243128847813,4885.19842669336,5839.576896771334,6529.030637616166,3851.468550158534,4739.247453475783,5116.873230654843,4346.868803644192,5780.6460718128965,5283.18966576195,5253.677788447396,4182.318532051922,4579.528423701424,5332.55258145869,6160.501534866154,5064.7000821427855 +IT,172598.9057049173,100312.33974168803,109683.21050744873,113141.95504995789,104153.60992530867,105687.75931391625,108887.88674499927,120408.08444781008,120463.4718424679,176044.98820314519,110983.47718210719,108661.3888675,154479.99952596257,116866.21274767391,140970.07450678368,148795.1431459142,123467.91624840492,141856.2012294592,135145.74559391223,121025.70232844367,123064.2692081519,158675.8479818625,126408.22188494769,122698.61897984576,134048.90222163082,141351.40587318933,183551.69201916258,152672.0987165877,152463.24070715246,129214.69623803807,126406.52003570941,111266.52387106014,105493.22493839792,124930.0152276459,118811.00097800085,107475.13484215035,108705.61617459086,105553.12911266208,89696.47047566871,79022.57929325203,102862.21537887226,110384.19134436303,114736.91039530469,100806.28542965792,84803.88252804938,120635.19844583895,89284.70639754581,91327.36842742827,112272.49254052607,121214.98394688763,100240.69732658492,110327.24980482439,67667.3134977533,93496.05161894935,74054.55044130566,77775.69469931335,75858.44303177306,92083.61659880447,94760.12813659209,130298.81027721112,113068.08204604911,102862.27823880903,124544.42911673438,173919.9577739064,87247.90399947706,91232.2769509342,68511.40136893332,110593.38607511377,103785.63569256017,87869.12433673623,92697.61009264145 +LT,3126.90226977831,1475.6787373817235,2475.307120672793,1105.461669987713,2895.355169108127,2328.291848466701,2773.100951693534,3565.1300060634676,1726.340017569786,2441.2869150017605,2739.255286641479,3393.627534013301,1948.972938917477,1503.9304931932238,2081.2181025571476,2387.4356286402945,2953.7202011167815,2442.643404228762,1575.9037853337413,3106.077653607497,2438.6998492607095,2321.3335398002305,2455.4955733417105,2703.979150849733,2983.428649152806,1271.81865757996,2291.2165997279003,3624.3054978962264,3066.52482874976,2692.4730181815085,3077.131323187848,2583.193918114293,2346.227785803459,1679.5438811966314,3224.9829735377853,3118.546035682447,2841.2519919875017,2793.689370450914,2688.2457239306805,3241.443663108373,2559.3234295762136,1702.002348908642,2110.46892607847,2728.2093372090403,2507.383271401867,987.8894968242066,1210.6320667482842,2619.0625969612056,2041.7494156189375,1077.1024719580128,1501.0630179433206,2093.0805484121715,761.1714789679556,1438.6161577229564,2052.75406588365,780.7629955514127,1806.053023318598,1637.6952437203324,1294.75789506843,2045.7605537026564,2014.9263059368886,1513.2399919084748,1700.3251444304037,1310.6134461272318,707.068706016363,991.8607300754669,2210.5243412620116,1945.761381971759,656.9514438734058,938.7049363615082,1091.9224609932446 +LU,418.42920407960173,402.65326898169104,245.66041042705334,140.5213247873591,281.65083355306945,239.8578158452654,369.22101543741394,483.78479198242695,296.9130979040409,276.1232951910135,459.8791399297912,360.88050800564,165.48000564979304,183.08661927563494,549.4327262552889,615.540914852196,430.5724249074097,464.9649192455573,366.43220700786026,562.8555247965114,228.93628708423907,211.2546971696525,190.68337965968735,337.11947715404034,376.86900759463407,153.06162250371963,361.9412034686643,457.17182659872975,474.4467062976349,501.35146460886807,586.4359860050876,446.87720467222556,554.4763793156474,495.34599197145104,293.22338952418016,311.9404169698979,470.26963100793296,539.1966800252237,318.1508533879876,216.80440341571304,213.96493869659722,161.79868975175182,244.05142665329072,449.452274496719,514.0163752203341,140.3179752581265,173.96942733863747,312.73695003201067,389.83220575942556,401.09422881834865,533.6829497064967,363.9324325652479,243.9448809708681,141.83825708815394,174.8785415794147,180.93947242990794,337.92060394534246,342.3576188750888,243.21751195509597,234.05807708972188,220.32797632404777,264.38928306703747,336.6463620917912,349.32621250528973,289.85643977541105,421.5786735040934,195.16848080033785,348.6723038250213,181.01976524286118,391.4073725411393,335.53788814460387 +LV,2742.6556866506367,1837.8325027812643,2554.170761349389,1408.345733539928,2983.189997195677,2825.061612760662,3252.3974243255384,2886.2924715658505,1956.4363533791625,2047.5738516633846,2396.823663944175,4066.8157610189646,1865.1937720377932,1213.3814003625498,1799.0008428733133,2700.1570058485836,2685.678969351212,2489.884608424664,1643.1417843905015,2409.13500668555,2592.9272177648704,2235.207139152776,2147.227486416904,2582.289866216919,2592.1022950383003,1400.2389662856958,2360.0229699498527,4017.8529783072768,2384.614549536931,2229.7220033855633,3208.439621353169,2678.7332051326716,2668.5315414635693,1705.3124319563372,2816.1467857724074,2779.7212753206704,2548.8356765576805,2583.5815153824487,2861.3233286998366,3801.9993997175525,2885.329135114707,2265.350140707051,1837.2956559244064,2087.5337557102102,2249.8356476827344,853.5386928200795,1689.684828383291,2653.3751344813045,2109.0096432617606,1182.3316650686795,1487.9132295244858,1974.1178903388873,1027.3242775723352,1736.361793878124,1741.9204323497202,701.3946960765046,1526.0106468283361,1844.9956663852768,1722.690767934754,2194.735679264339,1757.0892689497684,1974.9007736328324,1883.6056453393867,1405.6090440523008,1355.4439688992024,1295.7921755717964,2182.9434502570357,1620.95881111662,955.2966396067089,1490.1655775988,1270.5915407733335 +MK,3934.258904454459,3692.0813330320566,3668.815892166788,5481.091007849558,8488.277349663611,8514.814821434962,4861.071723840239,6878.625964986481,6399.572820343104,7510.160197503697,3410.068805763449,5394.1653784487335,11852.098687247973,6238.562291909563,5957.282159254423,7239.168545674971,6704.994620967638,5343.254672711665,7690.122171456297,7543.1707758996035,6080.923842798452,9518.22452251825,7931.36999814296,7898.373553315725,5296.06496225987,7158.993856907015,5568.700579027411,6653.618095140364,7658.60790313609,7384.3372015299765,6359.355811805509,6311.4116310067075,4303.248848133382,4384.047563174607,4091.082807237865,4939.915122052565,4226.085259691547,2768.0039025432975,3428.9467644601264,2294.222990992098,4834.495252793617,3325.6396521134557,2522.690466358069,2976.927612194815,4721.490759581051,6218.919122814915,3840.4729075548557,4211.680380445357,5623.802967061423,3683.311545054556,3010.5387111938044,4030.600677147787,3746.016920616422,4369.907262961726,4375.170573617275,4171.022704527618,2702.082556773276,2500.5709190188645,4370.4076446296785,5204.859735574909,2501.3689966568136,2522.4142074268602,3898.568577966785,4120.496629723428,4145.529030725163,3980.5145191045517,2967.8383253548755,4325.530795978238,2351.320803369623,2869.298937488066,4644.825806471018 +ME,11787.036249138713,18552.496612764233,11097.228684668844,11567.930061084586,19293.29710814638,13959.20571478613,13046.18440612437,18177.491235848003,15681.709335959564,19349.950981118494,11658.5954682407,15979.557715971854,21966.784229466284,16319.44299708603,17409.40677448169,19267.118980997075,14169.834731486397,15882.152400499632,19649.60512617265,23317.326436161944,15993.83236649745,16724.197023812518,13680.494674103275,16947.38155713745,13396.216553276576,17144.464963358845,16187.204710073258,20417.876685607593,22530.120155126962,16197.908505488407,14818.545420474018,12719.274460270883,9622.464989039141,13369.875886578782,13078.849292203078,14951.85122042927,13681.691957367619,12435.207850184568,11885.89235643587,9516.042262408026,16926.063160570764,13462.909968102742,10042.999218445517,11916.714437943356,15158.632522247093,19735.53511234594,11491.695345503264,14075.890065382568,13952.132515246369,11677.32833931845,13022.018989752623,13358.031703205903,11534.177624564669,17635.48411958614,14124.491791464301,12392.738752899031,9187.905148236658,11106.432128782126,13780.482580236776,19889.70854314474,7004.765863876614,8204.168680103983,13925.208337027952,12712.182323875111,8528.030231315688,13924.890575378206,9037.809310147335,13972.095665379566,11360.358971424077,10280.43536084397,13775.545442858487 +NL,132.53876435280281,105.25304624029256,97.45390390948565,59.629996943856206,126.17294913858377,100.92785922135535,110.87608266899032,166.47617616014315,115.03831975395074,102.21679880589629,222.71184206736731,164.32395798051633,88.95057195947412,70.61286255591416,116.64203846425596,229.2598570422647,176.71083656110923,163.34614915846544,135.23541074052787,153.03725726999411,84.2996330000842,75.50229060079856,87.79575328623356,149.0911400069578,185.89892961221338,65.86854604814957,122.15247506918442,163.86971747802477,177.38160465294413,173.97671792234624,210.94537197363266,138.15528852664815,157.7989701791954,164.8431327190436,156.27033317337316,144.7881412408737,172.55396177597567,195.57239229094031,113.32208112175368,57.91841778157319,69.80575559844598,81.95247611107445,154.25504072397393,213.16593963287744,185.6861094126344,50.88580789479815,88.6770750590963,180.59511129061775,175.09641999581325,134.73866646771805,175.89672487625361,170.10222446753016,112.93557838603432,69.46018341018433,89.82211025011819,57.645723008783136,122.86182387892966,128.20825436811367,82.72980960226074,90.07937846739043,113.22245968360778,109.86527486394354,113.13774281842522,111.87536788688416,113.53271254663122,140.01537175074324,68.64291161119651,101.90820271032081,52.313311323047174,116.8375292031251,120.98114114085548 +NO,268338.23623559956,311516.6976002237,354741.82081862347,287750.3093395867,277336.7794878445,277384.3177020413,343804.7636848019,309909.39898200217,279870.8316011512,240316.6988310159,337135.04130569333,320967.163231472,302346.276731951,373063.16157364036,335043.0981929385,271478.1367425455,378240.8226290755,318135.4534804216,235625.6466770364,244547.82624561962,342177.033729037,330660.24479289196,373161.4848287328,318697.1789732917,345399.49658985186,314419.8189439592,230963.73709983748,236070.00307120252,312676.53469423676,195102.01076090848,231756.2941107393,196457.66930311252,255890.52889680286,207980.18388759278,250218.19793663023,221472.83820330602,212315.28230042476,222774.69138098884,210372.25250462766,210846.3448041017,177438.35447152023,214108.95671057343,190417.02324135366,175558.31404488848,199041.30532793878,154342.13931409398,175902.47007757265,215120.2640019617,216733.8944940455,261370.58822336013,212333.63219332727,174324.40313248086,166539.1770850326,202012.88317391058,238213.58919108333,170401.06338583003,217109.81457913358,185684.48673203168,210764.408869451,180452.94255978477,359959.68381729786,220133.80814962948,201815.91045998363,193256.2376798832,225141.35101977925,194502.14845672756,241700.45583039062,203049.47691663646,189440.21996991744,241270.48318453485,196628.83730256284 +PL,10323.210329129464,10102.880830361863,15243.556644277647,6316.576710839603,13844.440254769646,15125.266765937784,15697.445163802722,20547.692524530372,11483.751822471613,12736.801698643669,14242.313560202334,19975.533453959804,14197.61841088385,13385.258603233342,17555.946503015533,19526.389852009346,24990.22970093532,18554.95543612019,10610.557656768206,19356.459381427685,19657.930603206125,18732.50311338716,16842.606570419066,21220.5523273715,24157.44207211756,15607.0096942104,21677.41127186343,21298.26291533998,22732.35472759213,23096.919661671975,20539.587419825057,15690.593097829358,10514.527177352287,9757.003902443616,18214.127020951153,14743.470132504332,14160.372458977477,15574.81487531224,12340.373515759937,8851.92371521615,10791.411372967183,10330.247763133175,10795.54750377256,14738.227630388099,14374.438386354948,12177.648415014886,14450.3889852054,16473.394758962142,14491.781691174727,11650.64866730713,13724.64695911459,14468.081620316902,7425.533290488975,8039.550610683221,9468.54351850099,6656.34004885665,7842.310345436549,9593.638690519654,8828.256800239144,15874.058827778785,13537.338641985261,6174.396712827303,9509.500411581042,9436.102173346171,5360.954908688417,6309.399306468634,12402.024613286878,10602.512257049677,4725.103685514244,6963.3130148445025,9124.964758601689 +PT,8372.129135066036,5969.31083523099,2397.6586063764958,3916.9962424030286,9654.699566273128,9603.363148222781,4163.162870337118,9179.465624121007,12131.854156488425,23406.646477036124,10635.077955993413,11029.404714084018,16475.917880350513,12087.55769685171,9804.492049893286,26178.184180984652,5669.820254013018,7559.528320921835,15405.930172416956,11163.387795923936,7088.025310785357,10191.330496756716,7128.069257230898,9335.631563569985,5236.13128329125,4572.251664471752,32605.577182903042,149323.86523586328,34294.81803755196,6061.11721308257,5836.654497517657,4843.7020037722705,7814.782411245904,9395.759000250582,14057.157961109715,8192.288977642056,7655.760207241933,9203.645337709624,7637.7450966202805,7971.812432879156,7421.37678917759,2935.2871895485287,6329.939163930326,8820.796560748415,8944.109217212255,14719.27363438586,11050.779819799696,8379.75073967695,5590.525909815882,10485.131539563748,20476.028147427794,7715.162648805314,11939.81827695821,5116.2596877306005,3047.0903835389504,9273.489011971804,5135.313604526994,3331.418010762715,6338.048693933388,12105.616460835923,7097.468638389797,3348.213951094401,11773.920694867731,13465.586888738457,4379.602415677836,11394.463621230614,3104.8789319036796,7216.150544702967,6653.765864361187,5720.580737490315,6685.493238874142 +RO,19346.344374357534,19295.50358149612,27702.617554877004,24644.67023787785,52797.882559506776,42262.7162454671,34689.01143849651,42583.74114324521,28973.147767939397,36882.64868179707,29063.471368254675,39575.626278602576,31898.858414072816,36283.98261870408,42207.50410750205,41714.957670180156,43270.313090823794,40474.54716426176,53565.91349208634,74865.02611452111,48190.11154819992,57257.26135564704,46523.80416861758,44300.83791323246,54901.1701096736,47361.27151969814,49156.09266449866,56461.30009557872,56062.45724028393,58506.511677323855,48499.732173446595,45901.64074770707,33171.12855766741,38310.49272614435,39377.41586450999,28638.803870927226,29424.553978129992,39429.14347226504,39300.73151290347,27525.04968129815,47095.77682827757,30948.90232192865,32939.14262154816,30155.99148400138,35067.04977301419,35592.25066656211,45790.67404045973,42596.77718301701,39223.35618808953,25767.93476452563,30963.38609617001,30034.908201595936,23323.05819720473,28517.276888551194,41560.85308154171,43999.15897330077,21427.19650317167,25084.131072037453,18450.764512377853,38182.39266803458,20422.517191285966,14090.243196787245,21998.55223596247,30809.187151501046,22071.90474198888,31906.454754357772,20763.073093214778,25620.747974742615,19248.368008063957,23289.011422937554,26669.846131314298 +RS,5968.673897869134,7450.873145908507,8457.218084424714,10466.134979346216,18700.932768467817,18783.264457747777,9647.157790116991,14650.396388121844,12360.14810912279,11640.673528541236,8882.446980807292,14762.619600006037,18399.930168952076,13646.806400326906,14489.119148714197,14104.804002759012,15462.34970923992,10894.228431882033,15095.088711819322,17853.281396922546,12916.46431873845,14207.379133135786,14418.834370720482,12752.5957180177,15544.588217112372,15820.904121022999,13534.055257847373,15215.146639151637,12996.626380040323,13029.919161637963,10196.668003968522,10284.440766030819,7022.693160251435,8401.249663323453,8435.803252725058,10265.915943082338,8623.546341166757,7782.429841191159,9309.259412625106,4849.32865233776,9480.40891035497,8237.330612702019,6688.9341497267815,7113.796951355988,8945.392918392941,11391.406447517316,8770.78174795334,7797.074404297926,11205.567652241964,7490.638744688466,6890.626375097134,8721.152535625733,6986.899718851523,10014.515393386338,11863.685949406314,12576.12226693591,6930.903954576986,6933.68818306879,8436.351253425808,11680.836931112251,5673.257881932308,4512.930178739657,6094.8735580904795,11616.485592765608,8423.704814584986,9582.260510301834,6280.960312130538,10196.683253538626,5869.428152205924,6637.126459584064,8223.324863109141 +SK,6885.47211985414,6956.591212284293,7116.928783228905,4573.269245439668,8169.130903254093,6799.092645889461,7190.199390340817,8819.311158887818,6679.080394471699,8681.76846617681,6714.688996801623,8780.135433744465,9674.052829336924,7510.89080672348,12315.672504913706,11551.288938768861,10658.095118642217,7760.467099195778,7334.701885838525,11743.053621084162,9101.455526104724,11252.254293590808,7089.2297957961755,11692.575497480175,11870.620362608648,8320.411050776946,12664.612697391287,10840.053026355938,9544.145158945727,8285.851845920657,6554.955003499832,6014.385824757616,5119.229272436076,5566.197178944401,8283.751441606055,4576.739002618681,5529.727054286627,4884.285979430572,7030.767074231632,6380.185010419688,6092.06334940043,4789.6919845964085,3988.728835990378,6100.445127127096,6174.336930810738,5839.492530664778,6739.897695678873,6073.547495791424,6239.663814901599,5985.626769812626,5946.910319392802,5350.4706690989715,2897.742723724252,3812.447551889599,5110.734824948982,5466.7571969447445,3822.1224333717755,4189.745228198809,3608.9397299109723,8925.718578619328,5520.297117401385,2338.023634704523,3634.014208205775,5079.021357670912,3789.3939110245783,3599.858750509022,4222.707040620564,3096.1701009538024,2158.8092391216674,4542.203594136759,4220.210061470592 +SI,11752.798116380785,9378.602450871665,7169.30972105447,8221.267552376507,10568.142173580174,7705.119699907013,8559.440940655051,9496.2682906945,10641.418485889739,13644.09405840664,8756.865265119286,10877.994032024846,13095.473005855994,10408.222250010325,17110.767148034232,13257.44359305617,10163.420442559087,10743.809603872312,11213.570708968457,12419.028940460184,8518.188189494229,14975.496301661744,10217.449212514946,10521.007791805137,10759.209828867228,9699.803414833696,14530.647692616923,14530.715084460895,12439.143395173185,11990.046930204517,9001.647305603035,10111.423546501868,6824.172499114327,10267.968760960974,10233.709492652128,9562.175707725224,11231.941186997657,8178.692540046128,8568.526412713521,8103.883097000008,9639.895318751553,9157.43571368136,6814.50158997294,8089.913064615838,8170.645053083449,10765.044719853075,6713.170108103339,8936.8362631332,9542.147586182518,8964.110093884974,8653.114721452737,8353.75428643612,4558.589088593759,9432.405903987477,6927.92152111369,7408.760392367348,5834.871486466167,9336.912851251047,8839.494837985663,10766.693939731143,5073.937522450624,6475.598951785893,9482.419717110955,13414.771212079502,5906.5584945059145,9942.151265372298,7696.092255607837,9521.835825354157,8476.124990692872,6361.010018809483,7504.908031281684 +SE,134822.12256461562,136958.6213484757,175679.6273876132,133962.36617697845,126787.36233921513,130861.49290945902,151133.2982760823,140247.52694114187,130508.52271890236,137421.63579450184,164132.0529341604,142467.57286064525,135886.87305352974,163762.3962091317,161128.38161511483,148196.28633931337,167892.87677861055,132017.696130452,105953.99289036485,119407.22923207263,147139.6039546916,151309.55114963037,164893.61323375857,137608.18237925606,149284.78771947857,108749.15747050248,131859.531763871,117290.48017785426,131751.97960784068,111117.27409251402,110551.615114603,81550.1135393761,114677.940027603,102356.46697333359,128977.80283219383,94483.98267503879,114937.96457142044,93256.91327642264,96751.29026217168,91074.34139788701,87931.96567208324,96560.83822983531,116746.94389793588,64626.25905820224,75156.91767736348,51489.37512699482,74729.26978432719,134673.01513808029,110440.65771030058,149033.9827088587,143576.83069821307,93040.02354987273,64075.160200006496,102493.08906248632,106664.82884715112,90351.43771324471,94801.81222609026,92445.93458012305,88919.20770998915,108111.72872590217,127440.80298482065,124444.06618883039,98961.91573013023,87913.90602563386,115956.48217115598,95526.84265190935,103459.08985036932,86058.90377748368,99040.9103276256,110493.22773914765,94182.7249029169 diff --git a/envs/environment.yaml b/envs/environment.yaml index 8bd7428f..f5cb9aae 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -39,6 +39,7 @@ dependencies: - proj - fiona <= 1.18.20 # Till issue https://github.com/Toblerity/Fiona/issues/1085 is not solved - country_converter + - scikit-learn # Keep in conda environment when calling ipython - ipython diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index c1400d28..f56cb9e3 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -66,13 +66,16 @@ import atlite import geopandas as gpd import pandas as pd +from sklearn.linear_model import LinearRegression + import country_converter as coco cc = coco.CountryConverter() -def get_eia_annual_hydro_generation(fn, countries): +def get_eia_annual_hydro_generation(fn, countries, capacities=False): - # in billion kWh/a = TWh/a + # energy: in billion kWh/a = TWh/a + # capacity: in billion kW = GW df = pd.read_csv(fn, skiprows=2, index_col=1, na_values=[u' ','--']).iloc[1:, 1:] df.index = df.index.str.strip() @@ -106,11 +109,51 @@ def get_eia_annual_hydro_generation(fn, countries): df.index = cc.convert(df.index, to='iso2') df.index.name = 'countries' - df = df.T[countries] * 1e6 # in MWh/a + # convert to MW of MWh/a + factor = 1e3 if capacities else 1e6 + df = df.T[countries] * factor return df +def correct_eia_stats_by_capacity(eia_stats, fn, countries, baseyear=2019): + cap = get_eia_annual_hydro_generation(fn, countries, capacities=True) + ratio = cap / cap.loc[str(baseyear)] + eia_stats_corrected = eia_stats / ratio + to_keep = ["AL", "AT", "CH", "DE", "GB", "NL", "RS", "RO", "SK"] + to_correct = eia_stats_corrected.columns.difference(to_keep) + eia_stats.loc[:,to_correct] = eia_stats_corrected.loc[:,to_correct] + + +def approximate_missing_eia_stats(eia_stats, runoff_fn, countries): + + runoff = pd.read_csv(runoff_fn, index_col=0).T[countries] + + # fix ES, PT data points + runoff.loc["1978", ["ES", "PT"]] = runoff.loc["1979", ["ES", "PT"]] + + runoff_eia = runoff.loc[eia_stats.index] + + eia_stats_approximated = {} + + for c in countries: + + X = runoff_eia[c].values.reshape(-1, 1) + Y = eia_stats[c].values.reshape(-1, 1) + + to_predict = runoff.index.difference(eia_stats.index) + X_pred = runoff.loc[to_predict, c].values.reshape(-1, 1) + + linear_regressor = LinearRegression() + linear_regressor.fit(X, Y) + Y_pred = linear_regressor.predict(X_pred) + + eia_stats_approximated[c] = pd.Series(Y_pred.T[0], index=to_predict) + + eia_stats_approximated = pd.DataFrame(eia_stats_approximated) + return pd.concat([eia_stats, eia_stats_approximated]).sort_index() + + logger = logging.getLogger(__name__) if __name__ == "__main__": @@ -130,8 +173,16 @@ if __name__ == "__main__": fn = snakemake.input.eia_hydro_generation eia_stats = get_eia_annual_hydro_generation(fn, countries) + if config_hydro.get('eia_correct_by_capacity'): + fn = snakemake.input.eia_hydro_capacity + correct_eia_stats_by_capacity(eia_stats, fn, countries) + + if config_hydro.get('eia_approximate_missing'): + fn = snakemake.input.era5_runoff + eia_stats = approximate_missing_eia_stats(eia_stats, fn, countries) + weather_year = snakemake.wildcards.weather_year - norm_year = snakemake.config['renewable']['hydro'].get('norm_year') + norm_year = config_hydro.get('eia_norm_year') if norm_year: eia_stats.loc[weather_year] = eia_stats.loc[norm_year] elif weather_year and weather_year not in eia_stats.index: From 5bb432141dd1067910d54b808364c66d86285f0e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 22 Jul 2022 18:34:16 +0200 Subject: [PATCH 046/173] hydro: add documentation for eia approximation/correction options --- doc/configtables/hydro.csv | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/configtables/hydro.csv b/doc/configtables/hydro.csv index 8148b256..6016e6f1 100644 --- a/doc/configtables/hydro.csv +++ b/doc/configtables/hydro.csv @@ -4,4 +4,6 @@ carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydr PHS_max_hours,h,float,"Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_." hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}","Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation `_." clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero." -norm_year,--,"Year in EIA hydro generation dataset; or False to disable","To specify a specific year by which hydro inflow is normed that deviates from the snapshots' year" \ No newline at end of file +eia_norm_year,--,"Year in EIA hydro generation dataset; or False to disable","To specify a specific year by which hydro inflow is normed that deviates from the snapshots' year" +eia_correct_by_capacity,--,boolean,"Correct EIA annual hydro generation data by installed capacity." +eia_approximate_missing,--,boolean,"Approximate hydro generation data for years not included in EIA dataset through a regression based on annual runoff." \ No newline at end of file From 16bd4b6c2cbe4f2f3b315b19ed27b4f8c804ca27 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 22 Jul 2022 19:14:00 +0200 Subject: [PATCH 047/173] fix snapshot creation based on weather year --- scripts/build_heat_demand.py | 6 +++--- scripts/build_solar_thermal_profiles.py | 2 +- scripts/build_temperature_profiles.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index fe44de16..1e165c69 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -11,9 +11,9 @@ if __name__ == '__main__': from helper import mock_snakemake snakemake = mock_snakemake( 'build_heat_demands', - weather_year='', + weather_year='1969', simpl='', - clusters=48, + clusters=37, ) cutout_name = snakemake.input.cutout @@ -25,7 +25,7 @@ if __name__ == '__main__': else: snapshots = snakemake.config['snapshots'] - time = pd.date_range(freq='m', **snapshots) + time = pd.date_range(freq='h', **snapshots) if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index 8bdc07c5..ec9e59c4 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -27,7 +27,7 @@ if __name__ == '__main__': else: snapshots = snakemake.config['snapshots'] - time = pd.date_range(freq='m', **snapshots) + time = pd.date_range(freq='h', **snapshots) if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index ab006409..3e690cca 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -25,7 +25,7 @@ if __name__ == '__main__': else: snapshots = snakemake.config['snapshots'] - time = pd.date_range(freq='m', **snapshots) + time = pd.date_range(freq='h', **snapshots) if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] From 6e47c711952ab94ac098f62d922420e27a27c3e0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 22 Jul 2022 20:48:24 +0200 Subject: [PATCH 048/173] clustered_population: add missing cutout input --- Snakefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Snakefile b/Snakefile index 68d32443..54829b6a 100644 --- a/Snakefile +++ b/Snakefile @@ -97,6 +97,7 @@ rule build_clustered_population_layouts: rule build_simplified_population_layouts: input: + cutout=pypsaeur("cutouts/" + config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", From d47a9aac72309420b8e941132ccf1c30169343c5 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 23 Jul 2022 10:37:32 +0200 Subject: [PATCH 049/173] energy_totals: update eurostat to 2021 --- Snakefile | 10 +-- scripts/build_energy_totals.py | 153 ++++++++++++++------------------- 2 files changed, 68 insertions(+), 95 deletions(-) diff --git a/Snakefile b/Snakefile index 54829b6a..fa2323f8 100644 --- a/Snakefile +++ b/Snakefile @@ -54,8 +54,7 @@ datafiles = [ "data/emobility/KFZ__count", "data/emobility/Pkw__count", "data/h2_salt_caverns_GWh_per_sqkm.geojson", - directory("data/eurostat-energy_balances-june_2016_edition"), - directory("data/eurostat-energy_balances-may_2018_edition"), + directory("data/eurostat-energy_balances-april_2022_edition"), directory("data/jrc-idees-2015"), ] @@ -235,11 +234,6 @@ rule build_solar_thermal_profiles: script: "scripts/build_solar_thermal_profiles.py" -def input_eurostat(w): - # 2016 includes BA, 2017 does not - report_year = config["energy"]["eurostat_report_year"] - return f"data/eurostat-energy_balances-june_{report_year}_edition" - rule build_energy_totals: input: nuts3_shapes=pypsaeur('resources/nuts3_shapes.geojson'), @@ -247,7 +241,7 @@ rule build_energy_totals: swiss="data/switzerland-sfoe/switzerland-new_format.csv", idees="data/jrc-idees-2015", district_heat_share='data/district_heat_share.csv', - eurostat=input_eurostat + eurostat=directory("data/eurostat-energy_balances-june_2021_edition"), output: energy_name='resources/energy_totals.csv', co2_name='resources/co2_totals.csv', diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 3f376b0c..f2e565e1 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -17,55 +17,6 @@ def reverse(dictionary): """reverses a keys and values of a dictionary""" return {v: k for k, v in dictionary.items()} - -# translations for Eurostat -eurostat_country_to_alpha2 = { - "EU28": "EU", - "EA19": "EA", - "Belgium": "BE", - "Bulgaria": "BG", - "Czech Republic": "CZ", - "Denmark": "DK", - "Germany": "DE", - "Estonia": "EE", - "Ireland": "IE", - "Greece": "GR", - "Spain": "ES", - "France": "FR", - "Croatia": "HR", - "Italy": "IT", - "Cyprus": "CY", - "Latvia": "LV", - "Lithuania": "LT", - "Luxembourg": "LU", - "Hungary": "HU", - "Malta": "MA", - "Netherlands": "NL", - "Austria": "AT", - "Poland": "PL", - "Portugal": "PT", - "Romania": "RO", - "Slovenia": "SI", - "Slovakia": "SK", - "Finland": "FI", - "Sweden": "SE", - "United Kingdom": "GB", - "Iceland": "IS", - "Norway": "NO", - "Montenegro": "ME", - "FYR of Macedonia": "MK", - "Albania": "AL", - "Serbia": "RS", - "Turkey": "TU", - "Bosnia and Herzegovina": "BA", - "Kosovo\n(UNSCR 1244/99)": "KO", # 2017 version - # 2016 version - "Kosovo\n(under United Nations Security Council Resolution 1244/99)": "KO", - "Moldova": "MO", - "Ukraine": "UK", - "Switzerland": "CH", -} - non_EU = ["NO", "CH", "ME", "MK", "RS", "BA", "AL"] idees_rename = {"GR": "EL", "GB": "UK"} @@ -127,47 +78,68 @@ to_ipcc = { } -def build_eurostat(countries, year): - """Return multi-index for all countries' energy data in TWh/a.""" +def eurostat_per_country(country): + + country_fn = idees_rename.get(country, country) + fn = snakemake.input.eurostat + f"/{country_fn}-Energy-balance-sheets-June-2021-edition.xlsb" - report_year = snakemake.config["energy"]["eurostat_report_year"] - filenames = { - 2016: f"/{year}-Energy-Balances-June2016edition.xlsx", - 2017: f"/{year}-ENERGY-BALANCES-June2017edition.xlsx" - } - - dfs = pd.read_excel( - snakemake.input.eurostat + filenames[report_year], + df = pd.read_excel( + fn, sheet_name=None, - skiprows=1, - index_col=list(range(4)), + skiprows=4, + index_col=list(range(3)), + na_values=["+", "-", "=", "Z", ":"], ) - # sorted_index necessary for slicing - lookup = eurostat_country_to_alpha2 - labelled_dfs = {lookup[df.columns[0]]: df - for df in dfs.values() - if lookup[df.columns[0]] in countries} - df = pd.concat(labelled_dfs, sort=True).sort_index() + df.pop("Cover") - # drop non-numeric and country columns - non_numeric_cols = df.columns[df.dtypes != float] - country_cols = df.columns.intersection(lookup.keys()) - to_drop = non_numeric_cols.union(country_cols) - df.drop(to_drop, axis=1, inplace=True) + return pd.concat(df) + + +def build_eurostat(countries, year=None): + """Return multi-index for all countries' energy data in TWh/a.""" + + nprocesses = snakemake.threads + tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries), + desc='Build from eurostat database') + with mp.Pool(processes=nprocesses) as pool: + dfs = list(tqdm(pool.imap(eurostat_per_country, countries), **tqdm_kwargs)) + + index_names = ['country', 'year', 'lvl1', 'lvl2', 'lvl3'] + df = pd.concat(dfs, keys=countries, names=index_names) + + df.dropna(how='all', axis=0, inplace=True) + df.dropna(how='all', axis=1, inplace=True) + df = df[df.index.get_level_values('lvl1') != 'ktoe'] + + i = df.index.to_frame(index=False) + i.loc[i.lvl2 == 'Primary production', ['lvl1', 'lvl3']] = 'Main' + i.loc[i.lvl2 == 'Gross electricity production', 'lvl1'] = "Gross production" + i.ffill(inplace=True) + df.index = pd.MultiIndex.from_frame(i) + + df.drop(list(range(1990, 2020)), axis=1, inplace=True) + df.drop("Unnamed: 7", axis=1, inplace=True) + df.fillna(0., inplace=True) # convert ktoe/a to TWh/a df *= 11.63 / 1e3 + if year: + df = df.xs(str(year), level='year') + return df -def build_swiss(year): +def build_swiss(year=None): """Return a pd.Series of Swiss energy data in TWh/a""" fn = snakemake.input.swiss - df = pd.read_csv(fn, index_col=[0,1]).loc["CH", str(year)] + df = pd.read_csv(fn, index_col=[0,1]).loc["CH"] + + if year: + df = df[str(year)] # convert PJ/a to TWh/a df /= 3.6 @@ -406,8 +378,17 @@ def build_idees(countries, year): def build_energy_totals(countries, eurostat, swiss, idees): - eurostat_fuels = {"electricity": "Electricity", - "total": "Total all products"} + eurostat_fuels = dict( + electricity="Electricity", + total="Total" + ) + + eurostat_sectors = dict( + residential="Households", + services="Commercial & public services", + road="Road", + rail="Rail" + ) to_drop = ["passenger cars", "passenger car efficiency"] df = idees.reindex(countries).drop(to_drop, axis=1) @@ -417,8 +398,8 @@ def build_energy_totals(countries, eurostat, swiss, idees): # add international navigation - slicer = idx[in_eurostat, :, "Bunkers", :] - fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() + slicer = idx[in_eurostat, :, "International maritime bunkers", :] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() df.loc[in_eurostat, "total international navigation"] = fill_values # add swiss energy data @@ -434,12 +415,10 @@ def build_energy_totals(countries, eurostat, swiss, idees): for sector in ["residential", "services", "road", "rail"]: - eurostat_sector = sector.capitalize() - # fuel use for fuel in ["electricity", "total"]: - slicer = idx[to_fill, :, :, eurostat_sector] + slicer = idx[to_fill, :, :, eurostat_sectors[sector]] fill_values = eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=0).sum() df.loc[to_fill, f"{fuel} {sector}"] = fill_values @@ -489,17 +468,17 @@ def build_energy_totals(countries, eurostat, swiss, idees): # Missing aviation slicer = idx[to_fill, :, :, "Domestic aviation"] - fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() + fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() df.loc[to_fill, "total domestic aviation"] = fill_values - slicer = idx[to_fill, :, :, "International aviation"] - fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() + slicer = idx[to_fill, :, "International aviation", :] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() df.loc[to_fill, "total international aviation"] = fill_values # missing domestic navigation - slicer = idx[to_fill, :, :, "Domestic Navigation"] - fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() + slicer = idx[to_fill, :, :, "Domestic navigation"] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() df.loc[to_fill, "total domestic navigation"] = fill_values # split road traffic for non-IDEES @@ -702,7 +681,7 @@ if __name__ == "__main__": idees_countries = countries.intersection(eu28) data_year = config["energy_totals_year"] - eurostat = build_eurostat(countries, data_year) + eurostat = build_eurostat(countries.difference(['CH']), data_year) swiss = build_swiss(data_year) idees = build_idees(idees_countries, data_year) From eee6651d12ff8ae87e4bf3a30119541c870decc1 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 23 Jul 2022 11:19:37 +0200 Subject: [PATCH 050/173] energy_totals: fetch all years 2000-2015 from IDEES --- scripts/build_energy_totals.py | 158 ++++++++++++++++----------------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index f2e565e1..1845c93c 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -147,90 +147,89 @@ def build_swiss(year=None): return df -def idees_per_country(ct, year): +def idees_per_country(country): base_dir = snakemake.input.idees ct_totals = {} - ct_idees = idees_rename.get(ct, ct) + ct_idees = idees_rename.get(country, country) fn_residential = f"{base_dir}/JRC-IDEES-2015_Residential_{ct_idees}.xlsx" fn_tertiary = f"{base_dir}/JRC-IDEES-2015_Tertiary_{ct_idees}.xlsx" fn_transport = f"{base_dir}/JRC-IDEES-2015_Transport_{ct_idees}.xlsx" # residential - df = pd.read_excel(fn_residential, "RES_hh_fec", index_col=0)[year] + df = pd.read_excel(fn_residential, "RES_hh_fec", index_col=0) - ct_totals["total residential space"] = df["Space heating"] + ct_totals["total residential space"] = df.loc["Space heating"] rows = ["Advanced electric heating", "Conventional electric heating"] - ct_totals["electricity residential space"] = df[rows].sum() + ct_totals["electricity residential space"] = df.loc[rows].sum() - ct_totals["total residential water"] = df.at["Water heating"] + ct_totals["total residential water"] = df.loc["Water heating"] assert df.index[23] == "Electricity" - ct_totals["electricity residential water"] = df[23] + ct_totals["electricity residential water"] = df.iloc[23] - ct_totals["total residential cooking"] = df["Cooking"] + ct_totals["total residential cooking"] = df.loc["Cooking"] assert df.index[30] == "Electricity" - ct_totals["electricity residential cooking"] = df[30] + ct_totals["electricity residential cooking"] = df.iloc[30] - df = pd.read_excel(fn_residential, "RES_summary", index_col=0)[year] + df = pd.read_excel(fn_residential, "RES_summary", index_col=0) row = "Energy consumption by fuel - Eurostat structure (ktoe)" - ct_totals["total residential"] = df[row] + ct_totals["total residential"] = df.loc[row] assert df.index[47] == "Electricity" - ct_totals["electricity residential"] = df[47] + ct_totals["electricity residential"] = df.iloc[47] assert df.index[46] == "Derived heat" - ct_totals["derived heat residential"] = df[46] + ct_totals["derived heat residential"] = df.iloc[46] assert df.index[50] == 'Thermal uses' - ct_totals["thermal uses residential"] = df[50] + ct_totals["thermal uses residential"] = df.iloc[50] # services - df = pd.read_excel(fn_tertiary, "SER_hh_fec", index_col=0)[year] + df = pd.read_excel(fn_tertiary, "SER_hh_fec", index_col=0) - ct_totals["total services space"] = df["Space heating"] + ct_totals["total services space"] = df.loc["Space heating"] rows = ["Advanced electric heating", "Conventional electric heating"] - ct_totals["electricity services space"] = df[rows].sum() + ct_totals["electricity services space"] = df.loc[rows].sum() - ct_totals["total services water"] = df["Hot water"] + ct_totals["total services water"] = df.loc["Hot water"] assert df.index[24] == "Electricity" - ct_totals["electricity services water"] = df[24] + ct_totals["electricity services water"] = df.iloc[24] - ct_totals["total services cooking"] = df["Catering"] + ct_totals["total services cooking"] = df.loc["Catering"] assert df.index[31] == "Electricity" - ct_totals["electricity services cooking"] = df[31] + ct_totals["electricity services cooking"] = df.iloc[31] - df = pd.read_excel(fn_tertiary, "SER_summary", index_col=0)[year] + df = pd.read_excel(fn_tertiary, "SER_summary", index_col=0) row = "Energy consumption by fuel - Eurostat structure (ktoe)" - ct_totals["total services"] = df[row] + ct_totals["total services"] = df.loc[row] assert df.index[50] == "Electricity" - ct_totals["electricity services"] = df[50] + ct_totals["electricity services"] = df.iloc[50] assert df.index[49] == "Derived heat" - ct_totals["derived heat services"] = df[49] + ct_totals["derived heat services"] = df.iloc[49] assert df.index[53] == 'Thermal uses' - ct_totals["thermal uses services"] = df[53] - + ct_totals["thermal uses services"] = df.iloc[53] # agriculture, forestry and fishing start = "Detailed split of energy consumption (ktoe)" end = "Market shares of energy uses (%)" - df = pd.read_excel(fn_tertiary, "AGR_fec", index_col=0).loc[start:end, year] + df = pd.read_excel(fn_tertiary, "AGR_fec", index_col=0).loc[start:end] rows = [ "Lighting", @@ -238,142 +237,143 @@ def idees_per_country(ct, year): "Specific electricity uses", "Pumping devices (electric)" ] - ct_totals["total agriculture electricity"] = df[rows].sum() + ct_totals["total agriculture electricity"] = df.loc[rows].sum() rows = ["Specific heat uses", "Low enthalpy heat"] - ct_totals["total agriculture heat"] = df[rows].sum() + ct_totals["total agriculture heat"] = df.loc[rows].sum() rows = [ "Motor drives", "Farming machine drives (diesel oil incl. biofuels)", "Pumping devices (diesel oil incl. biofuels)", ] - ct_totals["total agriculture machinery"] = df[rows].sum() + ct_totals["total agriculture machinery"] = df.loc[rows].sum() row = "Agriculture, forestry and fishing" - ct_totals["total agriculture"] = df[row] + ct_totals["total agriculture"] = df.loc[row] # transport - df = pd.read_excel(fn_transport, "TrRoad_ene", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrRoad_ene", index_col=0) - ct_totals["total road"] = df["by fuel (EUROSTAT DATA)"] + ct_totals["total road"] = df.loc["by fuel (EUROSTAT DATA)"] - ct_totals["electricity road"] = df["Electricity"] + ct_totals["electricity road"] = df.loc["Electricity"] - ct_totals["total two-wheel"] = df["Powered 2-wheelers (Gasoline)"] + ct_totals["total two-wheel"] = df.loc["Powered 2-wheelers (Gasoline)"] assert df.index[19] == "Passenger cars" - ct_totals["total passenger cars"] = df[19] + ct_totals["total passenger cars"] = df.iloc[19] assert df.index[30] == "Battery electric vehicles" - ct_totals["electricity passenger cars"] = df[30] + ct_totals["electricity passenger cars"] = df.iloc[30] assert df.index[31] == "Motor coaches, buses and trolley buses" - ct_totals["total other road passenger"] = df[31] + ct_totals["total other road passenger"] = df.iloc[31] assert df.index[39] == "Battery electric vehicles" - ct_totals["electricity other road passenger"] = df[39] + ct_totals["electricity other road passenger"] = df.iloc[39] assert df.index[41] == "Light duty vehicles" - ct_totals["total light duty road freight"] = df[41] + ct_totals["total light duty road freight"] = df.iloc[41] assert df.index[49] == "Battery electric vehicles" - ct_totals["electricity light duty road freight"] = df[49] + ct_totals["electricity light duty road freight"] = df.iloc[49] row = "Heavy duty vehicles (Diesel oil incl. biofuels)" - ct_totals["total heavy duty road freight"] = df[row] + ct_totals["total heavy duty road freight"] = df.loc[row] assert df.index[61] == "Passenger cars" - ct_totals["passenger car efficiency"] = df[61] + ct_totals["passenger car efficiency"] = df.iloc[61] - df = pd.read_excel(fn_transport, "TrRail_ene", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrRail_ene", index_col=0) - ct_totals["total rail"] = df["by fuel (EUROSTAT DATA)"] + ct_totals["total rail"] = df.loc["by fuel (EUROSTAT DATA)"] - ct_totals["electricity rail"] = df["Electricity"] + ct_totals["electricity rail"] = df.loc["Electricity"] assert df.index[15] == "Passenger transport" - ct_totals["total rail passenger"] = df[15] + ct_totals["total rail passenger"] = df.iloc[15] assert df.index[16] == "Metro and tram, urban light rail" assert df.index[19] == "Electric" assert df.index[20] == "High speed passenger trains" - ct_totals["electricity rail passenger"] = df[[16, 19, 20]].sum() + ct_totals["electricity rail passenger"] = df.iloc[[16, 19, 20]].sum() assert df.index[21] == "Freight transport" - ct_totals["total rail freight"] = df[21] + ct_totals["total rail freight"] = df.iloc[21] assert df.index[23] == "Electric" - ct_totals["electricity rail freight"] = df[23] + ct_totals["electricity rail freight"] = df.iloc[23] - df = pd.read_excel(fn_transport, "TrAvia_ene", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrAvia_ene", index_col=0) assert df.index[6] == "Passenger transport" - ct_totals["total aviation passenger"] = df[6] + ct_totals["total aviation passenger"] = df.iloc[6] assert df.index[10] == "Freight transport" - ct_totals["total aviation freight"] = df[10] + ct_totals["total aviation freight"] = df.iloc[10] assert df.index[7] == "Domestic" - ct_totals["total domestic aviation passenger"] = df[7] + ct_totals["total domestic aviation passenger"] = df.iloc[7] assert df.index[8] == "International - Intra-EU" assert df.index[9] == "International - Extra-EU" - ct_totals["total international aviation passenger"] = df[[8,9]].sum() + ct_totals["total international aviation passenger"] = df.iloc[[8,9]].sum() assert df.index[11] == "Domestic and International - Intra-EU" - ct_totals["total domestic aviation freight"] = df[11] + ct_totals["total domestic aviation freight"] = df.iloc[11] assert df.index[12] == "International - Extra-EU" - ct_totals["total international aviation freight"] = df[12] + ct_totals["total international aviation freight"] = df.iloc[12] ct_totals["total domestic aviation"] = ct_totals["total domestic aviation freight"] \ - + ct_totals["total domestic aviation passenger"] + + ct_totals["total domestic aviation passenger"] ct_totals["total international aviation"] = ct_totals["total international aviation freight"] \ - + ct_totals["total international aviation passenger"] + + ct_totals["total international aviation passenger"] - df = pd.read_excel(fn_transport, "TrNavi_ene", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrNavi_ene", index_col=0) # coastal and inland - ct_totals["total domestic navigation"] = df["by fuel (EUROSTAT DATA)"] + ct_totals["total domestic navigation"] = df.loc["by fuel (EUROSTAT DATA)"] - df = pd.read_excel(fn_transport, "TrRoad_act", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrRoad_act", index_col=0) assert df.index[85] == "Passenger cars" - ct_totals["passenger cars"] = df[85] + ct_totals["passenger cars"] = df.iloc[85] - return pd.Series(ct_totals, name=ct) + return pd.DataFrame(ct_totals) -def build_idees(countries, year): +def build_idees(countries, year=None): nprocesses = snakemake.threads - func = partial(idees_per_country, year=year) tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries), desc='Build from IDEES database') with mp.Pool(processes=nprocesses) as pool: - totals_list = list(tqdm(pool.imap(func, countries), **tqdm_kwargs)) + dfs = list(tqdm(pool.imap(idees_per_country, countries), **tqdm_kwargs)) - - totals = pd.concat(totals_list, axis=1) + df = pd.concat(dfs, keys=countries, names=['country', 'year']) # convert ktoe to TWh - exclude = totals.index.str.fullmatch("passenger cars") - totals.loc[~exclude] *= 11.63 / 1e3 + exclude = df.columns.str.fullmatch("passenger cars") + df.loc[:,~exclude] *= 11.63 / 1e3 # convert TWh/100km to kWh/km - totals.loc["passenger car efficiency"] *= 10 + df["passenger car efficiency"] *= 10 # district heating share - district_heat = totals.loc[["derived heat residential", - "derived heat services"]].sum() - total_heat = totals.loc[["thermal uses residential", - "thermal uses services"]].sum() - totals.loc["district heat share"] = district_heat.div(total_heat) + subset = ["derived heat residential", "derived heat services"] + district_heat = df[subset].sum(axis=1) + subset = ["thermal uses residential", "thermal uses services"] + total_heat = df[subset].sum(axis=1) + df["district heat share"] = district_heat.div(total_heat) - return totals.T + if year: + df = df.xs(int(year), level='year') + + return df def build_energy_totals(countries, eurostat, swiss, idees): From ba8b3f9ce71ab40a610d6c03c39986ad47e877f0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 23 Jul 2022 17:05:07 +0200 Subject: [PATCH 051/173] energy_totals: combined energy totals for all years --- Snakefile | 1 + scripts/build_energy_totals.py | 87 +++++++++++++++++++++------------- 2 files changed, 54 insertions(+), 34 deletions(-) diff --git a/Snakefile b/Snakefile index fa2323f8..04356842 100644 --- a/Snakefile +++ b/Snakefile @@ -243,6 +243,7 @@ rule build_energy_totals: district_heat_share='data/district_heat_share.csv', eurostat=directory("data/eurostat-energy_balances-june_2021_edition"), output: + energy_name_full='resources/energy_totals_full.csv', energy_name='resources/energy_totals.csv', co2_name='resources/co2_totals.csv', transport_name='resources/transport_data.csv' diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 1845c93c..18f9ee6e 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -125,21 +125,25 @@ def build_eurostat(countries, year=None): # convert ktoe/a to TWh/a df *= 11.63 / 1e3 + df.index = df.index.set_levels(df.index.levels[1].astype(int), level=1) + if year: - df = df.xs(str(year), level='year') + df = df.xs(year, level='year') return df def build_swiss(year=None): - """Return a pd.Series of Swiss energy data in TWh/a""" + """Return a pd.DataFrame of Swiss energy data in TWh/a""" fn = snakemake.input.swiss - df = pd.read_csv(fn, index_col=[0,1]).loc["CH"] + df = pd.read_csv(fn, index_col=[0,1]).stack().unstack('item') + df.index.names = ["country", "year"] + df.index = df.index.set_levels(df.index.levels[1].astype(int), level=1) if year: - df = df[str(year)] + df = df.xs(year, level='year') # convert PJ/a to TWh/a df /= 3.6 @@ -373,6 +377,8 @@ def build_idees(countries, year=None): if year: df = df.xs(int(year), level='year') + df.columns.name = 'item' + return df @@ -391,15 +397,21 @@ def build_energy_totals(countries, eurostat, swiss, idees): ) to_drop = ["passenger cars", "passenger car efficiency"] - df = idees.reindex(countries).drop(to_drop, axis=1) + + new_index = pd.MultiIndex.from_product( + [countries, eurostat.index.levels[1]], + names=["country", "year"] + ) + + df = idees.reindex(new_index).drop(to_drop, axis=1) eurostat_countries = eurostat.index.levels[0] - in_eurostat = df.index.intersection(eurostat_countries) + in_eurostat = df.index.levels[0].intersection(eurostat_countries) # add international navigation - slicer = idx[in_eurostat, :, "International maritime bunkers", :] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() + slicer = idx[in_eurostat, :, :, "International maritime bunkers", :] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0,1]).sum() df.loc[in_eurostat, "total international navigation"] = fill_values # add swiss energy data @@ -409,17 +421,20 @@ def build_energy_totals(countries, eurostat, swiss, idees): # get values for missing countries based on Eurostat EnergyBalances # divide cooking/space/water according to averages in EU28 - missing = df.index[df["total residential"].isna()] - to_fill = missing.intersection(eurostat_countries) + to_fill = df.index[df["total residential"].isna() & df.index.get_level_values('country').isin(eurostat_countries)] uses = ["space", "cooking", "water"] + c = to_fill.get_level_values('country') + y = to_fill.get_level_values('year') + for sector in ["residential", "services", "road", "rail"]: # fuel use for fuel in ["electricity", "total"]: - slicer = idx[to_fill, :, :, eurostat_sectors[sector]] - fill_values = eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=0).sum() + + slicer = idx[c, y, :, :, eurostat_sectors[sector]] + fill_values = eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=[0,1]).sum() df.loc[to_fill, f"{fuel} {sector}"] = fill_values for sector in ["residential", "services"]: @@ -462,28 +477,28 @@ def build_energy_totals(countries, eurostat, swiss, idees): nonelectric_use = no_norway[f"total {sector} {use}"] - no_norway[f"electricity {sector} {use}"] nonelectric = no_norway[f"total {sector}"] - no_norway[f"electricity {sector}"] fraction = nonelectric_use.div(nonelectric).mean() - df.loc["NO", f"total {sector} {use}"] = total_heating * fraction - df.loc["NO", f"electricity {sector} {use}"] = total_heating * fraction * elec_fraction + df.loc["NO", f"total {sector} {use}"] = (total_heating * fraction).values + df.loc["NO", f"electricity {sector} {use}"] = (total_heating * fraction * elec_fraction).values # Missing aviation - slicer = idx[to_fill, :, :, "Domestic aviation"] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() + slicer = idx[c, y, :, :, "Domestic aviation"] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0,1]).sum() df.loc[to_fill, "total domestic aviation"] = fill_values - slicer = idx[to_fill, :, "International aviation", :] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() + slicer = idx[c, y, :, "International aviation", :] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0,1]).sum() df.loc[to_fill, "total international aviation"] = fill_values # missing domestic navigation - slicer = idx[to_fill, :, :, "Domestic navigation"] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() + slicer = idx[c, y, :, :, "Domestic navigation"] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0,1]).sum() df.loc[to_fill, "total domestic navigation"] = fill_values # split road traffic for non-IDEES missing = df.index[df["total passenger cars"].isna()] - for fuel in ["total", "electricity"]: + for fuel in ["electricity", "total"]: selection = [ f"{fuel} passenger cars", f"{fuel} other road passenger", @@ -525,19 +540,20 @@ def build_energy_totals(countries, eurostat, swiss, idees): df.loc[missing, f"total aviation {purpose}"] = df.loc[missing, attrs].sum(axis=1) if "BA" in df.index: - # fill missing data for BA (services and road energy data) - # proportional to RS with ratio of total residential demand - missing = df.loc["BA"] == 0.0 - ratio = df.at["BA", "total residential"] / df.at["RS", "total residential"] - df.loc['BA', missing] = ratio * df.loc["RS", missing] + # fill missing data for BA proportional to RS + ratio = (df.loc["BA"].loc[2014:2020] / df.loc["RS"].loc[2014:2020]).mean() + df.loc["BA"] = (ratio * df.loc["RS"]).values # Missing district heating share dh_share = pd.read_csv(snakemake.input.district_heat_share, - index_col=0, usecols=[0, 1]) + index_col=0, usecols=[0, 1]) + + dh_share = pd.concat({y: dh_share for y in range(1990, 2021)}, names=["year", "country"]).swaplevel() + dh_share = dh_share.div(100).reindex(df.index) + # make conservative assumption and take minimum from both data sets - df["district heat share"] = (pd.concat([df["district heat share"], - dh_share.reindex(index=df.index)/100], - axis=1).min(axis=1)) + item = "district heat share" + df[item] = pd.concat([dh_share, df[item]], axis=1).min(axis=1) return df @@ -673,6 +689,7 @@ if __name__ == "__main__": snakemake = mock_snakemake('build_energy_totals') config = snakemake.config["energy"] + data_year = int(config["energy_totals_year"]) nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index") population = nuts3["pop"].groupby(nuts3.country).sum() @@ -680,12 +697,14 @@ if __name__ == "__main__": countries = population.index idees_countries = countries.intersection(eu28) - data_year = config["energy_totals_year"] - eurostat = build_eurostat(countries.difference(['CH']), data_year) - swiss = build_swiss(data_year) - idees = build_idees(idees_countries, data_year) + eurostat = build_eurostat(countries.difference(['CH'])) + swiss = build_swiss() + idees = build_idees(idees_countries) energy = build_energy_totals(countries, eurostat, swiss, idees) + energy.to_csv(snakemake.output.energy_name_full) + + energy = energy.xs(data_year, level='year') energy.to_csv(snakemake.output.energy_name) base_year_emissions = config["base_emissions_year"] From f70f006a47d65bcef63ef62b569dc7bd9a9032c0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 23 Jul 2022 17:05:56 +0200 Subject: [PATCH 052/173] energy_totals: co2 calculation with eurostat 2021 data --- scripts/build_energy_totals.py | 41 ++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 18f9ee6e..5482ec3b 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -606,23 +606,26 @@ def build_eea_co2(year=1990): return emissions / 1e3 -def build_eurostat_co2(countries, year=1990): +def build_eurostat_co2(countries, eurostat=None, year=1990): - eurostat = build_eurostat(countries, year) + if eurostat is None: + df = build_eurostat(countries, year) + else: + df = eurostat.xs(year, level='year') - specific_emissions = pd.Series(index=eurostat.columns, dtype=float) + specific_emissions = pd.Series(index=df.columns, dtype=float) # emissions in tCO2_equiv per MWh_th - specific_emissions["Solid fuels"] = 0.36 # Approximates coal - specific_emissions["Oil (total)"] = 0.285 # Average of distillate and residue - specific_emissions["Gas"] = 0.2 # For natural gas + specific_emissions["Solid fossil fuels"] = 0.36 # Approximates coal + specific_emissions["Oil and petroleum products"] = 0.285 # Average of distillate and residue + specific_emissions["Natural gas"] = 0.2 # For natural gas # oil values from https://www.eia.gov/tools/faqs/faq.cfm?id=74&t=11 # Distillate oil (No. 2) 0.276 # Residual oil (No. 6) 0.298 # https://www.eia.gov/electricity/annual/html/epa_a_03.html - return eurostat.multiply(specific_emissions).sum(axis=1) + return df.multiply(specific_emissions).sum(axis=1) def build_co2_totals(countries, eea_co2, eurostat_co2): @@ -632,19 +635,19 @@ def build_co2_totals(countries, eea_co2, eurostat_co2): for ct in countries.intersection(["BA", "RS", "AL", "ME", "MK"]): mappings = { - "electricity": (ct, "+", "Conventional Thermal Power Stations", "of which From Coal"), - "residential non-elec": (ct, "+", "+", "Residential"), - "services non-elec": (ct, "+", "+", "Services"), - "road non-elec": (ct, "+", "+", "Road"), - "rail non-elec": (ct, "+", "+", "Rail"), - "domestic navigation": (ct, "+", "+", "Domestic Navigation"), - "international navigation": (ct, "-", "Bunkers"), - "domestic aviation": (ct, "+", "+", "Domestic aviation"), - "international aviation": (ct, "+", "+", "International aviation"), + "electricity": (ct, "Transformation input", "Electricity & heat generation", "Main"), + "residential non-elec": (ct, "Final energy consumption", "Other sectors", "Households"), + "services non-elec": (ct, "Final energy consumption", "Other sectors", "Commercial & public services"), + "road non-elec": (ct, "Final energy consumption", "Transport sector", "Road"), + "rail non-elec": (ct, "Final energy consumption", "Transport sector", "Rail"), + "domestic navigation": (ct, "Final energy consumption", "Transport sector", "Domestic navigation"), + "international navigation": (ct, "Main", "International maritime bunkers"), + "domestic aviation": (ct, "Final energy consumption", "Transport sector", "Domestic aviation"), + "international aviation": (ct, "Main", "International aviation"), # does not include industrial process emissions or fuel processing/refining - "industrial non-elec": (ct, "+", "Industry"), + "industrial non-elec": (ct, "Final energy consumption", "Industry sector", "Non-energy use in industry sector"), # does not include non-energy emissions - "agriculture": (eurostat_co2.index.get_level_values(0) == ct) & eurostat_co2.index.isin(["Agriculture / Forestry", "Fishing"], level=3), + "agriculture": (eurostat_co2.index.get_level_values(0) == ct) & eurostat_co2.index.isin(["Agriculture & forestry", "Fishing"], level=3), } for i, mi in mappings.items(): @@ -709,7 +712,7 @@ if __name__ == "__main__": base_year_emissions = config["base_emissions_year"] eea_co2 = build_eea_co2(base_year_emissions) - eurostat_co2 = build_eurostat_co2(countries, base_year_emissions) + eurostat_co2 = build_eurostat_co2(countries, eurostat, base_year_emissions) co2 = build_co2_totals(countries, eea_co2, eurostat_co2) co2.to_csv(snakemake.output.co2_name) From 614909201ac3ae2a725d23e54c54e90f09345e77 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 23 Jul 2022 17:06:14 +0200 Subject: [PATCH 053/173] energy_totals: transport sector requires specific year --- scripts/build_energy_totals.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 5482ec3b..04147bb8 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -717,5 +717,6 @@ if __name__ == "__main__": co2 = build_co2_totals(countries, eea_co2, eurostat_co2) co2.to_csv(snakemake.output.co2_name) - transport = build_transport_data(countries, population, idees) + idees_transport = idees.xs(data_year, level='year') + transport = build_transport_data(countries, population, idees_transport) transport.to_csv(snakemake.output.transport_name) From 96b599aedd291e2888f05c52209278e22291c3d2 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 25 Jul 2022 11:52:51 +0200 Subject: [PATCH 054/173] use numpy instead of sklearn for regressino --- envs/environment.yaml | 1 - scripts/build_hydro_profile.py | 15 +++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index f5cb9aae..8bd7428f 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -39,7 +39,6 @@ dependencies: - proj - fiona <= 1.18.20 # Till issue https://github.com/Toblerity/Fiona/issues/1085 is not solved - country_converter - - scikit-learn # Keep in conda environment when calling ipython - ipython diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index f56cb9e3..b8d2eec8 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -66,7 +66,7 @@ import atlite import geopandas as gpd import pandas as pd -from sklearn.linear_model import LinearRegression +from numpy.polynomial import Polynomial import country_converter as coco cc = coco.CountryConverter() @@ -138,17 +138,16 @@ def approximate_missing_eia_stats(eia_stats, runoff_fn, countries): for c in countries: - X = runoff_eia[c].values.reshape(-1, 1) - Y = eia_stats[c].values.reshape(-1, 1) + X = runoff_eia[c] + Y = eia_stats[c] to_predict = runoff.index.difference(eia_stats.index) - X_pred = runoff.loc[to_predict, c].values.reshape(-1, 1) + X_pred = runoff.loc[to_predict, c] - linear_regressor = LinearRegression() - linear_regressor.fit(X, Y) - Y_pred = linear_regressor.predict(X_pred) + p = Polynomial.fit(X, Y, 1) + Y_pred = p(X_pred) - eia_stats_approximated[c] = pd.Series(Y_pred.T[0], index=to_predict) + eia_stats_approximated[c] = pd.Series(Y_pred, index=to_predict) eia_stats_approximated = pd.DataFrame(eia_stats_approximated) return pd.concat([eia_stats, eia_stats_approximated]).sort_index() From 0bad2097b051c2bd34a00893ec30c04cd06b3372 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 25 Jul 2022 14:35:54 +0200 Subject: [PATCH 055/173] interpolate heat demands for years outside 2007-2015 --- Snakefile | 19 ++++-- data/era5-annual-HDD-per-country.csv | 34 ++++++++++ scripts/build_energy_totals.py | 3 - scripts/build_heat_totals.py | 63 +++++++++++++++++++ ...build_population_weighted_energy_totals.py | 21 ++++--- scripts/prepare_sector_network.py | 2 + 6 files changed, 128 insertions(+), 14 deletions(-) create mode 100644 data/era5-annual-HDD-per-country.csv create mode 100644 scripts/build_heat_totals.py diff --git a/Snakefile b/Snakefile index 04356842..617d30f2 100644 --- a/Snakefile +++ b/Snakefile @@ -243,7 +243,6 @@ rule build_energy_totals: district_heat_share='data/district_heat_share.csv', eurostat=directory("data/eurostat-energy_balances-june_2021_edition"), output: - energy_name_full='resources/energy_totals_full.csv', energy_name='resources/energy_totals.csv', co2_name='resources/co2_totals.csv', transport_name='resources/transport_data.csv' @@ -253,6 +252,18 @@ rule build_energy_totals: script: 'scripts/build_energy_totals.py' +rule build_heat_totals: + input: + hdd="data/era5-annual-HDD-per-country.csv", + energy_totals="resources/energy_totals.csv", + output: + heat_totals="resources/heat_totals.csv" + threads: 1 + resources: mem_mb=2000 + benchmark: "benchmarks/build_heat_totals" + script: "scripts/build_heat_totals.py" + + rule build_biomass_potentials: input: enspreso_biomass=HTTP.remote("https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", keep_local=True), @@ -435,9 +446,9 @@ else: rule build_population_weighted_energy_totals: input: - energy_totals='resources/energy_totals.csv', + totals='resources/{kind}_totals.csv', clustered_pop_layout="resources/pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" - output: "resources/pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv" + output: "resources/pop_weighted_{kind}_totals{weather_year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=2000 script: "scripts/build_population_weighted_energy_totals.py" @@ -465,8 +476,8 @@ rule prepare_sector_network: input: overrides="data/override_component_attrs", network=pypsaeur('networks/elec{weather_year}_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc'), - energy_totals_name='resources/energy_totals.csv', pop_weighted_energy_totals="resources/pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv", + pop_weighted_heat_totals="resources/pop_weighted_heat_totals{weather_year}_s{simpl}_{clusters}.csv", transport_demand="resources/transport_demand{weather_year}_s{simpl}_{clusters}.csv", transport_data="resources/transport_data{weather_year}_s{simpl}_{clusters}.csv", avail_profile="resources/avail_profile{weather_year}_s{simpl}_{clusters}.csv", diff --git a/data/era5-annual-HDD-per-country.csv b/data/era5-annual-HDD-per-country.csv new file mode 100644 index 00000000..472257c6 --- /dev/null +++ b/data/era5-annual-HDD-per-country.csv @@ -0,0 +1,34 @@ +name,1951,1952,1953,1954,1955,1956,1957,1958,1959,1960,1961,1962,1963,1964,1965,1966,1967,1968,1969,1970,1971,1972,1973,1974,1975,1976,1977,1978,1979,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021 +AL,56,58,65,67,55,70,58,59,61,54,54,62,62,62,68,59,60,57,57,58,61,57,64,60,61,61,54,63,57,65,63,58,61,57,57,58,61,62,56,53,67,62,62,53,61,61,60,60,56,58,57,53,61,58,65,64,55,55,54,53,58,58,50,46,54,50,54,47,49,50,54 +AT,374,416,379,422,419,449,387,393,378,389,367,437,425,406,424,376,385,400,409,413,394,407,408,379,384,400,380,414,405,428,397,384,377,411,420,406,409,378,369,372,415,373,383,334,380,416,371,373,368,343,371,337,370,370,384,362,333,340,351,388,337,353,358,302,324,336,347,315,320,322,358 +BE,95,105,94,102,109,113,91,97,88,89,84,111,114,100,100,91,94,101,102,101,96,102,101,90,97,97,92,103,108,105,99,94,96,99,112,107,108,85,84,83,101,92,94,82,89,110,90,89,84,79,89,78,89,90,87,86,76,87,87,107,75,90,99,69,81,88,82,81,80,71,89 +BG,283,294,334,367,295,363,306,283,320,268,277,303,324,313,323,259,297,292,320,286,300,306,323,303,296,314,287,310,293,326,301,312,295,306,326,313,333,320,276,272,332,303,318,266,307,335,325,303,273,275,284,272,322,277,298,296,259,266,266,273,313,295,258,250,261,265,281,261,232,250,277 +BA,144,164,167,183,158,194,157,157,152,143,141,173,180,176,176,155,162,157,165,165,163,154,170,156,153,168,145,170,153,176,163,154,157,159,167,160,166,158,146,139,174,152,162,143,157,171,156,158,147,132,145,130,156,146,164,148,134,132,139,147,144,147,133,113,137,134,144,128,125,127,141 +CH,214,225,210,226,221,240,211,215,203,214,200,234,231,214,229,213,217,219,225,227,217,221,224,213,216,220,208,219,220,229,216,208,208,227,224,217,217,200,202,200,217,204,208,183,206,217,195,205,203,192,200,189,199,204,209,193,187,194,193,213,179,196,204,178,181,189,191,174,184,176,197 +CZ,297,344,307,351,350,377,313,322,301,314,294,358,361,345,351,297,296,320,342,336,319,326,322,282,296,315,299,330,328,347,316,308,298,323,350,326,346,297,280,283,333,299,316,279,312,364,319,296,288,265,309,285,308,302,311,300,269,275,289,338,281,298,306,242,261,283,287,262,255,259,302 +DE,1254,1428,1237,1416,1484,1567,1282,1342,1229,1266,1183,1486,1528,1390,1399,1269,1211,1341,1447,1429,1300,1388,1356,1184,1258,1339,1221,1377,1433,1433,1348,1288,1273,1384,1492,1425,1491,1201,1156,1135,1367,1241,1312,1155,1295,1548,1288,1220,1172,1085,1250,1168,1269,1236,1239,1198,1072,1159,1202,1459,1112,1232,1305,992,1095,1188,1148,1092,1070,1019,1230 +DK,160,176,152,175,186,190,163,178,154,169,154,184,194,176,182,181,153,170,182,186,159,166,159,147,147,172,161,171,190,177,175,164,155,162,191,181,190,151,137,132,160,147,166,154,162,191,162,154,147,134,161,148,158,151,150,144,134,137,150,193,144,159,160,125,138,145,139,142,134,125,150 +ES,962,911,927,948,785,1092,934,848,821,890,742,952,981,967,949,878,890,839,933,899,971,961,943,931,928,928,797,906,887,911,793,787,807,920,860,891,781,783,684,767,911,852,889,724,627,759,605,735,796,750,757,677,746,813,856,714,776,785,736,856,673,810,830,649,684,722,688,768,708,672,728 +EE,297,310,283,297,321,336,278,310,281,305,253,298,323,292,306,324,283,310,333,317,285,285,289,256,245,327,302,324,299,313,289,278,260,271,328,294,332,281,232,240,257,257,285,288,265,306,281,278,262,235,273,271,278,270,270,260,250,230,268,315,253,286,259,254,225,261,255,254,235,209,270 +FI,3285,3442,3085,3179,3880,3819,3296,3626,3180,3386,3065,3513,3467,3329,3526,3873,3284,3693,3640,3494,3572,3178,3458,2963,3074,3691,3490,3743,3419,3607,3572,3331,3296,3192,3894,3516,3793,3402,2888,3075,3209,3163,3319,3341,3213,3343,3302,3468,3259,2915,3303,3307,3231,3159,2998,3101,3018,3009,3219,3604,2961,3351,2985,2983,2795,3030,3127,3061,3116,2726,3260 +FR,1430,1480,1422,1489,1396,1633,1338,1378,1240,1332,1196,1584,1630,1499,1471,1331,1408,1412,1473,1469,1475,1453,1545,1356,1462,1445,1312,1454,1502,1545,1396,1315,1424,1484,1599,1518,1520,1269,1237,1227,1501,1375,1389,1168,1280,1454,1222,1337,1263,1190,1299,1132,1294,1346,1376,1263,1199,1301,1277,1522,1060,1300,1408,1034,1143,1266,1236,1150,1165,1043,1279 +GB,906,934,809,901,925,935,813,890,787,849,825,1000,1046,912,970,917,839,874,894,869,805,880,862,842,838,851,863,873,972,880,896,823,824,838,923,944,903,801,764,746,859,824,857,793,794,886,738,752,745,771,816,726,751,743,748,734,706,790,777,930,707,832,841,683,759,769,720,764,750,725,764 +GR,166,162,210,211,160,215,184,169,196,164,172,182,184,192,199,162,185,176,171,164,184,185,195,186,188,190,160,186,172,191,184,192,190,179,174,180,194,193,176,165,206,197,195,161,179,191,192,180,167,177,174,164,191,172,184,189,161,158,155,143,188,179,148,136,165,150,164,139,146,148,156 +HR,116,140,140,155,137,166,131,132,122,119,114,149,158,152,144,124,133,129,143,138,137,131,144,124,124,139,121,140,130,149,133,132,130,133,148,141,144,128,118,114,143,123,138,115,126,145,132,129,124,106,120,109,135,127,140,123,109,110,116,130,123,123,118,92,114,116,119,111,102,105,119 +HU,236,291,282,321,289,333,272,273,246,248,235,288,321,309,298,244,272,268,293,283,276,264,283,244,255,281,267,287,271,311,272,282,265,274,313,294,305,274,243,238,299,262,288,240,271,310,291,272,267,232,269,243,300,267,289,265,229,230,243,275,267,259,250,198,235,251,255,234,211,228,259 +IE,216,215,191,208,213,211,190,204,184,205,195,234,250,208,232,210,216,213,222,214,190,226,204,213,198,211,210,205,239,212,209,204,201,211,229,241,217,200,192,192,209,209,213,201,195,219,180,186,191,204,206,189,192,193,187,188,173,202,204,241,192,205,210,186,205,199,184,200,191,194,189 +IT,652,683,660,700,615,778,640,653,616,631,587,715,726,674,710,665,655,656,682,672,678,639,695,659,655,670,604,681,675,729,684,645,657,695,688,671,676,630,616,610,712,626,650,555,623,639,578,618,619,563,585,546,629,612,675,597,548,568,590,637,562,602,587,481,541,528,570,529,540,520,577 +LT,344,371,342,368,368,405,318,353,329,346,303,355,386,362,371,354,322,365,401,372,330,345,339,312,289,388,348,370,370,382,342,326,301,326,393,352,397,335,275,272,313,311,336,329,325,377,335,325,306,274,323,313,328,324,325,316,299,277,316,366,305,332,314,298,273,308,296,302,267,252,327 +LU,8,9,8,9,10,10,8,9,8,8,7,10,10,9,9,8,8,9,9,9,9,9,9,8,9,9,8,9,10,10,9,9,9,9,10,10,10,8,8,8,9,8,9,8,8,10,8,8,8,7,8,7,8,8,8,8,7,8,8,9,7,8,9,6,7,8,8,7,7,7,8 +LV,383,402,372,394,407,439,351,391,361,384,326,386,420,385,402,401,360,401,434,407,361,372,371,337,316,424,385,411,392,411,372,357,330,354,425,379,428,362,300,301,334,337,366,366,348,402,365,358,334,302,353,346,357,352,354,337,324,300,346,402,330,365,342,329,297,337,330,328,297,272,351 +MK,64,66,74,79,66,79,67,66,71,63,64,72,73,70,77,66,69,66,67,67,70,67,75,69,70,69,62,72,66,73,71,69,70,67,69,67,70,71,65,63,76,68,69,59,69,70,71,68,64,65,65,62,69,64,70,68,60,59,61,59,67,66,56,53,61,58,62,54,55,57,63 +ME,40,43,45,48,41,50,42,42,42,39,39,44,45,45,48,43,43,42,42,43,43,41,44,43,42,45,39,45,42,46,45,41,43,42,42,41,44,44,42,39,47,43,43,38,43,44,42,43,39,39,40,37,42,41,45,43,38,38,38,38,39,40,36,32,37,36,39,34,35,35,39 +NL,113,126,110,121,130,137,108,118,107,107,104,132,141,121,120,114,105,119,124,122,111,120,116,104,111,116,107,120,133,122,118,112,110,118,134,127,130,99,99,94,119,106,113,102,109,138,111,102,97,93,106,98,109,103,100,100,87,102,103,131,93,108,117,82,95,104,96,98,93,85,105 +NO,3339,3541,3020,3307,3639,3731,3315,3547,3122,3353,3400,3807,3641,3496,3783,3951,3259,3533,3425,3431,3339,3128,3312,3028,3094,3405,3369,3436,3457,3380,3502,3181,3141,3117,3541,3359,3449,3200,2890,2848,3057,3032,3196,3207,3170,3301,3060,3203,3062,2878,3166,3028,2983,2943,2874,2828,2918,2954,3018,3442,2796,3157,2946,2754,2793,2859,2963,2930,2981,2711,3035 +PL,1249,1438,1294,1478,1446,1586,1279,1349,1269,1305,1215,1435,1527,1450,1467,1280,1193,1321,1504,1431,1293,1347,1314,1200,1184,1419,1270,1393,1431,1491,1336,1265,1178,1304,1494,1377,1507,1255,1077,1078,1312,1252,1310,1197,1302,1512,1326,1249,1171,1052,1268,1192,1296,1230,1262,1244,1117,1092,1220,1426,1177,1274,1251,1068,1063,1163,1158,1120,1019,1017,1252 +PT,107,103,99,109,80,131,106,93,93,97,78,106,104,115,103,98,103,97,112,110,115,116,106,109,109,109,99,139,107,102,89,90,95,104,99,108,85,89,75,87,104,96,104,85,66,88,62,82,96,90,90,78,87,97,103,90,91,91,84,96,80,101,100,81,79,87,79,96,81,72,78 +RO,736,811,883,948,822,968,810,789,826,731,744,826,890,867,873,706,791,776,858,792,787,798,838,782,752,855,772,836,780,880,801,811,763,807,917,815,882,827,710,700,846,801,848,701,806,869,842,805,740,710,760,721,844,754,799,780,684,695,695,757,793,781,709,662,681,721,720,690,624,648,740 +RS,222,252,273,300,249,313,248,249,254,231,226,270,283,271,274,233,253,243,267,254,251,246,272,242,243,265,233,265,241,273,251,255,244,253,275,256,268,254,231,220,275,243,263,223,250,270,259,250,237,217,234,217,262,238,265,247,217,214,222,229,251,244,214,192,222,223,232,211,197,212,234 +SK,178,210,195,217,208,228,192,193,185,185,178,208,213,211,219,179,187,192,203,200,192,188,195,178,178,197,188,204,193,215,191,190,180,193,212,198,207,187,172,173,202,186,191,171,186,204,193,184,177,162,184,174,190,184,191,182,163,161,170,188,175,179,174,142,161,170,176,157,152,158,180 +SI,61,72,67,73,69,78,65,66,62,61,59,75,75,73,73,62,65,67,71,69,67,68,71,62,61,69,62,71,68,74,67,65,65,69,73,71,71,63,59,58,70,61,63,55,61,71,63,62,62,54,59,55,65,63,67,61,54,57,57,65,58,59,59,46,55,57,58,53,52,53,61 +SE,3891,4219,3560,3919,4426,4488,3950,4223,3662,3988,3814,4451,4260,4021,4358,4613,3929,4280,4255,4254,4043,3806,3975,3634,3625,4238,4132,4314,4246,4287,4301,3913,3840,3819,4588,4139,4376,3931,3476,3446,3785,3695,3893,3991,3916,4073,3757,3950,3781,3446,3898,3778,3755,3769,3632,3561,3606,3590,3806,4397,3474,3935,3675,3452,3421,3635,3693,3705,3689,3247,3807 diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 04147bb8..d9763779 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -705,9 +705,6 @@ if __name__ == "__main__": idees = build_idees(idees_countries) energy = build_energy_totals(countries, eurostat, swiss, idees) - energy.to_csv(snakemake.output.energy_name_full) - - energy = energy.xs(data_year, level='year') energy.to_csv(snakemake.output.energy_name) base_year_emissions = config["base_emissions_year"] diff --git a/scripts/build_heat_totals.py b/scripts/build_heat_totals.py new file mode 100644 index 00000000..34d5dcdf --- /dev/null +++ b/scripts/build_heat_totals.py @@ -0,0 +1,63 @@ +"""Approximate heat demand for all weather years.""" + +import pandas as pd + +from itertools import product +from numpy.polynomial import Polynomial + +idx = pd.IndexSlice + + +def approximate_heat_demand(energy_totals, hdd): + + if isinstance(hdd, str): + hdd = pd.read_csv(hdd, index_col=0).T + hdd.index = hdd.index.astype(int) + + demands = {} + + for kind, sector in product(["total", "electricity"], ["services", "residential"]): + + row = idx[:, 2007:2015] + col = f"{kind} {sector} space" + demand = energy_totals.loc[row, col].unstack(0) + + demand_approx = {} + + for c in countries: + + Y = demand[c].dropna() + X = hdd.loc[Y.index, c] + + to_predict = hdd.index.difference(Y.index) + X_pred = hdd.loc[to_predict, c] + + p = Polynomial.fit(X, Y, 1) + Y_pred = p(X_pred) + + demand_approx[c] = pd.Series(Y_pred, index=to_predict) + + demand_approx = pd.DataFrame(demand_approx) + demand_approx = pd.concat([demand, demand_approx]).sort_index() + demands[f"{kind} {sector} space"] = demand_approx.groupby(demand_approx.index).sum() + + demands = pd.concat(demands).unstack().T.clip(lower=0) + demands.index.names = ["country", "year"] + + return demands + + +if __name__ == "__main__": + if 'snakemake' not in globals(): + from helper import mock_snakemake + snakemake = mock_snakemake('build_energy_totals') + + hdd = pd.read_csv(snakemake.input.hdd, index_col=0).T + + energy_totals = pd.read_csv(snakemake.input.energy_totals, index_col=[0,1]) + + countries = hdd.columns + + heat_demand = approximate_heat_demand(energy_totals, hdd) + + heat_demand.to_csv(snakemake.output.heat_totals) diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index 938983d5..933a0417 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -1,4 +1,4 @@ -"""Build population-weighted energy totals.""" +"""Build population-weighted energy and heat totals.""" import pandas as pd @@ -7,16 +7,23 @@ if __name__ == '__main__': from helper import mock_snakemake snakemake = mock_snakemake( 'build_population_weighted_energy_totals', + weather_year='', simpl='', - clusters=48, + clusters=37, ) + config = snakemake.config["energy"] + data_year = int(config["energy_totals_year"]) + if snakemake.wildcards.weather_year and snakemake.wildcards.kind == 'heat': + data_year = int(snakemake.wildcards.weather_year) + pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) - energy_totals = pd.read_csv(snakemake.input.energy_totals, index_col=0) + totals = pd.read_csv(snakemake.input.totals, index_col=[0,1]) + totals = totals.xs(data_year, level='year') - nodal_energy_totals = energy_totals.loc[pop_layout.ct].fillna(0.) - nodal_energy_totals.index = pop_layout.index - nodal_energy_totals = nodal_energy_totals.multiply(pop_layout.fraction, axis=0) + nodal_totals = totals.loc[pop_layout.ct].fillna(0.) + nodal_totals.index = pop_layout.index + nodal_totals = nodal_totals.multiply(pop_layout.fraction, axis=0) - nodal_energy_totals.to_csv(snakemake.output[0]) + nodal_totals.to_csv(snakemake.output[0]) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 14e824c0..3f25251f 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -2359,6 +2359,8 @@ if __name__ == "__main__": snakemake.config['costs']['lifetime']) pop_weighted_energy_totals = pd.read_csv(snakemake.input.pop_weighted_energy_totals, index_col=0) + pop_weighted_heat_totals = pd.read_csv(snakemake.input.pop_weighted_heat_totals, index_col=0) + pop_weighted_energy_totals.update(pop_weighted_heat_totals) patch_electricity_network(n) From 88ec493b3ab405a3ba307075366490e844ba6723 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 25 Jul 2022 14:40:13 +0200 Subject: [PATCH 056/173] Snakefile: correct updated eurostat dataset publication year! --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 617d30f2..1e53dc76 100644 --- a/Snakefile +++ b/Snakefile @@ -54,7 +54,7 @@ datafiles = [ "data/emobility/KFZ__count", "data/emobility/Pkw__count", "data/h2_salt_caverns_GWh_per_sqkm.geojson", - directory("data/eurostat-energy_balances-april_2022_edition"), + directory("data/eurostat-energy_balances-june_2021_edition"), directory("data/jrc-idees-2015"), ] From aff5e091957e87e793622402837368b15db2dc1b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 25 Jul 2022 17:04:56 +0200 Subject: [PATCH 057/173] hydro: store corrected eia stats --- Snakefile | 4 +++- scripts/build_hydro_profile.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 34c77981..9e964115 100644 --- a/Snakefile +++ b/Snakefile @@ -234,7 +234,9 @@ rule build_hydro_profile: eia_hydro_capacity='data/eia_hydro_annual_capacity.csv', era5_runoff='data/era5-annual-runoff-per-country.csv', cutout=f"cutouts/{config['renewable']['hydro']['cutout']}.nc" if "hydro" in config["renewable"] else "config['renewable']['hydro']['cutout'] not configured", - output: 'resources/profile{weather_year}_hydro.nc' + output: + profile='resources/profile{weather_year}_hydro.nc', + eia_hydro='resources/eia_hydro_annual_generation.csv', log: "logs/build_hydro_profile{weather_year}.log" resources: mem_mb=5000 script: 'scripts/build_hydro_profile.py' diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index b8d2eec8..4a9fc409 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -180,6 +180,8 @@ if __name__ == "__main__": fn = snakemake.input.era5_runoff eia_stats = approximate_missing_eia_stats(eia_stats, fn, countries) + eia_stats.to_csv(snakemake.output.eia_hydro) + weather_year = snakemake.wildcards.weather_year norm_year = config_hydro.get('eia_norm_year') if norm_year: @@ -195,4 +197,4 @@ if __name__ == "__main__": if 'clip_min_inflow' in config_hydro: inflow = inflow.where(inflow > config_hydro['clip_min_inflow'], 0) - inflow.to_netcdf(snakemake.output[0]) + inflow.to_netcdf(snakemake.output.profile) From 4890277ea36dac6d03d27d6a97661439fb012f20 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 25 Jul 2022 17:47:22 +0200 Subject: [PATCH 058/173] hydro: add {weather_year} wildcard to all outputs --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 9e964115..4674de2a 100644 --- a/Snakefile +++ b/Snakefile @@ -236,7 +236,7 @@ rule build_hydro_profile: cutout=f"cutouts/{config['renewable']['hydro']['cutout']}.nc" if "hydro" in config["renewable"] else "config['renewable']['hydro']['cutout'] not configured", output: profile='resources/profile{weather_year}_hydro.nc', - eia_hydro='resources/eia_hydro_annual_generation.csv', + eia_hydro='resources/eia_hydro_annual_generation{weather_year}.csv', log: "logs/build_hydro_profile{weather_year}.log" resources: mem_mb=5000 script: 'scripts/build_hydro_profile.py' From ea703d4588d52e11935d870af78f4bee20ad9dc7 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 26 Jul 2022 18:24:16 +0200 Subject: [PATCH 059/173] Snakefile: adjust memory requirements of jobs --- Snakefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Snakefile b/Snakefile index 4674de2a..91c48a00 100644 --- a/Snakefile +++ b/Snakefile @@ -88,7 +88,7 @@ rule build_powerplants: output: "resources/powerplants.csv" log: "logs/build_powerplants.log" threads: 1 - resources: mem_mb=500 + resources: mem_mb=2000 script: "scripts/build_powerplants.py" @@ -222,7 +222,7 @@ rule build_renewable_profiles: log: "logs/build_renewable_profile{weather_year}_{technology}.log" benchmark: "benchmarks/build_renewable_profiles{weather_year}_{technology}" threads: ATLITE_NPROCESSES - resources: mem_mb=ATLITE_NPROCESSES * 5000 + resources: mem_mb=ATLITE_NPROCESSES * 4000 wildcard_constraints: technology="(?!hydro).*" # Any technology other than hydro script: "scripts/build_renewable_profiles.py" @@ -238,7 +238,7 @@ rule build_hydro_profile: profile='resources/profile{weather_year}_hydro.nc', eia_hydro='resources/eia_hydro_annual_generation{weather_year}.csv', log: "logs/build_hydro_profile{weather_year}.log" - resources: mem_mb=5000 + resources: mem_mb=2000 script: 'scripts/build_hydro_profile.py' From aa9f047abbb3b646b1881095d748d7937750c5bc Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 29 Jul 2022 13:38:21 +0200 Subject: [PATCH 060/173] add option for artificial load 1951-2021, rather than OPSD --- Snakefile | 29 ++++++++++++----- config.default.yaml | 2 ++ config.tutorial.yaml | 2 ++ scripts/build_artificial_load_data.py | 47 +++++++++++++++++++++++++++ test/config.test1.yaml | 2 ++ 5 files changed, 73 insertions(+), 9 deletions(-) create mode 100755 scripts/build_artificial_load_data.py diff --git a/Snakefile b/Snakefile index 91c48a00..55ae3d20 100644 --- a/Snakefile +++ b/Snakefile @@ -67,18 +67,29 @@ if config['enable'].get('retrieve_databundle', True): log: "logs/retrieve_databundle.log" script: 'scripts/retrieve_databundle.py' +if config["enable"].get('retrieve_opsd_load_data', True): + rule retrieve_load_data: + input: HTTP.remote("data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv", keep_local=True, static=True) + output: "data/load_raw.csv" + run: move(input[0], output[0]) -rule retrieve_load_data: - input: HTTP.remote("data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv", keep_local=True, static=True) - output: "data/load_raw.csv" - run: move(input[0], output[0]) + rule build_load_data: + input: "data/load_raw.csv" + output: "resources/load{weather_year}.csv" + log: "logs/build_load_data{weather_year}.log" + script: 'scripts/build_load_data.py' +if config["enable"].get('retrieve_artificial_load_data', False): + rule retrieve_artificial_load_data: + input: HTTP.remote("https://sandbox.zenodo.org/record/1089549/files/demand_hourly.csv", keep_local=True, static=True) + output: "data/load_artificial_raw.csv" + run: move(input[0], output[0]) -rule build_load_data: - input: "data/load_raw.csv" - output: "resources/load{weather_year}.csv" - log: "logs/build_load_data{weather_year}.log" - script: 'scripts/build_load_data.py' + rule build_artificial_load_data: + input: "data/load_artificial_raw.csv" + output: "resources/load{weather_year}.csv" + log: "logs/build_artificial_load_data{weather_year}.log" + script: "scripts/build_artificial_load_data.py" rule build_powerplants: diff --git a/config.default.yaml b/config.default.yaml index 45d664a2..b79f2559 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -29,6 +29,8 @@ enable: retrieve_cost_data: true build_cutout: false retrieve_cutout: true + retrieve_opsd_load_data: true + retrieve_artificial_load_data: false build_natura_raster: false retrieve_natura_raster: true custom_busmap: false diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 2217f581..fcf53d46 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -30,6 +30,8 @@ enable: retrieve_cost_data: true build_cutout: false retrieve_cutout: true + retrieve_opsd_load_data: true + retrieve_artificial_load_data: false build_natura_raster: false retrieve_natura_raster: true custom_busmap: false diff --git a/scripts/build_artificial_load_data.py b/scripts/build_artificial_load_data.py new file mode 100755 index 00000000..c03bf9da --- /dev/null +++ b/scripts/build_artificial_load_data.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: 2022 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +"""This rule downloads the load data""" + +import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + +import pandas as pd + +if __name__ == "__main__": + + if 'snakemake' not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake('build_artificial_load_data', weather_year='') + + configure_logging(snakemake) + + weather_year = snakemake.wildcards.weather_year + if weather_year: + snapshots = dict( + start=weather_year, + end=str(int(weather_year)+1), + closed="left" + ) + else: + snapshots = snakemake.config['snapshots'] + snapshots = pd.date_range(freq='h', **snapshots) + + fixed_year = snakemake.config["load"].get("fixed_year", False) + years = slice(str(fixed_year), str(fixed_year)) if fixed_year else slice(snapshots[0], snapshots[-1]) + countries = snakemake.config['countries'] + + load = pd.read_csv( + snakemake.input[0], + index_col=0, + parse_dates=True + ).loc[snapshots, countries] + + assert not load.isna().any().any(), 'Load data contains nans.' + + if fixed_year: + load.index = load.index.map(lambda t: t.replace(year=snapshots.year[0])) + + load.to_csv(snakemake.output[0]) diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 48b8216a..cfe7cf57 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -29,6 +29,8 @@ enable: retrieve_cost_data: true build_cutout: false retrieve_cutout: true + retrieve_opsd_load_data: true + retrieve_artificial_load_data: false build_natura_raster: false retrieve_natura_raster: true custom_busmap: false From ce8d50813313614a1df51609808b4629a992dba4 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 29 Jul 2022 14:06:07 +0200 Subject: [PATCH 061/173] reenable custom busmap --- Snakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Snakefile b/Snakefile index 55ae3d20..0de698c9 100644 --- a/Snakefile +++ b/Snakefile @@ -299,6 +299,8 @@ rule cluster_network: regions_onshore="resources/regions_onshore_elec{weather_year}_s{simpl}.geojson", regions_offshore="resources/regions_offshore_elec{weather_year}_s{simpl}.geojson", busmap=ancient('resources/busmap_elec{weather_year}_s{simpl}.csv'), + custom_busmap=("data/custom_busmap_elec_s{simpl}_{clusters}.csv" + if config["enable"].get("custom_busmap", False) else []), tech_costs=COSTS output: network='networks/elec{weather_year}_s{simpl}_{clusters}.nc', From ee0c69727ffe365e6262e6683ec4f1e2346239a0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 29 Jul 2022 20:29:35 +0200 Subject: [PATCH 062/173] leap_days need to be filtered for daily data too in heat --- scripts/build_heat_demand.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 1e165c69..73e67633 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -25,9 +25,12 @@ if __name__ == '__main__': else: snapshots = snakemake.config['snapshots'] + drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) time = pd.date_range(freq='h', **snapshots) - if snakemake.config["atlite"].get("drop_leap_day", False): + daily = pd.date_range(freq='D', **snapshots) + if drop_leap_day: time = time[~((time.month == 2) & (time.day == 29))] + daily = daily[~((daily.month == 2) & (daily.day == 29))] cutout = atlite.Cutout(cutout_name).sel(time=time) @@ -44,6 +47,6 @@ if __name__ == '__main__': M = I.T.dot(np.diag(I.dot(stacked_pop))) heat_demand = cutout.heat_demand( - matrix=M.T, index=clustered_regions.index) + matrix=M.T, index=clustered_regions.index).sel(time=daily) heat_demand.to_netcdf(snakemake.output[f"heat_demand_{area}"]) From d4fa922a2a6af9abb8df01f03de58661de0fe3f7 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 29 Jul 2022 21:19:26 +0200 Subject: [PATCH 063/173] drop_leap_days when aggregating snapshots --- scripts/prepare_sector_network.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 3f25251f..45bd9abb 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -571,7 +571,7 @@ def add_co2limit(n, Nyears=1., limit=0.): ) # TODO PyPSA-Eur merge issue -def average_every_nhours(n, offset): +def average_every_nhours(n, offset, drop_leap_day=False): logger.info(f'Resampling the network to {offset}') m = n.copy(with_time=False) @@ -590,6 +590,10 @@ def average_every_nhours(n, offset): else: pnl[k] = df.resample(offset).mean() + if drop_leap_day: + sns = m.snapshots[~((m.snapshots.month == 2) & (m.snapshots.day == 29))] + m.set_snapshots(sns) + return m @@ -2428,7 +2432,8 @@ if __name__ == "__main__": for o in opts: m = re.match(r'^\d+h$', o, re.IGNORECASE) if m is not None: - n = average_every_nhours(n, m.group(0)) + drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) + n = average_every_nhours(n, m.group(0), drop_leap_day) break limit_type = "config" From 64e1ef895d894262dc722004b8d40259a184adeb Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 1 Aug 2022 11:55:31 +0200 Subject: [PATCH 064/173] add rules to solve operations with perfect and myopic foresight --- Snakefile | 37 ++++++ scripts/solve_network.py | 13 +- scripts/solve_operations_network.py | 126 ++++++++++++++++++ scripts/solve_operations_network_myopic.py | 143 +++++++++++++++++++++ 4 files changed, 314 insertions(+), 5 deletions(-) create mode 100644 scripts/solve_operations_network.py create mode 100644 scripts/solve_operations_network_myopic.py diff --git a/Snakefile b/Snakefile index 1e53dc76..981c340d 100644 --- a/Snakefile +++ b/Snakefile @@ -13,6 +13,7 @@ configfile: "config.yaml" wildcard_constraints: weather_year="[0-9]{4}|", + capacity_year="[0-9]{4}|", lv="[a-z0-9\.]+", simpl="[a-zA-Z0-9]*", clusters="[0-9]+m?", @@ -680,3 +681,39 @@ if config["foresight"] == "myopic": resources: mem_mb=config['solving']['mem'] benchmark: RDIR + "/benchmarks/solve_network/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}" script: "scripts/solve_network.py" + + +rule solve_operations_network: + input: + pre=RDIR + "/prenetworks/elec{weather_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + post=RDIR + "/postnetworks/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + output: "elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc" + shadow: "shallow" + log: + solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", + python=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_python.log", + threads: 4 + resources: mem_mb=10000 + benchmark: RDIR + "/benchmarks/solve_operations_network/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}" + script: "scripts/solve_operations_network.py" + + +def solved_previous_year(wildcards): + previous_year = int(wildcards.weather_year) - 1 + return RDIR + "/postnetworks/elec" + previous_year + "_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc" + + +rule solve_operations_network_myopic: + input: + pre=RDIR + "/prenetworks/elec{weather_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + post=RDIR + "/postnetworks/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + previous=solved_previous_year + output: "elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc" + shadow: "shallow" + log: + solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", + python=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_python.log", + threads: 4 + resources: mem_mb=10000 + benchmark: RDIR + "/benchmarks/solve_operations_network_myopic/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}" + script: "scripts/solve_operations_network_myopic.py" diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 19ef5f52..4dd4d969 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -71,7 +71,7 @@ def prepare_network(n, solve_opts=None): df.where(df>solve_opts['clip_p_max_pu'], other=0., inplace=True) if solve_opts.get('load_shedding'): - n.add("Carrier", "Load") + n.add("Carrier", "load") n.madd("Generator", n.buses.index, " load", bus=n.buses.index, carrier='load', @@ -246,7 +246,7 @@ def extra_functionality(n, snapshots): add_co2_sequestration_limit(n, snapshots) -def solve_network(n, config, opts='', **kwargs): +def solve_network(n, config, opts='', snapshots=None, **kwargs): solver_options = config['solving']['solver'].copy() solver_name = solver_options.pop('name') cf_solving = config['solving']['options'] @@ -259,12 +259,15 @@ def solve_network(n, config, opts='', **kwargs): n.config = config n.opts = opts + if snapshots is None: + snapshots = n.snapshots + if cf_solving.get('skip_iterations', False): - network_lopf(n, solver_name=solver_name, solver_options=solver_options, + network_lopf(n, snapshots, solver_name=solver_name, solver_options=solver_options, extra_functionality=extra_functionality, keep_shadowprices=keep_shadowprices, **kwargs) else: - ilopf(n, solver_name=solver_name, solver_options=solver_options, + ilopf(n, snapshots, solver_name=solver_name, solver_options=solver_options, track_iterations=track_iterations, min_iterations=min_iterations, max_iterations=max_iterations, @@ -295,7 +298,7 @@ if __name__ == "__main__": if tmpdir is not None: from pathlib import Path Path(tmpdir).mkdir(parents=True, exist_ok=True) - opts = snakemake.wildcards.opts.split('-') + opts = snakemake.wildcards.sector_opts.split('-') solve_opts = snakemake.config['solving']['options'] fn = getattr(snakemake.log, 'memory', None) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py new file mode 100644 index 00000000..2021c7c9 --- /dev/null +++ b/scripts/solve_operations_network.py @@ -0,0 +1,126 @@ +"""Solve operations network.""" + + +import pypsa +import numpy as np + +from solve_network import solve_network, prepare_network +from helper import override_component_attrs + +import logging +logger = logging.getLogger(__name__) +pypsa.pf.logger.setLevel(logging.WARNING) + + +def set_parameters_from_optimized(n, n_optim): + lines_typed_i = n.lines.index[n.lines.type != ''] + n.lines.loc[lines_typed_i, 'num_parallel'] = \ + n_optim.lines['num_parallel'].reindex(lines_typed_i, fill_value=0.) + n.lines.loc[lines_typed_i, 's_nom'] = ( + np.sqrt(3) * n.lines['type'].map(n.line_types.i_nom) * + n.lines.bus0.map(n.buses.v_nom) * n.lines.num_parallel) + + lines_untyped_i = n.lines.index[n.lines.type == ''] + for attr in ('s_nom', 'r', 'x'): + n.lines.loc[lines_untyped_i, attr] = \ + n_optim.lines[attr].reindex(lines_untyped_i, fill_value=0.) + n.lines['s_nom_extendable'] = False + + links_dc_i = n.links.index[n.links.p_nom_extendable] + n.links.loc[links_dc_i, 'p_nom'] = \ + n_optim.links['p_nom_opt'].reindex(links_dc_i, fill_value=0.) + n.links.loc[links_dc_i, 'p_nom_extendable'] = False + + gen_extend_i = n.generators.index[n.generators.p_nom_extendable] + n.generators.loc[gen_extend_i, 'p_nom'] = \ + n_optim.generators['p_nom_opt'].reindex(gen_extend_i, fill_value=0.) + n.generators.loc[gen_extend_i, 'p_nom_extendable'] = False + + stor_units_extend_i = n.storage_units.index[n.storage_units.p_nom_extendable] + n.storage_units.loc[stor_units_extend_i, 'p_nom'] = \ + n_optim.storage_units['p_nom_opt'].reindex(stor_units_extend_i, fill_value=0.) + n.storage_units.loc[stor_units_extend_i, 'p_nom_extendable'] = False + + stor_extend_i = n.stores.index[n.stores.e_nom_extendable] + n.stores.loc[stor_extend_i, 'e_nom'] = \ + n_optim.stores['e_nom_opt'].reindex(stor_extend_i, fill_value=0.) + n.stores.loc[stor_extend_i, 'e_nom_extendable'] = False + + return n + + +def remove_unused_components(n, threshold=50): + logger.info("Remove assets that are barely used to speed things up.") + + for c in n.iterate_components({"Store", "Link", "Generator"}): + attr = "e_nom" if c.name == "Store" else "p_nom" + to_remove = c.df.loc[c.df[attr] < threshold].index + logger.info(f"Removing barely used {c.name}s:\n{to_remove}") + n.mremove(c.name, to_remove) + + return n + + +def add_load_shedding(n, voll=1e4): + logger.info("Add load shedding to all buses.") + + if "load" in n.generators.carrier.unique(): + to_remove = n.generators.query("carrier == 'load'").index + logger.info(f"Removing pre-existing load shedding:\n{to_remove}") + n.mremove("Generator", to_remove) + + n.madd("Generator", n.buses.index, + suffix=" load", + bus=n.buses.index, + carrier='load', + marginal_cost=voll, + p_nom=1e6 + ) + + return n + + +if __name__ == "__main__": + if 'snakemake' not in globals(): + from helper import mock_snakemake + snakemake = mock_snakemake( + 'solve_operations_network', + capacity_year=1952, + simpl='', + opts='', + clusters=37, + lv=2.0, + sector_opts='Co2L0-25H-T-H-B-I-A', + planning_horizons=2030, + weather_year=2013 + ) + + logging.basicConfig(filename=snakemake.log.python, + level=snakemake.config['logging_level']) + + tmpdir = snakemake.config['solving'].get('tmpdir') + if tmpdir is not None: + from pathlib import Path + Path(tmpdir).mkdir(parents=True, exist_ok=True) + + overrides = override_component_attrs(snakemake.input.overrides) + + n = pypsa.Network(snakemake.input.pre, override_component_attrs=overrides) + n_post = pypsa.Network(snakemake.input.post, override_component_attrs=overrides) + n = set_parameters_from_optimized(n, n_post) + del n_post + + n = remove_unused_components(n) + n = add_load_shedding(n) + + opts = snakemake.wildcards.sector_opts.split('-') + solve_opts = snakemake.config['solving']['options'] + solve_opts['skip_iterations'] = True + + n = prepare_network(n, solve_opts) + + n = solve_network(n, config=snakemake.config, opts=opts, + solver_dir=tmpdir, + solver_logfile=snakemake.log.solver) + + n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/solve_operations_network_myopic.py b/scripts/solve_operations_network_myopic.py new file mode 100644 index 00000000..a0e30b0a --- /dev/null +++ b/scripts/solve_operations_network_myopic.py @@ -0,0 +1,143 @@ +"""Solve myopic operations network.""" + + +import pypsa +import pandas as pd + +from solve_network import solve_network, prepare_network +from solve_operations_network import set_parameters_from_optimized, remove_unused_components, add_load_shedding +from helper import override_component_attrs + +import logging +logger = logging.getLogger(__name__) +pypsa.pf.logger.setLevel(logging.WARNING) + + +def prepare_myopic(n, config, store_soc, storage_unit_soc): + + n.stores.e_cyclic = False + n.storage_units.cyclic_state_of_charge = False + + biomass_stores = n.stores.carrier.str.isin(["solid biomass", "biogas"]) + biomass_potential = n.stores.loc[biomass_stores, "e_initial"] + + n.stores.e_initial = store_soc + n.storage_units.state_of_charge_initial = storage_unit_soc + + n.remove("GlobalConstraint", "CO2Limit") + n.stores.at["co2 atmosphere", "marginal_cost"] = -config["co2_price"] + + # handle co2 sequestration + assert sum(n.stores.carriers == "co2 stored") == 1, "Myopic operation not implemented for spatially resolved CO2 sequestration." + n.stores.at["co2 stored", 'e_nom'] = config['co2_sequestration_limit'] * 1e6 # t/a + + # reset co2 emissions + n.stores.loc[n.stores.carrier == 'co2 stored', "e_initial"] = 0. + n.stores.at["co2 atmosphere", "e_initial"] = 0. + + # replenish fossil gas and oil with 1000 TWh each + fossil_stores = n.stores.carrier.str.isin(["gas", "oil"]) + n.stores.loc[fossil_stores, 'e_initial'] = 1e9 + n.stores.loc[fossil_stores, 'e_nom'] = 10e9 + + # replenish annual solid biomass and biogas potentials + n.stores.loc[biomass_stores, "e_initial"] = biomass_potential + + # set storage bidding prices + # TODO bidding prices for Fischer-Tropsch, Methanation for H2? + bidding_prices = config["bidding_prices"] + for c in n.iterate_components({"Store", "Link", "StorageUnit"}): + c.df.marginal_cost.update(c.df.carrier.map(bidding_prices).dropna()) + + # deduct industry solid biomass + assert sum(n.stores.carriers == "solid biomass") == 1, "Myopic operation not implemented for spatially resolved solid biomass." + n.stores.at["EU solid biomass", "e_initial"] -= n.loads.at["solid biomass for industry", "p_set"] * 8760 + n.remove("Load", "solid biomass for industry") + + return n + + +def solve_network_myopic(n, config, opts='', **kwargs): + + rolling_horizon = config["operations"]["rolling_horizon"] + + freq = int(pd.infer_freq(n.snapshots)[:-1]) + window = rolling_horizon["window"] * 24 // freq + overlap = rolling_horizon["overlap"] * 24 // freq + kept = window - overlap + length = len(n.snapshots) + + assert kept > 0, f"Overlap ({overlap} days) must be smaller than windows ({window} days)." + + for i in range(length // kept): + + snapshots = n.snapshots[i * kept:(i + 1) * kept + overlap] + logger.info(f"Optimising operations from {snapshots[0]} to {snapshots[-1]}") + + n = solve_network(n, config, opts=opts, snapshots=snapshots, **kwargs) + + last_kept = n.snapshots[(i + 1) * kept - 1] + logger.info(f"Setting initial SOCs from {last_kept} for next iteration.\n") + + n.stores.e_initial = n.stores_t.e.loc[last_kept] + n.storage_units.state_of_charge_initial = n.storage_units_t.state_of_charge.loc[last_kept] + + return n + + +if __name__ == "__main__": + if 'snakemake' not in globals(): + from helper import mock_snakemake + snakemake = mock_snakemake( + 'solve_operations_network_myopic', + capacity_year=1952, + simpl='', + opts='', + clusters=37, + lv=2.0, + sector_opts='Co2L0-25H-T-H-B-I-A', + planning_horizons=2030, + weather_year=2013 + ) + + logging.basicConfig(filename=snakemake.log.python, + level=snakemake.config['logging_level']) + + tmpdir = snakemake.config['solving'].get('tmpdir') + if tmpdir is not None: + from pathlib import Path + Path(tmpdir).mkdir(parents=True, exist_ok=True) + + config = snakemake.config["operations"] + overrides = override_component_attrs(snakemake.input.overrides) + + n = pypsa.Network(snakemake.input.pre, override_component_attrs=overrides) + + n_post = pypsa.Network(snakemake.input.post, override_component_attrs=overrides) + n = set_parameters_from_optimized(n, n_post) + del n_post + + n_previous = pypsa.Network(snakemake.input.previous, override_component_attrs=overrides) + store_soc = n_previous.stores_t.e.iloc[-1] + storage_unit_soc = n_previous.storage_units_t.state_of_charge.iloc[-1] + del n_previous + + n = remove_unused_components(n) + n = add_load_shedding(n) + n = prepare_myopic(n, config, store_soc, storage_unit_soc) + + opts = snakemake.wildcards.sector_opts.split('-') + solve_opts = snakemake.config['solving']['options'] + solve_opts['skip_iterations'] = True + + n = prepare_network(n, solve_opts) + + n = solve_network_myopic( + n, + config=snakemake.config, + opts=opts, + solver_dir=tmpdir, + solver_logfile=snakemake.log.solver + ) + + n.export_to_netcdf(snakemake.output[0]) From 7554e4f47203fc7a8affea99ec83d0d45feacde4 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 1 Aug 2022 11:58:20 +0200 Subject: [PATCH 065/173] operations: adjust output file path --- Snakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 981c340d..79d78274 100644 --- a/Snakefile +++ b/Snakefile @@ -687,7 +687,7 @@ rule solve_operations_network: input: pre=RDIR + "/prenetworks/elec{weather_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", post=RDIR + "/postnetworks/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - output: "elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc" + output: RDIR + "/operations/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc" shadow: "shallow" log: solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", @@ -708,7 +708,7 @@ rule solve_operations_network_myopic: pre=RDIR + "/prenetworks/elec{weather_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", post=RDIR + "/postnetworks/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", previous=solved_previous_year - output: "elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc" + output: RDIR + "/operations/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc" shadow: "shallow" log: solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", From a58d67c6c12bb9389b79eb6102a49320c4a9d2af Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 1 Aug 2022 11:59:57 +0200 Subject: [PATCH 066/173] operations: add config options to config.default.yaml --- config.default.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/config.default.yaml b/config.default.yaml index 3e6f76a0..3f1637bf 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -385,6 +385,20 @@ solving: #feasopt_tolerance: 1.e-6 mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 +operations: + rolling_horizon: + window: 10 # days + overlap: 8 # days + bidding_prices: # EUR/MW + H2 Electrolysis: -10 + H2 Fuel Cell: 200 + urban central water tanks charger: -10 + urban central water tanks discharger: 10 + hydro: 70 + solid biomass: 150 + biogas: 100 + co2_price: 500 # EUR/t + co2_sequestation_limit: 200 # Mt/a plotting: map: From 26b21144c67cb0f14ba1b9b3a50ab13973c7d483 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 1 Aug 2022 12:03:24 +0200 Subject: [PATCH 067/173] operations: add some comments on multiyear contiguity --- scripts/solve_operations_network_myopic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/solve_operations_network_myopic.py b/scripts/solve_operations_network_myopic.py index a0e30b0a..fae86cb7 100644 --- a/scripts/solve_operations_network_myopic.py +++ b/scripts/solve_operations_network_myopic.py @@ -21,9 +21,11 @@ def prepare_myopic(n, config, store_soc, storage_unit_soc): biomass_stores = n.stores.carrier.str.isin(["solid biomass", "biogas"]) biomass_potential = n.stores.loc[biomass_stores, "e_initial"] + # storage level contiguity across years n.stores.e_initial = store_soc n.storage_units.state_of_charge_initial = storage_unit_soc + # replace co2 limit with co2 price n.remove("GlobalConstraint", "CO2Limit") n.stores.at["co2 atmosphere", "marginal_cost"] = -config["co2_price"] @@ -44,7 +46,6 @@ def prepare_myopic(n, config, store_soc, storage_unit_soc): n.stores.loc[biomass_stores, "e_initial"] = biomass_potential # set storage bidding prices - # TODO bidding prices for Fischer-Tropsch, Methanation for H2? bidding_prices = config["bidding_prices"] for c in n.iterate_components({"Store", "Link", "StorageUnit"}): c.df.marginal_cost.update(c.df.carrier.map(bidding_prices).dropna()) From 574b77fcc3dcd7f97bbccc71917a14ad2fc7695f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 1 Aug 2022 12:11:41 +0200 Subject: [PATCH 068/173] operations: add final iteration until end-of-year --- scripts/solve_operations_network_myopic.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/solve_operations_network_myopic.py b/scripts/solve_operations_network_myopic.py index fae86cb7..c403465f 100644 --- a/scripts/solve_operations_network_myopic.py +++ b/scripts/solve_operations_network_myopic.py @@ -83,6 +83,10 @@ def solve_network_myopic(n, config, opts='', **kwargs): n.stores.e_initial = n.stores_t.e.loc[last_kept] n.storage_units.state_of_charge_initial = n.storage_units_t.state_of_charge.loc[last_kept] + # final segment until end of year + snapshots = n.snapshots[(i + 1) * kept - 1:] + n = solve_network(n, config, opts=opts, snapshots=snapshots, **kwargs) + return n From 26e7e1246b7e748f1a3d3093d1fa9160ab925f9f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 1 Aug 2022 12:14:45 +0200 Subject: [PATCH 069/173] operations: final iteration one step later --- scripts/solve_operations_network_myopic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/solve_operations_network_myopic.py b/scripts/solve_operations_network_myopic.py index c403465f..0591ee70 100644 --- a/scripts/solve_operations_network_myopic.py +++ b/scripts/solve_operations_network_myopic.py @@ -84,7 +84,7 @@ def solve_network_myopic(n, config, opts='', **kwargs): n.storage_units.state_of_charge_initial = n.storage_units_t.state_of_charge.loc[last_kept] # final segment until end of year - snapshots = n.snapshots[(i + 1) * kept - 1:] + snapshots = n.snapshots[(i + 1) * kept:] n = solve_network(n, config, opts=opts, snapshots=snapshots, **kwargs) return n From a65cf7cac48c62371723c6d28d2dc8bef8370bcd Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 5 Aug 2022 17:44:08 +0200 Subject: [PATCH 070/173] make_summary: add weather_year wildcard to multiindex --- scripts/make_summary.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 8d5f4e48..358b7b5c 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -528,7 +528,7 @@ def make_summaries(networks_dict): columns = pd.MultiIndex.from_tuples( networks_dict.keys(), - names=["cluster", "lv", "opt", "planning_horizon"] + names=["weather_year", "cluster", "lv", "opt", "planning_horizon"] ) df = {} @@ -562,8 +562,9 @@ if __name__ == "__main__": snakemake = mock_snakemake('make_summary') networks_dict = { - (cluster, lv, opt+sector_opt, planning_horizon) : - snakemake.config['results_dir'] + snakemake.config['run'] + f'/postnetworks/elec_s{simpl}_{cluster}_lv{lv}_{opt}_{sector_opt}_{planning_horizon}.nc' \ + (weather_year, cluster, lv, opt+sector_opt, planning_horizon) : + snakemake.config['results_dir'] + snakemake.config['run'] + f'/postnetworks/elec{weather_year}_s{simpl}_{cluster}_lv{lv}_{opt}_{sector_opt}_{planning_horizon}.nc' \ + for weather_year in snakemake.config['scenario']['weather_year'] for simpl in snakemake.config['scenario']['simpl'] \ for cluster in snakemake.config['scenario']['clusters'] \ for opt in snakemake.config['scenario']['opts'] \ From c1a7a5476f0255716e2895bbf1849a4889d126c3 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 5 Aug 2022 17:44:31 +0200 Subject: [PATCH 071/173] plot_network: geomap for PyPSA 0.20 --- scripts/plot_network.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/plot_network.py b/scripts/plot_network.py index b9dd4567..9643cc44 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -307,7 +307,7 @@ def plot_h2_map(network): ) n.plot( - geomap=False, + # geomap=False, bus_sizes=0, link_colors='#72d3d6', link_widths=link_widths_retro, @@ -441,7 +441,7 @@ def plot_ch4_map(network): ) n.plot( - geomap=False, + # geomap=False, ax=ax, bus_sizes=0., link_colors='#e8d1d1', @@ -451,7 +451,7 @@ def plot_ch4_map(network): ) n.plot( - geomap=False, + # geomap=False, ax=ax, bus_sizes=0., link_colors=link_color_used, From b962f133d8d845b163c9f59f6ef4ad58c64d3df6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 31 Aug 2022 15:19:44 +0200 Subject: [PATCH 072/173] add simpl wildcard to operation rules --- Snakefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Snakefile b/Snakefile index 79d78274..ee7c5815 100644 --- a/Snakefile +++ b/Snakefile @@ -685,9 +685,9 @@ if config["foresight"] == "myopic": rule solve_operations_network: input: - pre=RDIR + "/prenetworks/elec{weather_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - post=RDIR + "/postnetworks/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - output: RDIR + "/operations/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc" + pre=RDIR + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + post=RDIR + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + output: RDIR + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc" shadow: "shallow" log: solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", @@ -705,10 +705,10 @@ def solved_previous_year(wildcards): rule solve_operations_network_myopic: input: - pre=RDIR + "/prenetworks/elec{weather_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - post=RDIR + "/postnetworks/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + pre=RDIR + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + post=RDIR + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", previous=solved_previous_year - output: RDIR + "/operations/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc" + output: RDIR + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc" shadow: "shallow" log: solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", From 7a7c7f0344f6d92230599e170845ab3075438791 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 10 Sep 2022 19:48:40 +0200 Subject: [PATCH 073/173] add missing overrides to snakemake.input for operational rules --- Snakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Snakefile b/Snakefile index ee7c5815..c11fba06 100644 --- a/Snakefile +++ b/Snakefile @@ -685,6 +685,7 @@ if config["foresight"] == "myopic": rule solve_operations_network: input: + overrides="data/override_component_attrs", pre=RDIR + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", post=RDIR + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", output: RDIR + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc" @@ -705,6 +706,7 @@ def solved_previous_year(wildcards): rule solve_operations_network_myopic: input: + overrides="data/override_component_attrs", pre=RDIR + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", post=RDIR + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", previous=solved_previous_year From bb7e2e7dc9e5c49bbe611c6320bdc3e3bfe469ae Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 12 Sep 2022 11:28:06 +0200 Subject: [PATCH 074/173] add reference to demand_hourly.csv on zenodo --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 0de698c9..161da8a9 100644 --- a/Snakefile +++ b/Snakefile @@ -81,7 +81,7 @@ if config["enable"].get('retrieve_opsd_load_data', True): if config["enable"].get('retrieve_artificial_load_data', False): rule retrieve_artificial_load_data: - input: HTTP.remote("https://sandbox.zenodo.org/record/1089549/files/demand_hourly.csv", keep_local=True, static=True) + input: HTTP.remote("https://zenodo.org/record/7070438/files/demand_hourly.csv", keep_local=True, static=True) output: "data/load_artificial_raw.csv" run: move(input[0], output[0]) From e16fa286ec505cb18c1e9cad23034dffc16e9fd2 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 29 Apr 2023 12:41:37 +0200 Subject: [PATCH 075/173] merge master --- doc/configtables/load.csv | 2 +- rules/collect.smk | 14 ++++++++++---- scripts/add_electricity.py | 23 +++++++++++------------ scripts/add_extra_components.py | 6 ++++-- scripts/build_cutout.py | 8 ++++---- scripts/build_electricity_demand.py | 16 +++++++++------- scripts/build_hydro_profile.py | 16 +++++++--------- scripts/build_renewable_profiles.py | 4 +++- scripts/cluster_network.py | 4 +++- scripts/prepare_network.py | 7 ++++++- 10 files changed, 58 insertions(+), 42 deletions(-) diff --git a/doc/configtables/load.csv b/doc/configtables/load.csv index 3eeb3a65..a206cc23 100644 --- a/doc/configtables/load.csv +++ b/doc/configtables/load.csv @@ -8,4 +8,4 @@ interpolate_limit,hours,integer,"Maximum gap size (consecutive nans) which inter time_shift_for_large_gaps,string,string,"Periods which are used for copying time-slices in order to fill large gaps of nans. Have to be valid ``pandas`` period strings." manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`." scaling_factor,--,float,"Global correction factor for the load time series." -fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year" \ No newline at end of file +fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year" diff --git a/rules/collect.smk b/rules/collect.smk index 603e665e..5092d67e 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -22,20 +22,25 @@ rule all: rule cluster_networks: input: - expand(RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}.nc", **config["scenario"]), + expand( + RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}.nc", + **config["scenario"] + ), rule extra_components_networks: input: expand( - RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc", **config["scenario"] + RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc", + **config["scenario"] ), rule prepare_elec_networks: input: expand( - RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + RESOURCES + + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config["scenario"] ), @@ -52,7 +57,8 @@ rule prepare_sector_networks: rule solve_elec_networks: input: expand( - RESULTS + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + RESULTS + + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config["scenario"] ), diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 0ecd415b..87c06aaf 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -701,19 +701,20 @@ def add_nice_carrier_names(n, config): n.carriers["color"] = colors - def drop_leap_day(n): - if not n.snapshots.is_leap_year.any(): return + if not n.snapshots.is_leap_year.any(): + return leap_days = (n.snapshots.day == 29) & (n.snapshots.month == 2) n.set_snapshots(n.snapshots[~leap_days]) - n.snapshot_weightings[:] = 8760/len(n.snapshots) + n.snapshot_weightings[:] = 8760 / len(n.snapshots) logger.info("Dropped February 29 from leap year.") if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('add_electricity', weather_year='') + + snakemake = mock_snakemake("add_electricity", weather_year="") configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) @@ -721,15 +722,13 @@ if __name__ == "__main__": weather_year = snakemake.wildcards.weather_year if weather_year: snapshots = dict( - start=weather_year, - end=str(int(weather_year)+1), - closed="left" + start=weather_year, end=str(int(weather_year) + 1), closed="left" ) else: - snapshots = snakemake.config['snapshots'] - n.set_snapshots(pd.date_range(freq='h', **snapshots)) - - Nyears = n.snapshot_weightings.objective.sum() / 8760. + snapshots = snakemake.config["snapshots"] + n.set_snapshots(pd.date_range(freq="h", **snapshots)) + + Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( snakemake.input.tech_costs, @@ -855,7 +854,7 @@ if __name__ == "__main__": add_nice_carrier_names(n, snakemake.config) - if snakemake.config['enable'].get('drop_leap_days', True): + if snakemake.config["enable"].get("drop_leap_days", True): drop_leap_day(n) n.meta = snakemake.config diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 8a3496e2..91abbad6 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -230,8 +230,10 @@ def attach_hydrogen_pipelines(n, costs, elec_opts): if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('add_extra_components', weather_year='', - simpl='', clusters=5) + + snakemake = mock_snakemake( + "add_extra_components", weather_year="", simpl="", clusters=5 + ) configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 3ce15bc9..60da4f55 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -108,15 +108,15 @@ if __name__ == "__main__": cutout_params = snakemake.config["atlite"]["cutouts"][snakemake.wildcards.cutout] - if hasattr(snakemake.wildcards, 'weather_year'): + if hasattr(snakemake.wildcards, "weather_year"): time = snakemake.wildcards.weather_year cutout_params["time"] = [time, time] if "time" not in cutout_params: - snapshots = pd.date_range(freq='h', **snakemake.config['snapshots']) - cutout_params["time"] = [snapshots[0], snapshots[-1]] + snapshots = pd.date_range(freq="h", **snakemake.config["snapshots"]) + cutout_params["time"] = [snapshots[0], snapshots[-1]] - cutout_params['time'] = slice(*cutout_params['time']) + cutout_params["time"] = slice(*cutout_params["time"]) if {"x", "y", "bounds"}.isdisjoint(cutout_params): # Determine the bounds from bus regions with a buffer of two grid cells diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index 6f16d336..4be46014 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -275,23 +275,25 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("build_electricity_demand", weather_year='') + snakemake = mock_snakemake("build_electricity_demand", weather_year="") configure_logging(snakemake) weather_year = snakemake.wildcards.weather_year if weather_year: snapshots = dict( - start=weather_year, - end=str(int(weather_year)+1), - closed="left" + start=weather_year, end=str(int(weather_year) + 1), closed="left" ) else: - snapshots = snakemake.config['snapshots'] - snapshots = pd.date_range(freq='h', **snapshots) + snapshots = snakemake.config["snapshots"] + snapshots = pd.date_range(freq="h", **snapshots) fixed_year = snakemake.config["load"].get("fixed_year", False) - years = slice(str(fixed_year), str(fixed_year)) if fixed_year else slice(snapshots[0], snapshots[-1]) + years = ( + slice(str(fixed_year), str(fixed_year)) + if fixed_year + else slice(snapshots[0], snapshots[-1]) + ) powerstatistics = snakemake.config["load"]["power_statistics"] interpolate_limit = snakemake.config["load"]["interpolate_limit"] diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 4996c501..708c0c8d 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -65,9 +65,8 @@ import atlite import country_converter as coco import geopandas as gpd import pandas as pd -from numpy.polynomial import Polynomial from _helpers import configure_logging - +from numpy.polynomial import Polynomial cc = coco.CountryConverter() @@ -132,11 +131,10 @@ def correct_eia_stats_by_capacity(eia_stats, fn, countries, baseyear=2019): eia_stats_corrected = eia_stats / ratio to_keep = ["AL", "AT", "CH", "DE", "GB", "NL", "RS", "RO", "SK"] to_correct = eia_stats_corrected.columns.difference(to_keep) - eia_stats.loc[:,to_correct] = eia_stats_corrected.loc[:,to_correct] + eia_stats.loc[:, to_correct] = eia_stats_corrected.loc[:, to_correct] def approximate_missing_eia_stats(eia_stats, runoff_fn, countries): - runoff = pd.read_csv(runoff_fn, index_col=0).T[countries] # fix ES, PT data points @@ -147,7 +145,6 @@ def approximate_missing_eia_stats(eia_stats, runoff_fn, countries): eia_stats_approximated = {} for c in countries: - X = runoff_eia[c] Y = eia_stats[c] @@ -168,7 +165,8 @@ logger = logging.getLogger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('build_hydro_profile', weather_year='') + + snakemake = mock_snakemake("build_hydro_profile", weather_year="") configure_logging(snakemake) config_hydro = snakemake.config["renewable"]["hydro"] @@ -185,18 +183,18 @@ if __name__ == "__main__": fn = snakemake.input.eia_hydro_generation eia_stats = get_eia_annual_hydro_generation(fn, countries) - if config_hydro.get('eia_correct_by_capacity'): + if config_hydro.get("eia_correct_by_capacity"): fn = snakemake.input.eia_hydro_capacity correct_eia_stats_by_capacity(eia_stats, fn, countries) - if config_hydro.get('eia_approximate_missing'): + if config_hydro.get("eia_approximate_missing"): fn = snakemake.input.era5_runoff eia_stats = approximate_missing_eia_stats(eia_stats, fn, countries) eia_stats.to_csv(snakemake.output.eia_hydro) weather_year = snakemake.wildcards.weather_year - norm_year = config_hydro.get('eia_norm_year') + norm_year = config_hydro.get("eia_norm_year") if norm_year: eia_stats.loc[weather_year] = eia_stats.loc[norm_year] elif weather_year and weather_year not in eia_stats.index: diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 43bc05a1..acdc46de 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -199,7 +199,9 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("build_renewable_profiles", technology="solar", weather_year="") + snakemake = mock_snakemake( + "build_renewable_profiles", technology="solar", weather_year="" + ) configure_logging(snakemake) nprocesses = int(snakemake.threads) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 0cb70069..95445caf 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -457,7 +457,9 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("cluster_network", simpl="", clusters="5", weather_year="") + snakemake = mock_snakemake( + "cluster_network", simpl="", clusters="5", weather_year="" + ) configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index eba20033..72d13d2d 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -243,7 +243,12 @@ if __name__ == "__main__": from _helpers import mock_snakemake snakemake = mock_snakemake( - "prepare_network", weather_year="", simpl="", clusters="40", ll="v0.3", opts="Co2L-24H" + "prepare_network", + weather_year="", + simpl="", + clusters="40", + ll="v0.3", + opts="Co2L-24H", ) configure_logging(snakemake) From add135fe0527c2bf29cbfcbcf1639931fee057bb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 30 Apr 2023 08:52:58 +0000 Subject: [PATCH 076/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- data/eia_hydro_annual_capacity.csv | 2 +- doc/configtables/hydro.csv | 2 +- rules/build_sector.smk | 2 +- scripts/build_artificial_load_data.py | 42 +++--- scripts/build_biomass_potentials.py | 4 +- scripts/build_clustered_population_layouts.py | 3 +- scripts/build_energy_totals.py | 16 ++- scripts/build_heat_demand.py | 11 +- scripts/build_heat_totals.py | 25 ++-- scripts/build_population_layouts.py | 6 +- ...build_population_weighted_energy_totals.py | 8 +- scripts/build_solar_thermal_profiles.py | 12 +- scripts/build_temperature_profiles.py | 8 +- scripts/build_transport_demand.py | 10 +- scripts/cluster_gas_network.py | 4 +- scripts/make_summary.py | 5 +- scripts/prepare_sector_network.py | 8 +- .../solve_operations_network_other_year.py | 125 ++++++++++-------- ...ve_operations_network_other_year_myopic.py | 103 +++++++++------ 19 files changed, 230 insertions(+), 166 deletions(-) diff --git a/data/eia_hydro_annual_capacity.csv b/data/eia_hydro_annual_capacity.csv index a95d71a4..e14697bc 100644 --- a/data/eia_hydro_annual_capacity.csv +++ b/data/eia_hydro_annual_capacity.csv @@ -47,4 +47,4 @@ Report generated on: 07-21-2022 16:15:18 "INTL.33-7-SWE-MK.A"," Sweden","14.859","14.919","15.215","15.29","15.445","15.69","15.813","15.996","16.112","15.759","15.904","15.891","16.021","15.867","16.072","15.725","15.776","16.371","16.169","16.432","16.506","16.523","16.187","16.098","16.302","16.302","16.234","16.592","16.352","16.544","16.624","16.478","16.315","16.395","15.897","16.23","16.367","16.403","16.332","16.332","16.379" "INTL.33-7-CHE-MK.A"," Switzerland","11.45","11.46","11.47","11.47","11.48","11.48","11.51","11.51","11.52","11.58","3.474","3.484","3.504","3.509","3.526","3.541","3.55","3.553","3.584","3.614","3.636","3.642","3.653","3.669","3.65","3.682","3.694","3.7","3.709","3.749","3.81","3.852","3.882","3.896","3.948","3.996","4.06","4.112","4.193","4.193","4.193" "INTL.33-7-TUR-MK.A"," Turkey","2.131","2.356","3.082","3.239","3.875","3.875","3.878","5.003","6.219","6.598","6.764","7.114","8.379","9.682","9.865","9.863","9.935","10.102","10.307","10.537","11.175","11.673","12.241","12.579","12.645","12.906","13.063","13.395","13.829","14.553","15.831","17.137","19.609","22.289","23.643","25.868","26.681","27.273","28.291","28.503","30.984" -"INTL.33-7-GBR-MK.A"," United Kingdom","2.451","2.451","2.451","2.721","4.188","4.19","4.192","4.197","4.196","1.424","1.11","1.415","1.423","1.425","1.425","1.432","1.455","1.488","1.475","1.477","1.485","1.629","1.59","1.486","1.499","1.501","1.515","1.522","1.626","1.638","1.637","1.673","1.693","1.709","1.73","1.777","1.836","1.873","1.878","1.878","1.879" \ No newline at end of file +"INTL.33-7-GBR-MK.A"," United Kingdom","2.451","2.451","2.451","2.721","4.188","4.19","4.192","4.197","4.196","1.424","1.11","1.415","1.423","1.425","1.425","1.432","1.455","1.488","1.475","1.477","1.485","1.629","1.59","1.486","1.499","1.501","1.515","1.522","1.626","1.638","1.637","1.673","1.693","1.709","1.73","1.777","1.836","1.873","1.878","1.878","1.879" diff --git a/doc/configtables/hydro.csv b/doc/configtables/hydro.csv index 6016e6f1..482671ad 100644 --- a/doc/configtables/hydro.csv +++ b/doc/configtables/hydro.csv @@ -6,4 +6,4 @@ hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero." eia_norm_year,--,"Year in EIA hydro generation dataset; or False to disable","To specify a specific year by which hydro inflow is normed that deviates from the snapshots' year" eia_correct_by_capacity,--,boolean,"Correct EIA annual hydro generation data by installed capacity." -eia_approximate_missing,--,boolean,"Approximate hydro generation data for years not included in EIA dataset through a regression based on annual runoff." \ No newline at end of file +eia_approximate_missing,--,boolean,"Approximate hydro generation data for years not included in EIA dataset through a regression based on annual runoff." diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 30ce4095..38d5406f 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -262,7 +262,7 @@ rule build_heat_totals: resources: mem_mb=2000 log: LOGS + "build_heat_totals.log", - benchmark: + benchmark: BENCHMARKS + "build_heat_totals", conda: "../envs/environment.yaml" diff --git a/scripts/build_artificial_load_data.py b/scripts/build_artificial_load_data.py index d4d6449e..7703ff28 100755 --- a/scripts/build_artificial_load_data.py +++ b/scripts/build_artificial_load_data.py @@ -1,45 +1,47 @@ +# -*- coding: utf-8 -*- # SPDX-FileCopyrightText: 2022 The PyPSA-Eur Authors # # SPDX-License-Identifier: MIT - -"""This rule downloads the load data""" +""" +This rule downloads the load data. +""" import logging + logger = logging.getLogger(__name__) +import pandas as pd from _helpers import configure_logging -import pandas as pd - if __name__ == "__main__": - - if 'snakemake' not in globals(): + if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('build_artificial_load_data', weather_year='') + + snakemake = mock_snakemake("build_artificial_load_data", weather_year="") configure_logging(snakemake) weather_year = snakemake.wildcards.weather_year if weather_year: snapshots = dict( - start=weather_year, - end=str(int(weather_year)+1), - inclusive="left" + start=weather_year, end=str(int(weather_year) + 1), inclusive="left" ) else: - snapshots = snakemake.config['snapshots'] - snapshots = pd.date_range(freq='h', **snapshots) + snapshots = snakemake.config["snapshots"] + snapshots = pd.date_range(freq="h", **snapshots) fixed_year = snakemake.config["load"].get("fixed_year", False) - years = slice(str(fixed_year), str(fixed_year)) if fixed_year else slice(snapshots[0], snapshots[-1]) - countries = snakemake.config['countries'] + years = ( + slice(str(fixed_year), str(fixed_year)) + if fixed_year + else slice(snapshots[0], snapshots[-1]) + ) + countries = snakemake.config["countries"] - load = pd.read_csv( - snakemake.input[0], - index_col=0, - parse_dates=True - ).loc[snapshots, countries] + load = pd.read_csv(snakemake.input[0], index_col=0, parse_dates=True).loc[ + snapshots, countries + ] - assert not load.isna().any().any(), 'Load data contains nans.' + assert not load.isna().any().any(), "Load data contains nans." if fixed_year: load.index = load.index.map(lambda t: t.replace(year=snapshots.year[0])) diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index dd22eed9..f87263b5 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -208,7 +208,9 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("build_biomass_potentials", weather_year="", simpl="", clusters="5") + snakemake = mock_snakemake( + "build_biomass_potentials", weather_year="", simpl="", clusters="5" + ) config = snakemake.config["biomass"] year = config["year"] diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index 174e9771..886f2489 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -25,7 +25,8 @@ if __name__ == "__main__": cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - if year: cutout_name = cutout_name.format(weather_year=year) + if year: + cutout_name = cutout_name.format(weather_year=year) cutout = atlite.Cutout(cutout_name) clustered_regions = ( diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 82a6cdb1..0eb5665c 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -73,7 +73,7 @@ to_ipcc = { def eurostat_per_country(country): - + country_fn = idees_rename.get(country, country) fn = snakemake.input.eurostat + f"/{country_fn}-Energy-balance-sheets-June-2021-edition.xlsb" @@ -91,7 +91,9 @@ def eurostat_per_country(country): def build_eurostat(countries, year=None): - """Return multi-index for all countries' energy data in TWh/a.""" + """ + Return multi-index for all countries' energy data in TWh/a. + """ nprocesses = snakemake.threads tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries), @@ -128,7 +130,9 @@ def build_eurostat(countries, year=None): def build_swiss(year=None): - """Return a pd.DataFrame of Swiss energy data in TWh/a""" + """ + Return a pd.DataFrame of Swiss energy data in TWh/a. + """ fn = snakemake.input.swiss @@ -350,7 +354,7 @@ def build_idees(countries, year=None): nprocesses = snakemake.threads disable_progress = snakemake.config["run"].get("disable_progressbar", False) - + func = partial(idees_per_country, year=year, base_dir=snakemake.input.idees) tqdm_kwargs = dict( ascii=False, @@ -359,7 +363,7 @@ def build_idees(countries, year=None): desc="Build from IDEES database", disable=disable_progress ) - + with mute_print(): with mp.Pool(processes=nprocesses) as pool: dfs = list(tqdm(pool.imap(func, countries), **tqdm_kwargs)) @@ -438,7 +442,7 @@ def build_energy_totals(countries, eurostat, swiss, idees): # fuel use for fuel in ["electricity", "total"]: - + slicer = idx[c, y, :, :, eurostat_sectors[sector]] fill_values = eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=[0,1]).sum() df.loc[to_fill, f"{fuel} {sector}"] = fill_values diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index fe039187..a52ad3d1 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -28,19 +28,18 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year if year: - snapshots = dict(start=year, end=str(int(year)+1), inclusive="left") + snapshots = dict(start=year, end=str(int(year) + 1), inclusive="left") cutout_name = cutout_name.format(weather_year=year) else: - snapshots = snakemake.config['snapshots'] - + snapshots = snakemake.config["snapshots"] + drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) - time = pd.date_range(freq='h', **snapshots) - daily = pd.date_range(freq='D', **snapshots) + time = pd.date_range(freq="h", **snapshots) + daily = pd.date_range(freq="D", **snapshots) if drop_leap_day: time = time[~((time.month == 2) & (time.day == 29))] daily = daily[~((daily.month == 2) & (daily.day == 29))] diff --git a/scripts/build_heat_totals.py b/scripts/build_heat_totals.py index 34d5dcdf..80f0fb30 100644 --- a/scripts/build_heat_totals.py +++ b/scripts/build_heat_totals.py @@ -1,15 +1,17 @@ -"""Approximate heat demand for all weather years.""" - -import pandas as pd +# -*- coding: utf-8 -*- +""" +Approximate heat demand for all weather years. +""" from itertools import product + +import pandas as pd from numpy.polynomial import Polynomial idx = pd.IndexSlice def approximate_heat_demand(energy_totals, hdd): - if isinstance(hdd, str): hdd = pd.read_csv(hdd, index_col=0).T hdd.index = hdd.index.astype(int) @@ -17,7 +19,6 @@ def approximate_heat_demand(energy_totals, hdd): demands = {} for kind, sector in product(["total", "electricity"], ["services", "residential"]): - row = idx[:, 2007:2015] col = f"{kind} {sector} space" demand = energy_totals.loc[row, col].unstack(0) @@ -25,7 +26,6 @@ def approximate_heat_demand(energy_totals, hdd): demand_approx = {} for c in countries: - Y = demand[c].dropna() X = hdd.loc[Y.index, c] @@ -39,22 +39,25 @@ def approximate_heat_demand(energy_totals, hdd): demand_approx = pd.DataFrame(demand_approx) demand_approx = pd.concat([demand, demand_approx]).sort_index() - demands[f"{kind} {sector} space"] = demand_approx.groupby(demand_approx.index).sum() + demands[f"{kind} {sector} space"] = demand_approx.groupby( + demand_approx.index + ).sum() demands = pd.concat(demands).unstack().T.clip(lower=0) demands.index.names = ["country", "year"] - + return demands if __name__ == "__main__": - if 'snakemake' not in globals(): + if "snakemake" not in globals(): from helper import mock_snakemake - snakemake = mock_snakemake('build_energy_totals') + + snakemake = mock_snakemake("build_energy_totals") hdd = pd.read_csv(snakemake.input.hdd, index_col=0).T - energy_totals = pd.read_csv(snakemake.input.energy_totals, index_col=[0,1]) + energy_totals = pd.read_csv(snakemake.input.energy_totals, index_col=[0, 1]) countries = hdd.columns diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index 62146443..40f313c6 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -20,8 +20,9 @@ import xarray as xr if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake + snakemake = mock_snakemake( - 'build_population_layouts', + "build_population_layouts", weather_year="", ) @@ -29,7 +30,8 @@ if __name__ == "__main__": cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - if year: cutout_name = cutout_name.format(weather_year=year) + if year: + cutout_name = cutout_name.format(weather_year=year) cutout = atlite.Cutout(cutout_name) grid_cells = cutout.grid.geometry diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index 607b9869..bc5f2d19 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -21,15 +21,15 @@ if __name__ == "__main__": config = snakemake.config["energy"] data_year = int(config["energy_totals_year"]) - if snakemake.wildcards.weather_year and snakemake.wildcards.kind == 'heat': + if snakemake.wildcards.weather_year and snakemake.wildcards.kind == "heat": data_year = int(snakemake.wildcards.weather_year) pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) - totals = pd.read_csv(snakemake.input.totals, index_col=[0,1]) - totals = totals.xs(data_year, level='year') + totals = pd.read_csv(snakemake.input.totals, index_col=[0, 1]) + totals = totals.xs(data_year, level="year") - nodal_totals = totals.loc[pop_layout.ct].fillna(0.) + nodal_totals = totals.loc[pop_layout.ct].fillna(0.0) nodal_totals.index = pop_layout.index nodal_totals = nodal_totals.multiply(pop_layout.fraction, axis=0) diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index 547074c3..dbca9b73 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -28,18 +28,18 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - config = snakemake.config['solar_thermal'] - + config = snakemake.config["solar_thermal"] + cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year if year: - snapshots = dict(start=year, end=str(int(year)+1), inclusive="left") + snapshots = dict(start=year, end=str(int(year) + 1), inclusive="left") cutout_name = cutout_name.format(weather_year=year) else: - snapshots = snakemake.config['snapshots'] - - time = pd.date_range(freq='h', **snapshots) + snapshots = snakemake.config["snapshots"] + + time = pd.date_range(freq="h", **snapshots) if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index 890a6a80..d5b58a94 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -31,12 +31,12 @@ if __name__ == "__main__": year = snakemake.wildcards.weather_year if year: - snapshots = dict(start=year, end=str(int(year)+1), inclusive="left") + snapshots = dict(start=year, end=str(int(year) + 1), inclusive="left") cutout_name = cutout_name.format(weather_year=year) else: - snapshots = snakemake.config['snapshots'] - - time = pd.date_range(freq='h', **snapshots) + snapshots = snakemake.config["snapshots"] + + time = pd.date_range(freq="h", **snapshots) if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index 3e059d91..a6a745de 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -163,7 +163,7 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_transport_demand", - weather_year='', + weather_year="", simpl="", clusters=48, ) @@ -179,8 +179,12 @@ if __name__ == "__main__": options = snakemake.config["sector"] year = snakemake.wildcards.weather_year - snapshots = dict(start=year, end=str(int(year)+1), inclusive="left") if year else snakemake.config['snapshots'] - snapshots = pd.date_range(freq='h', **snapshots, tz="UTC") + snapshots = ( + dict(start=year, end=str(int(year) + 1), inclusive="left") + if year + else snakemake.config["snapshots"] + ) + snapshots = pd.date_range(freq="h", **snapshots, tz="UTC") if snakemake.config["atlite"].get("drop_leap_day", False): leap_day = (snapshots.month == 2) & (snapshots.day == 29) snapshots = snapshots[~leap_day] diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index 1fd9d4f1..f1149ed5 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -108,7 +108,9 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("cluster_gas_network", weather_year="", simpl="", clusters="37") + snakemake = mock_snakemake( + "cluster_gas_network", weather_year="", simpl="", clusters="37" + ) logging.basicConfig(level=snakemake.config["logging"]["level"]) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index f0f614a2..1130f036 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -648,7 +648,8 @@ def make_summaries(networks_dict): ] columns = pd.MultiIndex.from_tuples( - networks_dict.keys(), names=["weather_year", "cluster", "ll", "opt", "planning_horizon"] + networks_dict.keys(), + names=["weather_year", "cluster", "ll", "opt", "planning_horizon"], ) df = {} @@ -688,7 +689,7 @@ if __name__ == "__main__": (weather_year, cluster, ll, opt + sector_opt, planning_horizon): "results/" + snakemake.params.RDIR + f"/postnetworks/elec_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc" - for weather_year in snakemake.config['scenario']['weather_year'] + for weather_year in snakemake.config["scenario"]["weather_year"] for simpl in snakemake.config["scenario"]["simpl"] for cluster in snakemake.config["scenario"]["clusters"] for opt in snakemake.config["scenario"]["opts"] diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 85a16124..055fd57b 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3272,8 +3272,12 @@ if __name__ == "__main__": nyears, ) - pop_weighted_energy_totals = pd.read_csv(snakemake.input.pop_weighted_energy_totals, index_col=0) * nyears - pop_weighted_heat_totals = pd.read_csv(snakemake.input.pop_weighted_heat_totals, index_col=0) * nyears + pop_weighted_energy_totals = ( + pd.read_csv(snakemake.input.pop_weighted_energy_totals, index_col=0) * nyears + ) + pop_weighted_heat_totals = ( + pd.read_csv(snakemake.input.pop_weighted_heat_totals, index_col=0) * nyears + ) pop_weighted_energy_totals.update(pop_weighted_heat_totals) patch_electricity_network(n) diff --git a/scripts/solve_operations_network_other_year.py b/scripts/solve_operations_network_other_year.py index 643db050..9b4d853f 100644 --- a/scripts/solve_operations_network_other_year.py +++ b/scripts/solve_operations_network_other_year.py @@ -1,50 +1,62 @@ -"""Solve operations network.""" +# -*- coding: utf-8 -*- +""" +Solve operations network. +""" -import pypsa -import numpy as np - -from solve_network import solve_network, prepare_network -from helper import override_component_attrs - import logging + +import numpy as np +import pypsa +from helper import override_component_attrs +from solve_network import prepare_network, solve_network + logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) def set_parameters_from_optimized(n, n_optim): - lines_typed_i = n.lines.index[n.lines.type != ''] - n.lines.loc[lines_typed_i, 'num_parallel'] = \ - n_optim.lines['num_parallel'].reindex(lines_typed_i, fill_value=0.) - n.lines.loc[lines_typed_i, 's_nom'] = ( - np.sqrt(3) * n.lines['type'].map(n.line_types.i_nom) * - n.lines.bus0.map(n.buses.v_nom) * n.lines.num_parallel) + lines_typed_i = n.lines.index[n.lines.type != ""] + n.lines.loc[lines_typed_i, "num_parallel"] = n_optim.lines["num_parallel"].reindex( + lines_typed_i, fill_value=0.0 + ) + n.lines.loc[lines_typed_i, "s_nom"] = ( + np.sqrt(3) + * n.lines["type"].map(n.line_types.i_nom) + * n.lines.bus0.map(n.buses.v_nom) + * n.lines.num_parallel + ) - lines_untyped_i = n.lines.index[n.lines.type == ''] - for attr in ('s_nom', 'r', 'x'): - n.lines.loc[lines_untyped_i, attr] = \ - n_optim.lines[attr].reindex(lines_untyped_i, fill_value=0.) - n.lines['s_nom_extendable'] = False + lines_untyped_i = n.lines.index[n.lines.type == ""] + for attr in ("s_nom", "r", "x"): + n.lines.loc[lines_untyped_i, attr] = n_optim.lines[attr].reindex( + lines_untyped_i, fill_value=0.0 + ) + n.lines["s_nom_extendable"] = False links_dc_i = n.links.index[n.links.p_nom_extendable] - n.links.loc[links_dc_i, 'p_nom'] = \ - n_optim.links['p_nom_opt'].reindex(links_dc_i, fill_value=0.) - n.links.loc[links_dc_i, 'p_nom_extendable'] = False + n.links.loc[links_dc_i, "p_nom"] = n_optim.links["p_nom_opt"].reindex( + links_dc_i, fill_value=0.0 + ) + n.links.loc[links_dc_i, "p_nom_extendable"] = False gen_extend_i = n.generators.index[n.generators.p_nom_extendable] - n.generators.loc[gen_extend_i, 'p_nom'] = \ - n_optim.generators['p_nom_opt'].reindex(gen_extend_i, fill_value=0.) - n.generators.loc[gen_extend_i, 'p_nom_extendable'] = False + n.generators.loc[gen_extend_i, "p_nom"] = n_optim.generators["p_nom_opt"].reindex( + gen_extend_i, fill_value=0.0 + ) + n.generators.loc[gen_extend_i, "p_nom_extendable"] = False stor_units_extend_i = n.storage_units.index[n.storage_units.p_nom_extendable] - n.storage_units.loc[stor_units_extend_i, 'p_nom'] = \ - n_optim.storage_units['p_nom_opt'].reindex(stor_units_extend_i, fill_value=0.) - n.storage_units.loc[stor_units_extend_i, 'p_nom_extendable'] = False + n.storage_units.loc[stor_units_extend_i, "p_nom"] = n_optim.storage_units[ + "p_nom_opt" + ].reindex(stor_units_extend_i, fill_value=0.0) + n.storage_units.loc[stor_units_extend_i, "p_nom_extendable"] = False stor_extend_i = n.stores.index[n.stores.e_nom_extendable] - n.stores.loc[stor_extend_i, 'e_nom'] = \ - n_optim.stores['e_nom_opt'].reindex(stor_extend_i, fill_value=0.) - n.stores.loc[stor_extend_i, 'e_nom_extendable'] = False + n.stores.loc[stor_extend_i, "e_nom"] = n_optim.stores["e_nom_opt"].reindex( + stor_extend_i, fill_value=0.0 + ) + n.stores.loc[stor_extend_i, "e_nom_extendable"] = False return n @@ -68,39 +80,44 @@ def add_load_shedding(n, voll=1e4): to_remove = n.generators.query("carrier == 'load'").index logger.info(f"Removing pre-existing load shedding:\n{to_remove}") n.mremove("Generator", to_remove) - - n.madd("Generator", n.buses.index, + + n.madd( + "Generator", + n.buses.index, suffix=" load", bus=n.buses.index, - carrier='load', + carrier="load", marginal_cost=voll, - p_nom=1e6 + p_nom=1e6, ) return n if __name__ == "__main__": - if 'snakemake' not in globals(): + if "snakemake" not in globals(): from helper import mock_snakemake + snakemake = mock_snakemake( - 'solve_operations_network', + "solve_operations_network", capacity_year=1952, - simpl='', - opts='', + simpl="", + opts="", clusters=37, lv=2.0, - sector_opts='Co2L0-25H-T-H-B-I-A', + sector_opts="Co2L0-25H-T-H-B-I-A", planning_horizons=2030, - weather_year=2013 + weather_year=2013, ) - logging.basicConfig(filename=snakemake.log.python, - level=snakemake.config['logging_level']) + logging.basicConfig( + filename=snakemake.log.python, level=snakemake.config["logging_level"] + ) - tmpdir = snakemake.config['solving'].get('tmpdir') + tmpdir = snakemake.config["solving"].get("tmpdir") if tmpdir is not None: from pathlib import Path + Path(tmpdir).mkdir(parents=True, exist_ok=True) overrides = override_component_attrs(snakemake.input.overrides) @@ -113,14 +130,18 @@ if __name__ == "__main__": n = remove_unused_components(n) n = add_load_shedding(n) - opts = snakemake.wildcards.sector_opts.split('-') - solve_opts = snakemake.config['solving']['options'] - solve_opts['skip_iterations'] = True + opts = snakemake.wildcards.sector_opts.split("-") + solve_opts = snakemake.config["solving"]["options"] + solve_opts["skip_iterations"] = True n = prepare_network(n, solve_opts) - - n = solve_network(n, config=snakemake.config, opts=opts, - solver_dir=tmpdir, - solver_logfile=snakemake.log.solver) - - n.export_to_netcdf(snakemake.output[0]) \ No newline at end of file + + n = solve_network( + n, + config=snakemake.config, + opts=opts, + solver_dir=tmpdir, + solver_logfile=snakemake.log.solver, + ) + + n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/solve_operations_network_other_year_myopic.py b/scripts/solve_operations_network_other_year_myopic.py index 0591ee70..093b94eb 100644 --- a/scripts/solve_operations_network_other_year_myopic.py +++ b/scripts/solve_operations_network_other_year_myopic.py @@ -1,20 +1,26 @@ -"""Solve myopic operations network.""" +# -*- coding: utf-8 -*- +""" +Solve myopic operations network. +""" -import pypsa -import pandas as pd - -from solve_network import solve_network, prepare_network -from solve_operations_network import set_parameters_from_optimized, remove_unused_components, add_load_shedding -from helper import override_component_attrs - import logging + +import pandas as pd +import pypsa +from helper import override_component_attrs +from solve_network import prepare_network, solve_network +from solve_operations_network import ( + add_load_shedding, + remove_unused_components, + set_parameters_from_optimized, +) + logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) def prepare_myopic(n, config, store_soc, storage_unit_soc): - n.stores.e_cyclic = False n.storage_units.cyclic_state_of_charge = False @@ -30,17 +36,19 @@ def prepare_myopic(n, config, store_soc, storage_unit_soc): n.stores.at["co2 atmosphere", "marginal_cost"] = -config["co2_price"] # handle co2 sequestration - assert sum(n.stores.carriers == "co2 stored") == 1, "Myopic operation not implemented for spatially resolved CO2 sequestration." - n.stores.at["co2 stored", 'e_nom'] = config['co2_sequestration_limit'] * 1e6 # t/a + assert ( + sum(n.stores.carriers == "co2 stored") == 1 + ), "Myopic operation not implemented for spatially resolved CO2 sequestration." + n.stores.at["co2 stored", "e_nom"] = config["co2_sequestration_limit"] * 1e6 # t/a # reset co2 emissions - n.stores.loc[n.stores.carrier == 'co2 stored', "e_initial"] = 0. - n.stores.at["co2 atmosphere", "e_initial"] = 0. + n.stores.loc[n.stores.carrier == "co2 stored", "e_initial"] = 0.0 + n.stores.at["co2 atmosphere", "e_initial"] = 0.0 # replenish fossil gas and oil with 1000 TWh each fossil_stores = n.stores.carrier.str.isin(["gas", "oil"]) - n.stores.loc[fossil_stores, 'e_initial'] = 1e9 - n.stores.loc[fossil_stores, 'e_nom'] = 10e9 + n.stores.loc[fossil_stores, "e_initial"] = 1e9 + n.stores.loc[fossil_stores, "e_nom"] = 10e9 # replenish annual solid biomass and biogas potentials n.stores.loc[biomass_stores, "e_initial"] = biomass_potential @@ -51,15 +59,18 @@ def prepare_myopic(n, config, store_soc, storage_unit_soc): c.df.marginal_cost.update(c.df.carrier.map(bidding_prices).dropna()) # deduct industry solid biomass - assert sum(n.stores.carriers == "solid biomass") == 1, "Myopic operation not implemented for spatially resolved solid biomass." - n.stores.at["EU solid biomass", "e_initial"] -= n.loads.at["solid biomass for industry", "p_set"] * 8760 + assert ( + sum(n.stores.carriers == "solid biomass") == 1 + ), "Myopic operation not implemented for spatially resolved solid biomass." + n.stores.at["EU solid biomass", "e_initial"] -= ( + n.loads.at["solid biomass for industry", "p_set"] * 8760 + ) n.remove("Load", "solid biomass for industry") return n -def solve_network_myopic(n, config, opts='', **kwargs): - +def solve_network_myopic(n, config, opts="", **kwargs): rolling_horizon = config["operations"]["rolling_horizon"] freq = int(pd.infer_freq(n.snapshots)[:-1]) @@ -68,11 +79,12 @@ def solve_network_myopic(n, config, opts='', **kwargs): kept = window - overlap length = len(n.snapshots) - assert kept > 0, f"Overlap ({overlap} days) must be smaller than windows ({window} days)." + assert ( + kept > 0 + ), f"Overlap ({overlap} days) must be smaller than windows ({window} days)." - for i in range(length // kept): - - snapshots = n.snapshots[i * kept:(i + 1) * kept + overlap] + for i in range(length // kept): + snapshots = n.snapshots[i * kept : (i + 1) * kept + overlap] logger.info(f"Optimising operations from {snapshots[0]} to {snapshots[-1]}") n = solve_network(n, config, opts=opts, snapshots=snapshots, **kwargs) @@ -81,36 +93,41 @@ def solve_network_myopic(n, config, opts='', **kwargs): logger.info(f"Setting initial SOCs from {last_kept} for next iteration.\n") n.stores.e_initial = n.stores_t.e.loc[last_kept] - n.storage_units.state_of_charge_initial = n.storage_units_t.state_of_charge.loc[last_kept] + n.storage_units.state_of_charge_initial = n.storage_units_t.state_of_charge.loc[ + last_kept + ] # final segment until end of year - snapshots = n.snapshots[(i + 1) * kept:] + snapshots = n.snapshots[(i + 1) * kept :] n = solve_network(n, config, opts=opts, snapshots=snapshots, **kwargs) return n if __name__ == "__main__": - if 'snakemake' not in globals(): + if "snakemake" not in globals(): from helper import mock_snakemake + snakemake = mock_snakemake( - 'solve_operations_network_myopic', + "solve_operations_network_myopic", capacity_year=1952, - simpl='', - opts='', + simpl="", + opts="", clusters=37, lv=2.0, - sector_opts='Co2L0-25H-T-H-B-I-A', + sector_opts="Co2L0-25H-T-H-B-I-A", planning_horizons=2030, - weather_year=2013 + weather_year=2013, ) - logging.basicConfig(filename=snakemake.log.python, - level=snakemake.config['logging_level']) + logging.basicConfig( + filename=snakemake.log.python, level=snakemake.config["logging_level"] + ) - tmpdir = snakemake.config['solving'].get('tmpdir') + tmpdir = snakemake.config["solving"].get("tmpdir") if tmpdir is not None: from pathlib import Path + Path(tmpdir).mkdir(parents=True, exist_ok=True) config = snakemake.config["operations"] @@ -121,8 +138,10 @@ if __name__ == "__main__": n_post = pypsa.Network(snakemake.input.post, override_component_attrs=overrides) n = set_parameters_from_optimized(n, n_post) del n_post - - n_previous = pypsa.Network(snakemake.input.previous, override_component_attrs=overrides) + + n_previous = pypsa.Network( + snakemake.input.previous, override_component_attrs=overrides + ) store_soc = n_previous.stores_t.e.iloc[-1] storage_unit_soc = n_previous.storage_units_t.state_of_charge.iloc[-1] del n_previous @@ -131,18 +150,18 @@ if __name__ == "__main__": n = add_load_shedding(n) n = prepare_myopic(n, config, store_soc, storage_unit_soc) - opts = snakemake.wildcards.sector_opts.split('-') - solve_opts = snakemake.config['solving']['options'] - solve_opts['skip_iterations'] = True + opts = snakemake.wildcards.sector_opts.split("-") + solve_opts = snakemake.config["solving"]["options"] + solve_opts["skip_iterations"] = True n = prepare_network(n, solve_opts) - + n = solve_network_myopic( n, config=snakemake.config, opts=opts, solver_dir=tmpdir, - solver_logfile=snakemake.log.solver + solver_logfile=snakemake.log.solver, ) - + n.export_to_netcdf(snakemake.output[0]) From e8772c33401d8e5ba4290678d820b18004f0d6d4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 30 Apr 2023 08:54:48 +0000 Subject: [PATCH 077/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_electricity.smk | 70 +++++++++++++++++++++++++------------ rules/postprocess.smk | 3 +- rules/solve_electricity.smk | 20 +++++++---- rules/solve_myopic.smk | 15 +++++--- rules/solve_overnight.smk | 64 +++++++++++++++++++++++---------- 5 files changed, 119 insertions(+), 53 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 6a67a0da..fdaedab9 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -17,7 +17,8 @@ if config["enable"].get("prepare_links_p_nom", False): script: "../scripts/prepare_links_p_nom.py" -if config["enable"].get('retrieve_opsd_load_data', True): + +if config["enable"].get("retrieve_opsd_load_data", True): rule build_electricity_demand: input: @@ -33,15 +34,23 @@ if config["enable"].get('retrieve_opsd_load_data', True): script: "../scripts/build_electricity_demand.py" -if config["enable"].get('retrieve_artificial_load_data', False): + +if config["enable"].get("retrieve_artificial_load_data", False): rule build_artificial_load_data: - input: ancient("data/load_artificial_raw.csv") - output: RESOURCES + "load{weather_year}.csv" - log: LOGS + "build_artificial_load_data{weather_year}.log" - resources: mem_mb=5000, - conda: "../envs/environment.yaml" - script: "../scripts/build_artificial_load_data.py" + input: + ancient("data/load_artificial_raw.csv"), + output: + RESOURCES + "load{weather_year}.csv", + log: + LOGS + "build_artificial_load_data{weather_year}.log", + resources: + mem_mb=5000, + conda: + "../envs/environment.yaml" + script: + "../scripts/build_artificial_load_data.py" + rule build_powerplants: input: @@ -155,15 +164,22 @@ if config["enable"].get("build_cutout", False): "../scripts/build_cutout.py" rule build_cutout_year: - input: rules.build_cutout.input - output: protected("cutouts/" + CDIR + "{cutout}-{weather_year}.nc") - log: "logs/" + CDIR + "build_cutout/{cutout}-{weather_year}.log" - benchmark: "benchmarks/" + CDIR + "build_cutout_{cutout}-{weather_year}" + input: + rules.build_cutout.input, + output: + protected("cutouts/" + CDIR + "{cutout}-{weather_year}.nc"), + log: + "logs/" + CDIR + "build_cutout/{cutout}-{weather_year}.log", + benchmark: + "benchmarks/" + CDIR + "build_cutout_{cutout}-{weather_year}" threads: ATLITE_NPROCESSES - resources: mem_mb=ATLITE_NPROCESSES * 1000 + resources: + mem_mb=ATLITE_NPROCESSES * 1000, conda: "../envs/environment.yaml" - script: "../scripts/build_cutout.py" + script: + "../scripts/build_cutout.py" + if config["enable"].get("build_natura_raster", False): @@ -319,8 +335,10 @@ rule simplify_network: regions_offshore=RESOURCES + "regions_offshore.geojson", output: network=RESOURCES + "networks/elec{weather_year}_s{simpl}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}.geojson", - regions_offshore=RESOURCES + "regions_offshore_elec{weather_year}_s{simpl}.geojson", + regions_onshore=RESOURCES + + "regions_onshore_elec{weather_year}_s{simpl}.geojson", + regions_offshore=RESOURCES + + "regions_offshore_elec{weather_year}_s{simpl}.geojson", busmap=RESOURCES + "busmap_elec{weather_year}_s{simpl}.csv", connection_costs=RESOURCES + "connection_costs{weather_year}_s{simpl}.csv", log: @@ -339,8 +357,10 @@ rule simplify_network: rule cluster_network: input: network=RESOURCES + "networks/elec{weather_year}_s{simpl}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}.geojson", - regions_offshore=RESOURCES + "regions_offshore_elec{weather_year}_s{simpl}.geojson", + regions_onshore=RESOURCES + + "regions_onshore_elec{weather_year}_s{simpl}.geojson", + regions_offshore=RESOURCES + + "regions_offshore_elec{weather_year}_s{simpl}.geojson", busmap=ancient(RESOURCES + "busmap_elec{weather_year}_s{simpl}.csv"), custom_busmap=( "data/custom_busmap_elec_s{simpl}_{clusters}.csv" @@ -350,8 +370,10 @@ rule cluster_network: tech_costs=COSTS, output: network=RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", - regions_offshore=RESOURCES + "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson", + regions_onshore=RESOURCES + + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", + regions_offshore=RESOURCES + + "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson", busmap=RESOURCES + "busmap_elec{weather_year}_s{simpl}_{clusters}.csv", linemap=RESOURCES + "linemap_elec{weather_year}_s{simpl}_{clusters}.csv", log: @@ -393,9 +415,13 @@ rule prepare_network: output: RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", log: - LOGS + "prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.log", + LOGS + + "prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.log", benchmark: - (BENCHMARKS + "prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}") + ( + BENCHMARKS + + "prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" + ) threads: 1 resources: mem_mb=4000, diff --git a/rules/postprocess.smk b/rules/postprocess.smk index c855b022..0a3260eb 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -13,7 +13,8 @@ rule plot_network: overrides="data/override_component_attrs", network=RESULTS + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - regions=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", + regions=RESOURCES + + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", output: map=RESULTS + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index b530f1d7..c63879aa 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -5,19 +5,25 @@ rule solve_network: input: - network=RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network=RESOURCES + + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: - network=RESULTS + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network=RESULTS + + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", log: solver=normpath( - LOGS + "solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log" + LOGS + + "solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log" ), python=LOGS + "solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", memory=LOGS + "solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log", benchmark: - BENCHMARKS + "solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" + ( + BENCHMARKS + + "solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" + ) threads: 4 resources: mem_mb=memory, @@ -31,9 +37,11 @@ rule solve_network: rule solve_operations_network: input: - network=RESULTS + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network=RESULTS + + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: - network=RESULTS + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc", + network=RESULTS + + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc", log: solver=normpath( LOGS diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 8fff7d21..ef9a5ade 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -11,10 +11,13 @@ rule add_existing_baseyear: powerplants=RESOURCES + "powerplants.csv", busmap_s=RESOURCES + "busmap_elec{weather_year}_s{simpl}.csv", busmap=RESOURCES + "busmap_elec{weather_year}_s{simpl}_{clusters}.csv", - clustered_pop_layout=RESOURCES + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", + clustered_pop_layout=RESOURCES + + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", costs="data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), - cop_soil_total=RESOURCES + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc", - cop_air_total=RESOURCES + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_soil_total=RESOURCES + + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_air_total=RESOURCES + + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", existing_heating="data/existing_infrastructure/existing_heating_raw.csv", existing_solar="data/existing_infrastructure/solar_capacity_IRENA.csv", existing_onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv", @@ -48,8 +51,10 @@ rule add_brownfield: + "prenetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", network_p=solved_previous_horizon, #solved network at previous time step costs="data/costs_{planning_horizons}.csv", - cop_soil_total=RESOURCES + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc", - cop_air_total=RESOURCES + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_soil_total=RESOURCES + + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_air_total=RESOURCES + + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", output: RESULTS + "prenetworks-brownfield/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 0829ce6a..e79efca2 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -41,31 +41,57 @@ rule solve_sector_network: rule solve_operations_network_other_year: input: overrides="data/override_component_attrs", - pre=RDIR + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - post=RDIR + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - output: RDIR + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc" - shadow: "shallow" + pre=RDIR + + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + post=RDIR + + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + output: + RDIR + + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc", + shadow: + "shallow" log: - solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", - python=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_python.log", + solver=RDIR + + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", + python=RDIR + + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_python.log", threads: 4 - resources: mem_mb=10000 - benchmark: RDIR + "/benchmarks/solve_operations_network/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}" - script: "../scripts/solve_operations_network_other_year.py" + resources: + mem_mb=10000, + benchmark: + ( + RDIR + + "/benchmarks/solve_operations_network/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}" + ) + script: + "../scripts/solve_operations_network_other_year.py" rule solve_operations_network_other_year_myopic: input: overrides="data/override_component_attrs", - pre=RDIR + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - post=RDIR + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - previous=solved_previous_year - output: RDIR + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc" - shadow: "shallow" + pre=RDIR + + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + post=RDIR + + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + previous=solved_previous_year, + output: + RDIR + + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc", + shadow: + "shallow" log: - solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", - python=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_python.log", + solver=RDIR + + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", + python=RDIR + + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_python.log", threads: 4 - resources: mem_mb=10000 - benchmark: RDIR + "/benchmarks/solve_operations_network_myopic/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}" - script: "../scripts/solve_operations_network_other_year_myopic.py" + resources: + mem_mb=10000, + benchmark: + ( + RDIR + + "/benchmarks/solve_operations_network_myopic/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}" + ) + script: + "../scripts/solve_operations_network_other_year_myopic.py" From d527797dc3edee0bfdf9656a2435751b93843035 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 26 Jul 2023 08:24:01 +0000 Subject: [PATCH 078/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_electricity.smk | 1 - 1 file changed, 1 deletion(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 8c16b058..d56484a9 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -56,7 +56,6 @@ if config["enable"].get("retrieve_artificial_load_data", False): "../scripts/build_artificial_load_data.py" - rule build_powerplants: params: powerplants_filter=config["electricity"]["powerplants_filter"], From 16eb4e644a9d6e1fd40feb4ddd7ee10f5207334f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 26 Jul 2023 10:52:58 +0200 Subject: [PATCH 079/173] small fixes to weather_year wildcard handling --- config/test/config.electricity.yaml | 2 ++ config/test/config.myopic.yaml | 2 ++ config/test/config.overnight.yaml | 2 ++ rules/build_electricity.smk | 2 +- rules/build_sector.smk | 2 +- rules/solve_overnight.smk | 56 ++++++++++++++--------------- 6 files changed, 36 insertions(+), 30 deletions(-) diff --git a/config/test/config.electricity.yaml b/config/test/config.electricity.yaml index 6798e38c..e491a767 100644 --- a/config/test/config.electricity.yaml +++ b/config/test/config.electricity.yaml @@ -12,6 +12,8 @@ run: shared_cutouts: true scenario: + weather_year: + - "" clusters: - 5 opts: diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index 0bb85ec6..edaccd61 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -13,6 +13,8 @@ run: foresight: myopic scenario: + weather_year: + - "" ll: - v1.5 clusters: diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index a2a0f5a4..3ceda43b 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -12,6 +12,8 @@ run: scenario: + weather_year: + - "" ll: - v1.5 clusters: diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index d56484a9..4ecdddae 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -326,7 +326,7 @@ rule add_electricity: costs=config["costs"], input: **{ - f"profile_{tech}": RESOURCES + f"profile{weather_year}_{tech}.nc" + f"profile_{tech}": RESOURCES + "profile{weather_year}" + f"_{tech}.nc" for tech in config["electricity"]["renewable_carriers"] }, **{ diff --git a/rules/build_sector.smk b/rules/build_sector.smk index c430a2ac..26adea50 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -748,7 +748,7 @@ rule prepare_sector_network: pop_weighted_energy_totals=RESOURCES + "pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv", pop_weighted_heat_totals=RESOURCES - + "pop_weighted_heat_totals{weather_year}_s{simpl}_{clusters}.csv" + + "pop_weighted_heat_totals{weather_year}_s{simpl}_{clusters}.csv", shipping_demand=RESOURCES + "shipping_demand{weather_year}_s{simpl}_{clusters}.csv", transport_demand=RESOURCES + "transport_demand{weather_year}_s{simpl}_{clusters}.csv", transport_data=RESOURCES + "transport_data{weather_year}_s{simpl}_{clusters}.csv", diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 86f0724d..6a4c7f7c 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -71,31 +71,31 @@ rule solve_operations_network_other_year: "../scripts/solve_operations_network_other_year.py" -rule solve_operations_network_other_year_myopic: - input: - overrides="data/override_component_attrs", - pre=RDIR - + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - post=RDIR - + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - previous=solved_previous_year, - output: - RDIR - + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc", - shadow: - "shallow" - log: - solver=RDIR - + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", - python=RDIR - + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_python.log", - threads: 4 - resources: - mem_mb=10000, - benchmark: - ( - RDIR - + "/benchmarks/solve_operations_network_myopic/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}" - ) - script: - "../scripts/solve_operations_network_other_year_myopic.py" +# rule solve_operations_network_other_year_myopic: +# input: +# overrides="data/override_component_attrs", +# pre=RDIR +# + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", +# post=RDIR +# + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", +# previous=solved_previous_year, +# output: +# RDIR +# + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc", +# shadow: +# "shallow" +# log: +# solver=RDIR +# + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", +# python=RDIR +# + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_python.log", +# threads: 4 +# resources: +# mem_mb=10000, +# benchmark: +# ( +# RDIR +# + "/benchmarks/solve_operations_network_myopic/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}" +# ) +# script: +# "../scripts/solve_operations_network_other_year_myopic.py" From 7e290d3e5281c6580efc25af4d33ed402f74186a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 8 Oct 2023 09:55:36 +0000 Subject: [PATCH 080/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/configtables/hydro.csv | 16 +-- doc/configtables/toplevel.csv | 24 ++-- rules/build_sector.smk | 240 +++++++++++++++++++++++----------- 3 files changed, 184 insertions(+), 96 deletions(-) diff --git a/doc/configtables/hydro.csv b/doc/configtables/hydro.csv index ad4bd7aa..f9c8f0d5 100644 --- a/doc/configtables/hydro.csv +++ b/doc/configtables/hydro.csv @@ -9,12 +9,12 @@ eia_norm_year,--,"Year in EIA hydro generation dataset; or False to disable","To eia_correct_by_capacity,--,boolean,"Correct EIA annual hydro generation data by installed capacity." eia_approximate_missing,--,boolean,"Approximate hydro generation data for years not included in EIA dataset through a regression based on annual runoff." ======= -,Unit,Values,Description -cutout,--,Must be 'europe-2013-era5',Specifies the directory where the relevant weather data ist stored. -carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydro power plants to build per-unit availability time series for. 'ror' stands for run-of-river plants, 'PHS' represents pumped-hydro storage, and 'hydro' stands for hydroelectric dams." -PHS_max_hours,h,float,Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. -hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}",Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation `_. -flatten_dispatch,bool,"{true, false}",Consider an upper limit for the hydro dispatch. The limit is given by the average capacity factor plus the buffer given in ``flatten_dispatch_buffer`` -flatten_dispatch_buffer,--,float,"If ``flatten_dispatch`` is true, specify the value added above the average capacity factor." -clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero." +,Unit,Values,Description +cutout,--,Must be 'europe-2013-era5',Specifies the directory where the relevant weather data ist stored. +carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydro power plants to build per-unit availability time series for. 'ror' stands for run-of-river plants, 'PHS' represents pumped-hydro storage, and 'hydro' stands for hydroelectric dams." +PHS_max_hours,h,float,Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. +hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}",Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation `_. +flatten_dispatch,bool,"{true, false}",Consider an upper limit for the hydro dispatch. The limit is given by the average capacity factor plus the buffer given in ``flatten_dispatch_buffer`` +flatten_dispatch_buffer,--,float,"If ``flatten_dispatch`` is true, specify the value added above the average capacity factor." +clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero." >>>>>>> master diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv index 28c37206..877b234b 100644 --- a/doc/configtables/toplevel.csv +++ b/doc/configtables/toplevel.csv @@ -25,16 +25,16 @@ co2_budget,--,"Dictionary with planning horizons as keys.","CO2 budget as a frac ======= >>>>>>> origin/master ======= -,Unit,Values,Description -version,--,0.x.x,Version of PyPSA-Eur. Descriptive only. -tutorial,bool,"{true, false}",Switch to retrieve the tutorial data set instead of the full data set. -logging,,, --- level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only" --- format,--,,Custom format for log messages. See `LogRecord `_ attributes. -private,,, --- keys,,, --- -- entsoe_api,--,,Optionally specify the ENTSO-E API key. See the guidelines to get `ENTSO-E API key `_ -remote,,, --- ssh,--,,Optionally specify the SSH of a remote cluster to be synchronized. --- path,--,,Optionally specify the file path within the remote cluster to be synchronized. +,Unit,Values,Description +version,--,0.x.x,Version of PyPSA-Eur. Descriptive only. +tutorial,bool,"{true, false}",Switch to retrieve the tutorial data set instead of the full data set. +logging,,, +-- level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only" +-- format,--,,Custom format for log messages. See `LogRecord `_ attributes. +private,,, +-- keys,,, +-- -- entsoe_api,--,,Optionally specify the ENTSO-E API key. See the guidelines to get `ENTSO-E API key `_ +remote,,, +-- ssh,--,,Optionally specify the SSH of a remote cluster to be synchronized. +-- path,--,,Optionally specify the file path within the remote cluster to be synchronized. >>>>>>> master diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 3c859b3c..1cc54ec0 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -30,16 +30,21 @@ rule build_clustered_population_layouts: pop_layout_total=RESOURCES + "pop_layout_total{weather_year}.nc", pop_layout_urban=RESOURCES + "pop_layout_urban{weather_year}.nc", pop_layout_rural=RESOURCES + "pop_layout_rural{weather_year}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", + regions_onshore=RESOURCES + + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", output: - clustered_pop_layout=RESOURCES + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", + clustered_pop_layout=RESOURCES + + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", log: LOGS + "build_clustered_population_layouts{weather_year}_{simpl}_{clusters}.log", resources: mem_mb=10000, benchmark: - BENCHMARKS + "build_clustered_population_layouts/{weather_year}_s{simpl}_{clusters}" + ( + BENCHMARKS + + "build_clustered_population_layouts/{weather_year}_s{simpl}_{clusters}" + ) conda: "../envs/environment.yaml" script: @@ -51,7 +56,8 @@ rule build_simplified_population_layouts: pop_layout_total=RESOURCES + "pop_layout_total{weather_year}.nc", pop_layout_urban=RESOURCES + "pop_layout_urban{weather_year}.nc", pop_layout_rural=RESOURCES + "pop_layout_rural{weather_year}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}.geojson", + regions_onshore=RESOURCES + + "regions_onshore_elec{weather_year}_s{simpl}.geojson", cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", output: clustered_pop_layout=RESOURCES + "pop_layout_elec{weather_year}_s{simpl}.csv", @@ -117,7 +123,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: regions_offshore=RESOURCES + "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson", output: - clustered_gas_network=RESOURCES + "gas_network_elec{weather_year}_s{simpl}_{clusters}.csv", + clustered_gas_network=RESOURCES + + "gas_network_elec{weather_year}_s{simpl}_{clusters}.csv", resources: mem_mb=4000, log: @@ -144,10 +151,12 @@ rule build_heat_demands: snapshots=config["snapshots"], input: pop_layout=RESOURCES + "pop_layout{weather_year}_{scope}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", + regions_onshore=RESOURCES + + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", output: - heat_demand=RESOURCES + "heat_demand_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc", + heat_demand=RESOURCES + + "heat_demand_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc", resources: mem_mb=20000, threads: 8 @@ -166,18 +175,25 @@ rule build_temperature_profiles: snapshots=config["snapshots"], input: pop_layout=RESOURCES + "pop_layout{weather_year}_{scope}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", + regions_onshore=RESOURCES + + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", output: - temp_soil=RESOURCES + "temp_soil_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc", - temp_air=RESOURCES + "temp_air_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_soil=RESOURCES + + "temp_soil_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_air=RESOURCES + + "temp_air_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc", resources: mem_mb=20000, threads: 8 log: - LOGS + "build_temperature_profiles_{scope}_{weather_year}_{simpl}_{clusters}.log", + LOGS + + "build_temperature_profiles_{scope}_{weather_year}_{simpl}_{clusters}.log", benchmark: - BENCHMARKS + "build_temperature_profiles/{scope}_{weather_year}_s{simpl}_{clusters}" + ( + BENCHMARKS + + "build_temperature_profiles/{scope}_{weather_year}_s{simpl}_{clusters}" + ) conda: "../envs/environment.yaml" script: @@ -188,19 +204,31 @@ rule build_cop_profiles: params: heat_pump_sink_T=config["sector"]["heat_pump_sink_T"], input: - temp_soil_total=RESOURCES + "temp_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc", - temp_soil_rural=RESOURCES + "temp_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc", - temp_soil_urban=RESOURCES + "temp_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc", - temp_air_total=RESOURCES + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", - temp_air_rural=RESOURCES + "temp_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc", - temp_air_urban=RESOURCES + "temp_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_soil_total=RESOURCES + + "temp_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_soil_rural=RESOURCES + + "temp_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_soil_urban=RESOURCES + + "temp_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_air_total=RESOURCES + + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_air_rural=RESOURCES + + "temp_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_air_urban=RESOURCES + + "temp_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc", output: - cop_soil_total=RESOURCES + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc", - cop_soil_rural=RESOURCES + "cop_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc", - cop_soil_urban=RESOURCES + "cop_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc", - cop_air_total=RESOURCES + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", - cop_air_rural=RESOURCES + "cop_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc", - cop_air_urban=RESOURCES + "cop_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_soil_total=RESOURCES + + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_soil_rural=RESOURCES + + "cop_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_soil_urban=RESOURCES + + "cop_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_air_total=RESOURCES + + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_air_rural=RESOURCES + + "cop_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_air_urban=RESOURCES + + "cop_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc", resources: mem_mb=20000, log: @@ -219,17 +247,23 @@ rule build_solar_thermal_profiles: solar_thermal=config["solar_thermal"], input: pop_layout=RESOURCES + "pop_layout{weather_year}_{scope}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", + regions_onshore=RESOURCES + + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", output: - solar_thermal=RESOURCES + "solar_thermal_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc", + solar_thermal=RESOURCES + + "solar_thermal_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc", resources: mem_mb=20000, threads: 16 log: - LOGS + "build_solar_thermal_profiles_{scope}_{weather_year}_s{simpl}_{clusters}.log", + LOGS + + "build_solar_thermal_profiles_{scope}_{weather_year}_s{simpl}_{clusters}.log", benchmark: - BENCHMARKS + "build_solar_thermal_profiles/{scope}_{weather_year}_s{simpl}_{clusters}" + ( + BENCHMARKS + + "build_solar_thermal_profiles/{scope}_{weather_year}_s{simpl}_{clusters}" + ) conda: "../envs/environment.yaml" script: @@ -269,13 +303,14 @@ rule build_heat_totals: hdd="data/era5-annual-HDD-per-country.csv", energy_totals=RESOURCES + "energy_totals.csv", output: - heat_totals=RESOURCES + "heat_totals.csv" + heat_totals=RESOURCES + "heat_totals.csv", threads: 1 - resources: mem_mb=2000 + resources: + mem_mb=2000, log: LOGS + "build_heat_totals.log", benchmark: - BENCHMARKS + "build_heat_totals", + BENCHMARKS + "build_heat_totals" conda: "../envs/environment.yaml" script: @@ -291,7 +326,8 @@ rule build_biomass_potentials: keep_local=True, ), nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 - regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", + regions_onshore=RESOURCES + + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"), swiss_cantons=ancient("data/bundle/ch_cantons.csv"), swiss_population=ancient("data/bundle/je-e-21.03.02.xls"), @@ -305,9 +341,13 @@ rule build_biomass_potentials: resources: mem_mb=1000, log: - LOGS + "build_biomass_potentials{weather_year}_s{simpl}_{clusters}_{planning_horizons}.log", + LOGS + + "build_biomass_potentials{weather_year}_s{simpl}_{clusters}_{planning_horizons}.log", benchmark: - BENCHMARKS + "build_biomass_potentials{weather_year}_s{simpl}_{clusters}_{planning_horizons}" + ( + BENCHMARKS + + "build_biomass_potentials{weather_year}_s{simpl}_{clusters}_{planning_horizons}" + ) conda: "../envs/environment.yaml" script: @@ -367,9 +407,13 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: resources: mem_mb=4000, log: - LOGS + "build_sequestration_potentials{weather_year}_s{simpl}_{clusters}.log", + LOGS + + "build_sequestration_potentials{weather_year}_s{simpl}_{clusters}.log", benchmark: - BENCHMARKS + "build_sequestration_potentials{weather_year}_s{simpl}_{clusters}" + ( + BENCHMARKS + + "build_sequestration_potentials{weather_year}_s{simpl}_{clusters}" + ) conda: "../envs/environment.yaml" script: @@ -386,10 +430,13 @@ if not config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_salt_cavern_potentials: input: salt_caverns="data/bundle-sector/h2_salt_caverns_GWh_per_sqkm.geojson", - regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", - regions_offshore=RESOURCES + "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson", + regions_onshore=RESOURCES + + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", + regions_offshore=RESOURCES + + "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson", output: - h2_cavern_potential=RESOURCES + "salt_cavern_potentials{weather_year}_s{simpl}_{clusters}.csv", + h2_cavern_potential=RESOURCES + + "salt_cavern_potentials{weather_year}_s{simpl}_{clusters}.csv", threads: 1 resources: mem_mb=2000, @@ -500,8 +547,10 @@ rule build_industrial_distribution_key: hotmaps_locate_missing=config["industry"].get("hotmaps_locate_missing", False), countries=config["countries"], input: - regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", - clustered_pop_layout=RESOURCES + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", + regions_onshore=RESOURCES + + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", + clustered_pop_layout=RESOURCES + + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", hotmaps_industrial_database="data/bundle-sector/Industrial_Database.csv", output: industrial_distribution_key=RESOURCES @@ -512,7 +561,10 @@ rule build_industrial_distribution_key: log: LOGS + "build_industrial_distribution_key{weather_year}_s{simpl}_{clusters}.log", benchmark: - BENCHMARKS + "build_industrial_distribution_key/{weather_year}_s{simpl}_{clusters}" + ( + BENCHMARKS + + "build_industrial_distribution_key/{weather_year}_s{simpl}_{clusters}" + ) conda: "../envs/environment.yaml" script: @@ -610,9 +662,13 @@ rule build_industrial_energy_demand_per_node_today: resources: mem_mb=1000, log: - LOGS + "build_industrial_energy_demand_per_node_today{weather_year}_s{simpl}_{clusters}.log", + LOGS + + "build_industrial_energy_demand_per_node_today{weather_year}_s{simpl}_{clusters}.log", benchmark: - BENCHMARKS + "build_industrial_energy_demand_per_node_today/{weather_year}_s{simpl}_{clusters}" + ( + BENCHMARKS + + "build_industrial_energy_demand_per_node_today/{weather_year}_s{simpl}_{clusters}" + ) conda: "../envs/environment.yaml" script: @@ -628,17 +684,21 @@ if config["sector"]["retrofitting"]["retro_endogen"]: input: building_stock="data/retro/data_building_stock.csv", data_tabula="data/retro/tabula-calculator-calcsetbuilding.csv", - air_temperature=RESOURCES + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", + air_temperature=RESOURCES + + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", u_values_PL="data/retro/u_values_poland.csv", tax_w="data/retro/electricity_taxes_eu.csv", construction_index="data/retro/comparative_level_investment.csv", floor_area_missing="data/retro/floor_area_missing.csv", - clustered_pop_layout=RESOURCES + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", + clustered_pop_layout=RESOURCES + + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", cost_germany="data/retro/retro_cost_germany.csv", window_assumptions="data/retro/window_assumptions.csv", output: - retro_cost=RESOURCES + "retro_cost_elec{weather_year}_s{simpl}_{clusters}.csv", - floor_area=RESOURCES + "floor_area_elec{weather_year}_s{simpl}_{clusters}.csv", + retro_cost=RESOURCES + + "retro_cost_elec{weather_year}_s{simpl}_{clusters}.csv", + floor_area=RESOURCES + + "floor_area_elec{weather_year}_s{simpl}_{clusters}.csv", resources: mem_mb=1000, log: @@ -661,14 +721,16 @@ if not config["sector"]["retrofitting"]["retro_endogen"]: rule build_population_weighted_energy_totals: input: energy_totals=RESOURCES + "{kind}}_totals.csv", - clustered_pop_layout=RESOURCES + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", + clustered_pop_layout=RESOURCES + + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", output: RESOURCES + "pop_weighted_{kind}_totals{weather_year}_s{simpl}_{clusters}.csv", threads: 1 resources: mem_mb=2000, log: - LOGS + "build_population_weighted_{kind}_totals{weather_year}_s{simpl}_{clusters}.log", + LOGS + + "build_population_weighted_{kind}_totals{weather_year}_s{simpl}_{clusters}.log", conda: "../envs/environment.yaml" script: @@ -679,7 +741,8 @@ rule build_shipping_demand: input: ports="data/attributed_ports.json", scope=RESOURCES + "europe_shape.geojson", - regions=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", + regions=RESOURCES + + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", demand=RESOURCES + "energy_totals.csv", output: RESOURCES + "shipping_demand{weather_year}_s{simpl}_{clusters}.csv", @@ -699,16 +762,20 @@ rule build_transport_demand: snapshots=config["snapshots"], sector=config["sector"], input: - clustered_pop_layout=RESOURCES + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", + clustered_pop_layout=RESOURCES + + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", pop_weighted_energy_totals=RESOURCES + "pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv", transport_data=RESOURCES + "transport_data.csv", traffic_data_KFZ="data/bundle-sector/emobility/KFZ__count", traffic_data_Pkw="data/bundle-sector/emobility/Pkw__count", - temp_air_total=RESOURCES + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_air_total=RESOURCES + + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", output: - transport_demand=RESOURCES + "transport_demand{weather_year}_s{simpl}_{clusters}.csv", - transport_data=RESOURCES + "transport_data{weather_year}_s{simpl}_{clusters}.csv", + transport_demand=RESOURCES + + "transport_demand{weather_year}_s{simpl}_{clusters}.csv", + transport_data=RESOURCES + + "transport_data{weather_year}_s{simpl}_{clusters}.csv", avail_profile=RESOURCES + "avail_profile{weather_year}_s{simpl}_{clusters}.csv", dsm_profile=RESOURCES + "dsm_profile{weather_year}_s{simpl}_{clusters}.csv", threads: 1 @@ -742,16 +809,20 @@ rule prepare_sector_network: **build_biomass_transport_costs_output, **gas_infrastructure, **build_sequestration_potentials_output, - network=RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network=RESOURCES + + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", energy_totals_name=RESOURCES + "energy_totals.csv", eurostat=input_eurostat, pop_weighted_energy_totals=RESOURCES + "pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv", pop_weighted_heat_totals=RESOURCES + "pop_weighted_heat_totals{weather_year}_s{simpl}_{clusters}.csv", - shipping_demand=RESOURCES + "shipping_demand{weather_year}_s{simpl}_{clusters}.csv", - transport_demand=RESOURCES + "transport_demand{weather_year}_s{simpl}_{clusters}.csv", - transport_data=RESOURCES + "transport_data{weather_year}_s{simpl}_{clusters}.csv", + shipping_demand=RESOURCES + + "shipping_demand{weather_year}_s{simpl}_{clusters}.csv", + transport_demand=RESOURCES + + "transport_demand{weather_year}_s{simpl}_{clusters}.csv", + transport_data=RESOURCES + + "transport_data{weather_year}_s{simpl}_{clusters}.csv", avail_profile=RESOURCES + "avail_profile{weather_year}_s{simpl}_{clusters}.csv", dsm_profile=RESOURCES + "dsm_profile{weather_year}_s{simpl}_{clusters}.csv", co2_totals_name=RESOURCES + "co2_totals.csv", @@ -768,28 +839,45 @@ rule prepare_sector_network: else "data/costs_{planning_horizons}.csv", profile_offwind_ac=RESOURCES + "profile{weather_year}_offwind-ac.nc", profile_offwind_dc=RESOURCES + "profile{weather_year}_offwind-dc.nc", - h2_cavern=RESOURCES + "salt_cavern_potentials{weather_year}_s{simpl}_{clusters}.csv", + h2_cavern=RESOURCES + + "salt_cavern_potentials{weather_year}_s{simpl}_{clusters}.csv", busmap_s=RESOURCES + "busmap_elec{weather_year}_s{simpl}.csv", busmap=RESOURCES + "busmap_elec{weather_year}_s{simpl}_{clusters}.csv", - clustered_pop_layout=RESOURCES + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", + clustered_pop_layout=RESOURCES + + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", simplified_pop_layout=RESOURCES + "pop_layout_elec{weather_year}_s{simpl}.csv", industrial_demand=RESOURCES + "industrial_energy_demand_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv", - heat_demand_urban=RESOURCES + "heat_demand_urban_elec{weather_year}_s{simpl}_{clusters}.nc", - heat_demand_rural=RESOURCES + "heat_demand_rural_elec{weather_year}_s{simpl}_{clusters}.nc", - heat_demand_total=RESOURCES + "heat_demand_total_elec{weather_year}_s{simpl}_{clusters}.nc", - temp_soil_total=RESOURCES + "temp_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc", - temp_soil_rural=RESOURCES + "temp_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc", - temp_soil_urban=RESOURCES + "temp_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc", - temp_air_total=RESOURCES + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", - temp_air_rural=RESOURCES + "temp_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc", - temp_air_urban=RESOURCES + "temp_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc", - cop_soil_total=RESOURCES + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc", - cop_soil_rural=RESOURCES + "cop_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc", - cop_soil_urban=RESOURCES + "cop_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc", - cop_air_total=RESOURCES + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", - cop_air_rural=RESOURCES + "cop_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc", - cop_air_urban=RESOURCES + "cop_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc", + heat_demand_urban=RESOURCES + + "heat_demand_urban_elec{weather_year}_s{simpl}_{clusters}.nc", + heat_demand_rural=RESOURCES + + "heat_demand_rural_elec{weather_year}_s{simpl}_{clusters}.nc", + heat_demand_total=RESOURCES + + "heat_demand_total_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_soil_total=RESOURCES + + "temp_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_soil_rural=RESOURCES + + "temp_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_soil_urban=RESOURCES + + "temp_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_air_total=RESOURCES + + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_air_rural=RESOURCES + + "temp_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc", + temp_air_urban=RESOURCES + + "temp_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_soil_total=RESOURCES + + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_soil_rural=RESOURCES + + "cop_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_soil_urban=RESOURCES + + "cop_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_air_total=RESOURCES + + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_air_rural=RESOURCES + + "cop_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc", + cop_air_urban=RESOURCES + + "cop_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc", solar_thermal_total=RESOURCES + "solar_thermal_total_elec{weather_year}_s{simpl}_{clusters}.nc" if config["sector"]["solar_thermal"] From 90fb0b13da3437cd3578eb8dae0b1ee9f64cbdc5 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Thu, 29 Feb 2024 11:38:21 +0100 Subject: [PATCH 081/173] Various bugfixes to make multiyear branch run --- rules/build_electricity.smk | 24 ++++++++++++------- rules/build_sector.smk | 13 ++++++---- rules/postprocess.smk | 8 +++---- rules/retrieve.smk | 3 --- scripts/base_network.py | 2 +- scripts/build_energy_totals.py | 10 ++++---- scripts/build_heat_totals.py | 8 +++++++ scripts/build_hydro_profile.py | 2 ++ ...build_population_weighted_energy_totals.py | 2 +- scripts/build_renewable_profiles.py | 10 +++++++- scripts/build_shipping_demand.py | 3 ++- 11 files changed, 55 insertions(+), 30 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 0c949193..99e671fd 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -202,12 +202,26 @@ if config["enable"].get("build_cutout", False): "../scripts/build_cutout.py" +# TODO: consider if it's fine to just use the default cutout instead, i.e.: +# cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", +def cutouts_for_extent_input(wildcards): + # We only need these cutouts in order to determine geographic + # boundaries, so just pick the first weather year available + weather_year = config["scenario"]["weather_year"][0] + cutouts = [] + for k in config["electricity"]["renewable_carriers"]: + cutout = config["renewable"][k]["cutout"] + cutout = cutout.replace("{weather_year}", weather_year) + cutouts.append(f"cutouts/{CDIR}{cutout}.nc") + return cutouts + + if config["enable"].get("build_natura_raster", False): rule build_natura_raster: input: natura=ancient("data/bundle/natura/Natura2000_end2015.shp"), - cutouts=expand("cutouts/" + CDIR + "{cutouts}.nc", **config["atlite"]), + cutouts=cutouts_for_extent_input, output: RESOURCES + "natura.tiff", resources: @@ -223,13 +237,7 @@ if config["enable"].get("build_natura_raster", False): rule build_ship_raster: input: ship_density="data/shipdensity_global.zip", - cutouts=expand( - "cutouts/" + CDIR + "{cutout}.nc", - cutout=[ - config["renewable"][k]["cutout"] - for k in config["electricity"]["renewable_carriers"] - ], - ), + cutouts=cutouts_for_extent_input, output: RESOURCES + "shipdensity_raster.tif", log: diff --git a/rules/build_sector.smk b/rules/build_sector.smk index c68e5a5a..ec5b09f9 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -150,7 +150,7 @@ rule build_heat_demands: params: snapshots=config["snapshots"], input: - pop_layout=RESOURCES + "pop_layout{weather_year}_{scope}.nc", + pop_layout=RESOURCES + "pop_layout_{scope}{weather_year}.nc", regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", @@ -174,7 +174,7 @@ rule build_temperature_profiles: params: snapshots=config["snapshots"], input: - pop_layout=RESOURCES + "pop_layout{weather_year}_{scope}.nc", + pop_layout=RESOURCES + "pop_layout_{scope}{weather_year}.nc", regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", @@ -246,7 +246,7 @@ rule build_solar_thermal_profiles: snapshots=config["snapshots"], solar_thermal=config["solar_thermal"], input: - pop_layout=RESOURCES + "pop_layout{weather_year}_{scope}.nc", + pop_layout=RESOURCES + "pop_layout_{scope}{weather_year}.nc", regions_onshore=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", @@ -684,7 +684,8 @@ if config["sector"]["retrofitting"]["retro_endogen"]: input: building_stock="data/retro/data_building_stock.csv", data_tabula="data/bundle-sector/retro/tabula-calculator-calcsetbuilding.csv", - air_temperature=RESOURCES + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", + air_temperature=RESOURCES + + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", u_values_PL="data/retro/u_values_poland.csv", tax_w="data/retro/electricity_taxes_eu.csv", construction_index="data/retro/comparative_level_investment.csv", @@ -719,7 +720,7 @@ if not config["sector"]["retrofitting"]["retro_endogen"]: rule build_population_weighted_energy_totals: input: - energy_totals=RESOURCES + "{kind}}_totals.csv", + energy_totals=RESOURCES + "{kind}_totals.csv", clustered_pop_layout=RESOURCES + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", output: @@ -743,6 +744,8 @@ rule build_shipping_demand: regions=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson", demand=RESOURCES + "energy_totals.csv", + params: + energy_totals_year=config["energy"]["energy_totals_year"], output: RESOURCES + "shipping_demand{weather_year}_s{simpl}_{clusters}.csv", threads: 1 diff --git a/rules/postprocess.smk b/rules/postprocess.smk index ec6c21f7..aaecbca1 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -28,7 +28,7 @@ if config["foresight"] != "perfect": benchmark: ( BENCHMARKS - + "plot_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + + "plot_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -181,14 +181,14 @@ rule plot_elec_statistics: plotting=config["plotting"], barplots=STATISTICS_BARPLOTS, input: - network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network=RESULTS + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: **{ f"{plot}_bar": RESULTS - + f"figures/statistics_{plot}_bar_elec_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" + + f"figures/statistics_{plot}_bar_elec{{weather_year}}_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" for plot in STATISTICS_BARPLOTS }, barplots_touch=RESULTS - + "figures/.statistics_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", + + "figures/.statistics_plots_elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}", script: "../scripts/plot_statistics.py" diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 63ba2e23..36c2ab91 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -144,9 +144,6 @@ if config["enable"]["retrieve"] and config["enable"].get( protected( directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition") ), - protected( - directory("data/bundle-sector/eurostat-energy_balances-june_2021_edition") - ), protected( directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition") ), diff --git a/scripts/base_network.py b/scripts/base_network.py index 663a0a15..98a7b7f4 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -558,7 +558,7 @@ def _set_countries_and_substations(n, config, country_shapes, offshore_shapes): ~buses["under_construction"] ) - c_nan_b = buses.country == "na" + c_nan_b = buses.country.fillna("na") == "na" if c_nan_b.sum() > 0: c_tag = _get_country(buses.loc[c_nan_b]) c_tag.loc[~c_tag.isin(countries)] = np.nan diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 1da5fa21..158e0d01 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -356,7 +356,7 @@ def build_idees(countries, year=None): func = partial(idees_per_country, year=year, base_dir=snakemake.input.idees) tqdm_kwargs = dict( ascii=False, - unit="" country", + unit=" country", total=len(countries), desc="Build from IDEES database", disable=disable_progress @@ -382,9 +382,6 @@ def build_idees(countries, year=None): total_heat = df[subset].sum(axis=1) df["district heat share"] = district_heat.div(total_heat) - if year: - df = df.xs(int(year), level='year') - df.columns.name = 'item' return df @@ -741,12 +738,13 @@ if __name__ == "__main__": countries = snakemake.params.countries idees_countries = pd.Index(countries).intersection(eu28) + countries_without_ch = pd.Index(countries).difference(["CH"]) data_year = params["energy_totals_year"] report_year = snakemake.params.energy["eurostat_report_year"] input_eurostat = snakemake.input.eurostat - eurostat = build_eurostat(input_eurostat, countries, report_year, data_year) - swiss = build_swiss(data_year) + eurostat = build_eurostat(countries_without_ch) + swiss = build_swiss() idees = build_idees(idees_countries, data_year) energy = build_energy_totals(countries, eurostat, swiss, idees) diff --git a/scripts/build_heat_totals.py b/scripts/build_heat_totals.py index 80f0fb30..93e8f9c7 100644 --- a/scripts/build_heat_totals.py +++ b/scripts/build_heat_totals.py @@ -29,6 +29,14 @@ def approximate_heat_demand(energy_totals, hdd): Y = demand[c].dropna() X = hdd.loc[Y.index, c] + # Sometimes (looking at you, Switzerland) we only have + # _one_ year of heating data to base the prediction on. In + # this case we add a point at 0, 0 to make a "polynomial" + # fit work. + if len(X) == len(Y) == 1: + X.loc[-1] = 0 + Y.loc[-1] = 0 + to_predict = hdd.index.difference(Y.index) X_pred = hdd.loc[to_predict, c] diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 055523c8..baa05ab9 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -184,6 +184,8 @@ if __name__ == "__main__": fn = snakemake.input.eia_hydro_generation eia_stats = get_eia_annual_hydro_generation(fn, countries) + config_hydro = snakemake.config["renewable"]["hydro"] + if config_hydro.get("eia_correct_by_capacity"): fn = snakemake.input.eia_hydro_capacity correct_eia_stats_by_capacity(eia_stats, fn, countries) diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index bc5f2d19..fd232c89 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -26,7 +26,7 @@ if __name__ == "__main__": pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) - totals = pd.read_csv(snakemake.input.totals, index_col=[0, 1]) + totals = pd.read_csv(snakemake.input.energy_totals, index_col=[0, 1]) totals = totals.xs(data_year, level="year") nodal_totals = totals.loc[pop_layout.ct].fillna(0.0) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 829c5347..48ba64a2 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -216,7 +216,15 @@ if __name__ == "__main__": else: client = None - sns = pd.date_range(freq="h", **snakemake.config["snapshots"]) + weather_year = snakemake.wildcards.weather_year + if weather_year: + snapshots = dict( + start=weather_year, end=str(int(weather_year) + 1), inclusive="left" + ) + else: + snapshots = snakemake.config["snapshots"] + sns = pd.date_range(freq="h", **snapshots) + cutout = atlite.Cutout(snakemake.input.cutout).sel(time=sns) regions = gpd.read_file(snakemake.input.regions) assert not regions.empty, ( diff --git a/scripts/build_shipping_demand.py b/scripts/build_shipping_demand.py index 8000c66c..f95f441a 100644 --- a/scripts/build_shipping_demand.py +++ b/scripts/build_shipping_demand.py @@ -24,9 +24,10 @@ if __name__ == "__main__": scope = gpd.read_file(snakemake.input.scope).geometry[0] regions = gpd.read_file(snakemake.input.regions).set_index("name") - demand = pd.read_csv(snakemake.input.demand, index_col=0)[ + demand = pd.read_csv(snakemake.input.demand, index_col=[0, 1])[ "total international navigation" ] + demand = demand.xs(snakemake.params.energy_totals_year, level=1) # read port data into GeoDataFrame with open(snakemake.input.ports, "r", encoding="latin_1") as f: From f6f98c102c134bccce33dca004a203fa2534dcc5 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 1 Mar 2024 11:45:39 +0100 Subject: [PATCH 082/173] Use default cutout for ship and nature raster extents This avoids the need to either collect all cutouts for all weather years or arbitrarily pick all cutouts for a certain weather year. --- rules/build_electricity.smk | 24 ++++++++---------------- scripts/build_natura_raster.py | 5 ++--- scripts/build_ship_raster.py | 5 ++--- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 78904573..bcc69e2d 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -205,26 +205,15 @@ if config["enable"].get("build_cutout", False): "../scripts/build_cutout.py" -# TODO: consider if it's fine to just use the default cutout instead, i.e.: -# cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", -def cutouts_for_extent_input(wildcards): - # We only need these cutouts in order to determine geographic - # boundaries, so just pick the first weather year available - weather_year = config_provider("scenario", "weather_year")(wildcards)[0] - cutouts = [] - for k in config_provider("electricity", "renewable_carriers")(wildcards): - cutout = config_provider("renewable", k, "cutout")(wildcards) - cutout = cutout.replace("{weather_year}", weather_year) - cutouts.append(f"cutouts/{CDIR}{cutout}.nc") - return cutouts - - if config["enable"].get("build_natura_raster", False): rule build_natura_raster: input: natura=ancient("data/bundle/natura/Natura2000_end2015.shp"), - cutouts=cutouts_for_extent_input, + cutout=lambda w: "cutouts/" + + CDIR + + config_provider("atlite", "default_cutout")(w) + + ".nc", output: resources("natura.tiff"), resources: @@ -240,7 +229,10 @@ if config["enable"].get("build_natura_raster", False): rule build_ship_raster: input: ship_density="data/shipdensity_global.zip", - cutouts=cutouts_for_extent_input, + cutout=lambda w: "cutouts/" + + CDIR + + config_provider("atlite", "default_cutout")(w) + + ".nc", output: resources("shipdensity_raster.tif"), log: diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py index 2e2fcbbf..35fb0dbd 100644 --- a/scripts/build_natura_raster.py +++ b/scripts/build_natura_raster.py @@ -94,9 +94,8 @@ if __name__ == "__main__": configure_logging(snakemake) set_scenario_config(snakemake) - cutouts = snakemake.input.cutouts - xs, Xs, ys, Ys = zip(*(determine_cutout_xXyY(cutout) for cutout in cutouts)) - bounds = transform_bounds(4326, 3035, min(xs), min(ys), max(Xs), max(Ys)) + x, X, y, Y = determine_cutout_xXyY(snakemake.input.cutout) + bounds = transform_bounds(4326, 3035, x, y, X, Y) transform, out_shape = get_transform_and_shape(bounds, res=100) # adjusted boundaries diff --git a/scripts/build_ship_raster.py b/scripts/build_ship_raster.py index c8285180..47d725d8 100644 --- a/scripts/build_ship_raster.py +++ b/scripts/build_ship_raster.py @@ -59,8 +59,7 @@ if __name__ == "__main__": configure_logging(snakemake) set_scenario_config(snakemake) - cutouts = snakemake.input.cutouts - xs, Xs, ys, Ys = zip(*(determine_cutout_xXyY(cutout) for cutout in cutouts)) + x, X, y, Y = determine_cutout_xXyY(snakemake.input.cutout) with zipfile.ZipFile(snakemake.input.ship_density) as zip_f: resources = Path(snakemake.output[0]).parent @@ -68,7 +67,7 @@ if __name__ == "__main__": zip_f.extract(fn, resources) with rioxarray.open_rasterio(resources / fn) as ship_density: ship_density = ship_density.drop_vars(["band"]).sel( - x=slice(min(xs), max(Xs)), y=slice(max(Ys), min(ys)) + x=slice(x, X), y=slice(Y, y) ) ship_density.rio.to_raster(snakemake.output[0]) From 0debf66169fbc85b3c642fdbca33b19a9e85eee0 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 1 Mar 2024 11:50:35 +0100 Subject: [PATCH 083/173] Fix summary plot output --- rules/postprocess.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index ab08912d..8b968681 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -175,7 +175,7 @@ rule make_summary: ) ), ac_plot=expand( - resources("maps/power-network-s{simpl}-{clusters}.pdf"), + resources("maps/power-network{weather_year}-s{simpl}-{clusters}.pdf"), **config["scenario"], allow_missing=True, ), From 3c31caf9937bcd8f9fb43078bdc8ab70702febf0 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 1 Mar 2024 11:59:04 +0100 Subject: [PATCH 084/173] Fix typos in rule inputs --- rules/build_sector.smk | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 8e6587dd..f9edad14 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -54,9 +54,9 @@ rule build_clustered_population_layouts: rule build_simplified_population_layouts: input: - pop_layout_total=resources("pop_layout_total{weather_year}"), - pop_layout_urban=resources("pop_layout_urban{weather_year}"), - pop_layout_rural=resources("pop_layout_rural{weather_year}"), + pop_layout_total=resources("pop_layout_total{weather_year}.nc"), + pop_layout_urban=resources("pop_layout_urban{weather_year}.nc"), + pop_layout_rural=resources("pop_layout_rural{weather_year}.nc"), regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}.geojson"), cutout=lambda w: "cutouts/" + CDIR @@ -398,8 +398,8 @@ rule build_sequestration_potentials: rule build_salt_cavern_potentials: input: salt_caverns="data/bundle-sector/h2_salt_caverns_GWh_per_sqkm.geojson", - regions_onshore=resources("regions_onshore{weather_year}_elec_s{simpl}_{clusters}.geojson"), - regions_offshore=resources("regions_offshore{weather_year}_elec_s{simpl}_{clusters}.geojson"), + regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_offshore=resources("regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), output: h2_cavern_potential=resources("salt_cavern_potentials{weather_year}_s{simpl}_{clusters}.csv"), threads: 1 From c2e7c125c7ae25a8213f5a6af8ad7cde7af043f2 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 1 Mar 2024 13:24:02 +0100 Subject: [PATCH 085/173] Automatic formatting of snakemake rules --- rules/build_electricity.smk | 43 +++-- rules/build_sector.smk | 310 ++++++++++++++++++++++++++---------- rules/collect.smk | 7 +- rules/postprocess.smk | 25 ++- rules/retrieve.smk | 28 +++- rules/solve_electricity.smk | 9 +- rules/solve_myopic.smk | 20 ++- 7 files changed, 325 insertions(+), 117 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index bcc69e2d..3f353dd9 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -19,6 +19,7 @@ if config["enable"].get("prepare_links_p_nom", False): if config["enable"].get("retrieve_opsd_load_data", True): + rule build_electricity_demand: params: snapshots=config_provider("snapshots"), @@ -37,7 +38,7 @@ if config["enable"].get("retrieve_opsd_load_data", True): script: "../scripts/build_electricity_demand.py" - + if config["enable"].get("retrieve_artificial_load_data", False): rule build_artificial_load_data: @@ -390,7 +391,7 @@ rule build_hydro_profile: + ".nc", output: profile=resources("profile{weather_year}_hydro.nc"), - eia_hydro=resources("eia_hydro_stats{weather_year}.csv") + eia_hydro=resources("eia_hydro_stats{weather_year}.csv"), log: logs("build_hydro_profile{weather_year}.log"), resources: @@ -427,7 +428,7 @@ rule build_line_rating: def input_profile_tech(w): return { - f"profile_{tech}": resources(f"profile{{weather_year}}_{tech}.nc") + f"profile_{tech}": resources(f"profile{weather_year}_{tech}.nc") for tech in config_provider("electricity", "renewable_carriers")(w) } @@ -462,7 +463,7 @@ rule add_electricity: else resources("networks/base.nc") ), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs' , 'year') (w)}.csv" ), regions=resources("regions_onshore.geojson"), powerplants=resources("powerplants.csv"), @@ -507,14 +508,16 @@ rule simplify_network: input: network=resources("networks/elec{weather_year}.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs' , 'year') (w)}.csv" ), regions_onshore=resources("regions_onshore.geojson"), regions_offshore=resources("regions_offshore.geojson"), output: network=resources("networks/elec{weather_year}_s{simpl}.nc"), regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}.geojson"), - regions_offshore=resources("regions_offshore_elec{weather_year}_s{simpl}.geojson"), + regions_offshore=resources( + "regions_offshore_elec{weather_year}_s{simpl}.geojson" + ), busmap=resources("busmap_elec{weather_year}_s{simpl}.csv"), connection_costs=resources("connection_costs{weather_year}_s{simpl}.csv"), log: @@ -548,7 +551,9 @@ rule cluster_network: input: network=resources("networks/elec{weather_year}_s{simpl}.nc"), regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}.geojson"), - regions_offshore=resources("regions_offshore_elec{weather_year}_s{simpl}.geojson"), + regions_offshore=resources( + "regions_offshore_elec{weather_year}_s{simpl}.geojson" + ), busmap=ancient(resources("busmap_elec{weather_year}_s{simpl}.csv")), custom_busmap=lambda w: ( "data/custom_busmap_elec_s{simpl}_{clusters}.csv" @@ -556,12 +561,16 @@ rule cluster_network: else [] ), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs' , 'year') (w)}.csv" ), output: network=resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"), - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), - regions_offshore=resources("regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_onshore=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), + regions_offshore=resources( + "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), busmap=resources("busmap_elec{weather_year}_s{simpl}_{clusters}.csv"), linemap=resources("linemap_elec{weather_year}_s{simpl}_{clusters}.csv"), log: @@ -585,7 +594,7 @@ rule add_extra_components: input: network=resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs' , 'year') (w)}.csv" ), output: resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc"), @@ -619,15 +628,21 @@ rule prepare_network: input: resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs' , 'year') (w)}.csv" ), co2_price=lambda w: resources("co2_price.csv") if "Ept" in w.opts else [], output: resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), log: - logs("prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.log"), + logs( + "prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" + ), benchmark: - (benchmarks("prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}")) + ( + benchmarks( + "prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" + ) + ) threads: 1 resources: mem_mb=4000, diff --git a/rules/build_sector.smk b/rules/build_sector.smk index f9edad14..9fb6876b 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -33,19 +33,25 @@ rule build_clustered_population_layouts: pop_layout_total=resources("pop_layout_total{weather_year}.nc"), pop_layout_urban=resources("pop_layout_urban{weather_year}.nc"), pop_layout_rural=resources("pop_layout_rural{weather_year}.nc"), - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_onshore=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), cutout=lambda w: "cutouts/" + CDIR + config_provider("atlite", "default_cutout")(w) + ".nc", output: - clustered_pop_layout=resources("pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv"), + clustered_pop_layout=resources( + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + ), log: logs("build_clustered_population_layouts{weather_year}_{simpl}_{clusters}.log"), resources: mem_mb=10000, benchmark: - benchmarks("build_clustered_population_layouts/{weather_year}s{simpl}_{clusters}") + benchmarks( + "build_clustered_population_layouts/{weather_year}s{simpl}_{clusters}" + ) conda: "../envs/environment.yaml" script: @@ -99,10 +105,16 @@ rule build_gas_input_locations: ), entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson", storage="data/gas_network/scigrid-gas/data/IGGIELGN_Storages.geojson", - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), - regions_offshore=resources("regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_onshore=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), + regions_offshore=resources( + "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), output: - gas_input_nodes=resources("gas_input_locations{weather_year}_s{simpl}_{clusters}.geojson"), + gas_input_nodes=resources( + "gas_input_locations{weather_year}_s{simpl}_{clusters}.geojson" + ), gas_input_nodes_simplified=resources( "gas_input_locations{weather_year}_s{simpl}_{clusters}_simplified.csv" ), @@ -119,10 +131,16 @@ rule build_gas_input_locations: rule cluster_gas_network: input: cleaned_gas_network=resources("gas_network.csv"), - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), - regions_offshore=resources("regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_onshore=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), + regions_offshore=resources( + "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), output: - clustered_gas_network=resources("gas_network_elec{weather_year}_s{simpl}_{clusters}.csv"), + clustered_gas_network=resources( + "gas_network_elec{weather_year}_s{simpl}_{clusters}.csv" + ), resources: mem_mb=4000, log: @@ -139,13 +157,19 @@ rule build_daily_heat_demand: drop_leap_day=config_provider("enable", "drop_leap_day"), input: pop_layout=resources("pop_layout_{scope}{weather_year}.nc"), - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), - cutout=lambda w: "cutouts/" - + CDIR - + config_provider("atlite", "default_cutout")(w) - + ".nc", + regions_onshore=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), + cutout=( + lambda w: "cutouts/" + + CDIR + + config_provider("atlite", "default_cutout")(w) + + ".nc", + ), output: - heat_demand=resources("daily_heat_demand_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc"), + heat_demand=resources( + "daily_heat_demand_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc" + ), resources: mem_mb=20000, threads: 8 @@ -164,9 +188,13 @@ rule build_hourly_heat_demand: snapshots=config_provider("snapshots"), input: heat_profile="data/heat_load_profile_BDEW.csv", - heat_demand=resources("daily_heat_demand_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc"), + heat_demand=resources( + "daily_heat_demand_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc" + ), output: - heat_demand=resources("hourly_heat_demand_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc"), + heat_demand=resources( + "hourly_heat_demand_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc" + ), resources: mem_mb=2000, threads: 8 @@ -186,13 +214,17 @@ rule build_temperature_profiles: drop_leap_day=config_provider("enable", "drop_leap_day"), input: pop_layout=resources("pop_layout_{scope}{weather_year}.nc"), - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_onshore=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), cutout=lambda w: "cutouts/" + CDIR + config_provider("atlite", "default_cutout")(w) + ".nc", output: - temp_soil=resources("temp_soil_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc"), + temp_soil=resources( + "temp_soil_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc" + ), temp_air=resources("temp_air_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, @@ -200,7 +232,9 @@ rule build_temperature_profiles: log: logs("build_temperature_profiles_{scope}_{weather_year}_{simpl}_{clusters}.log"), benchmark: - benchmarks("build_temperature_profiles/{scope}_{weather_year}_s{simpl}_{clusters}") + benchmarks( + "build_temperature_profiles/{scope}_{weather_year}_s{simpl}_{clusters}" + ) conda: "../envs/environment.yaml" script: @@ -211,19 +245,43 @@ rule build_cop_profiles: params: heat_pump_sink_T=config_provider("sector", "heat_pump_sink_T"), input: - temp_soil_total=resources("temp_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc"), - temp_soil_rural=resources("temp_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc"), - temp_soil_urban=resources("temp_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc"), - temp_air_total=resources("temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc"), - temp_air_rural=resources("temp_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc"), - temp_air_urban=resources("temp_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc"), + temp_soil_total=resources( + "temp_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + temp_soil_rural=resources( + "temp_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + temp_soil_urban=resources( + "temp_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + temp_air_total=resources( + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + temp_air_rural=resources( + "temp_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + temp_air_urban=resources( + "temp_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + ), output: - cop_soil_total=resources("cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc"), - cop_soil_rural=resources("cop_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc"), - cop_soil_urban=resources("cop_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc"), - cop_air_total=resources("cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc"), - cop_air_rural=resources("cop_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc"), - cop_air_urban=resources("cop_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc"), + cop_soil_total=resources( + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + cop_soil_rural=resources( + "cop_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + cop_soil_urban=resources( + "cop_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + cop_air_total=resources( + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + cop_air_rural=resources( + "cop_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + cop_air_urban=resources( + "cop_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + ), resources: mem_mb=20000, log: @@ -243,20 +301,28 @@ rule build_solar_thermal_profiles: solar_thermal=config_provider("solar_thermal"), input: pop_layout=resources("pop_layout_{scope}{weather_year}.nc"), - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_onshore=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), cutout=lambda w: "cutouts/" + CDIR + config_provider("atlite", "default_cutout")(w) + ".nc", output: - solar_thermal=resources("solar_thermal_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc"), + solar_thermal=resources( + "solar_thermal_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc" + ), resources: mem_mb=20000, threads: 16 log: - logs("build_solar_thermal_profiles_{scope}_{weather_year}_s{simpl}_{clusters}.log"), + logs( + "build_solar_thermal_profiles_{scope}_{weather_year}_s{simpl}_{clusters}.log" + ), benchmark: - benchmarks("build_solar_thermal_profiles/{scope}_{weather_year}_s{simpl}_{clusters}") + benchmarks( + "build_solar_thermal_profiles/{scope}_{weather_year}_s{simpl}_{clusters}" + ) conda: "../envs/environment.yaml" script: @@ -304,7 +370,7 @@ rule build_heat_totals: log: logs("build_heat_totals.log"), benchmark: - benchmarks("build_heat_totals"), + benchmarks("build_heat_totals") conda: "../envs/environment.yaml" script: @@ -320,7 +386,9 @@ rule build_biomass_potentials: keep_local=True, ), nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_onshore=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"), swiss_cantons=ancient("data/bundle/ch_cantons.csv"), swiss_population=ancient("data/bundle/je-e-21.03.02.xls"), @@ -336,9 +404,13 @@ rule build_biomass_potentials: resources: mem_mb=1000, log: - logs("build_biomass_potentials{weather_year}_s{simpl}_{clusters}_{planning_horizons}.log"), + logs( + "build_biomass_potentials{weather_year}_s{simpl}_{clusters}_{planning_horizons}.log" + ), benchmark: - benchmarks("build_biomass_potentials{weather_year}_s{simpl}_{clusters}_{planning_horizons}") + benchmarks( + "build_biomass_potentials{weather_year}_s{simpl}_{clusters}_{planning_horizons}" + ) conda: "../envs/environment.yaml" script: @@ -376,8 +448,12 @@ rule build_sequestration_potentials: "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", keep_local=True, ), - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), - regions_offshore=resources("regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_onshore=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), + regions_offshore=resources( + "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), output: sequestration_potential=resources( "co2_sequestration_potential_elec{weather_year}_s{simpl}_{clusters}.csv" @@ -398,10 +474,16 @@ rule build_sequestration_potentials: rule build_salt_cavern_potentials: input: salt_caverns="data/bundle-sector/h2_salt_caverns_GWh_per_sqkm.geojson", - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), - regions_offshore=resources("regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_onshore=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), + regions_offshore=resources( + "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), output: - h2_cavern_potential=resources("salt_cavern_potentials{weather_year}_s{simpl}_{clusters}.csv"), + h2_cavern_potential=resources( + "salt_cavern_potentials{weather_year}_s{simpl}_{clusters}.csv" + ), threads: 1 resources: mem_mb=2000, @@ -543,8 +625,12 @@ rule build_industrial_distribution_key: ), countries=config_provider("countries"), input: - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), - clustered_pop_layout=resources("pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv"), + regions_onshore=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), + clustered_pop_layout=resources( + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + ), hotmaps_industrial_database="data/bundle-sector/Industrial_Database.csv", output: industrial_distribution_key=resources( @@ -556,7 +642,9 @@ rule build_industrial_distribution_key: log: logs("build_industrial_distribution_key{weather_year}_s{simpl}_{clusters}.log"), benchmark: - benchmarks("build_industrial_distribution_key/{weather_year}s{simpl}_{clusters}") + benchmarks( + "build_industrial_distribution_key/{weather_year}s{simpl}_{clusters}" + ) conda: "../envs/environment.yaml" script: @@ -670,9 +758,13 @@ rule build_industrial_energy_demand_per_node_today: resources: mem_mb=1000, log: - logs("build_industrial_energy_demand_per_node_today{weather_year}_s{simpl}_{clusters}.log"), + logs( + "build_industrial_energy_demand_per_node_today{weather_year}_s{simpl}_{clusters}.log" + ), benchmark: - benchmarks("build_industrial_energy_demand_per_node_today/{weather_year}s{simpl}_{clusters}") + benchmarks( + "build_industrial_energy_demand_per_node_today/{weather_year}s{simpl}_{clusters}" + ) conda: "../envs/environment.yaml" script: @@ -686,12 +778,16 @@ rule build_retro_cost: input: building_stock="data/retro/data_building_stock.csv", data_tabula="data/bundle-sector/retro/tabula-calculator-calcsetbuilding.csv", - air_temperature=resources("temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc"), + air_temperature=resources( + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), u_values_PL="data/retro/u_values_poland.csv", tax_w="data/retro/elec{weather_year}tricity_taxes_eu.csv", construction_index="data/retro/comparative_level_investment.csv", floor_area_missing="data/retro/floor_area_missing.csv", - clustered_pop_layout=resources("pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv"), + clustered_pop_layout=resources( + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + ), cost_germany="data/retro/retro_cost_germany.csv", window_assumptions="data/retro/window_assumptions.csv", output: @@ -712,14 +808,18 @@ rule build_retro_cost: rule build_population_weighted_energy_totals: input: energy_totals=resources("{kind}_totals.csv"), - clustered_pop_layout=resources("pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv"), + clustered_pop_layout=resources( + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + ), output: resources("pop_weighted_{kind}_totals{weather_year}_s{simpl}_{clusters}.csv"), threads: 1 resources: mem_mb=2000, log: - logs("build_population_weighted_{kind}_totals{weather_year}_s{simpl}_{clusters}.log"), + logs( + "build_population_weighted_{kind}_totals{weather_year}_s{simpl}_{clusters}.log" + ), conda: "../envs/environment.yaml" script: @@ -730,7 +830,9 @@ rule build_shipping_demand: input: ports="data/attributed_ports.json", scope=resources("europe_shape.geojson"), - regions=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), demand=resources("energy_totals.csv"), params: energy_totals_year=config_provider("energy", "energy_totals_year"), @@ -753,16 +855,22 @@ rule build_transport_demand: drop_leap_day=config_provider("enable", "drop_leap_day"), sector=config_provider("sector"), input: - clustered_pop_layout=resources("pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv"), + clustered_pop_layout=resources( + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + ), pop_weighted_energy_totals=resources( "pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv" ), transport_data=resources("transport_data.csv"), traffic_data_KFZ="data/bundle-sector/emobility/KFZ__count", traffic_data_Pkw="data/bundle-sector/emobility/Pkw__count", - temp_air_total=resources("temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc"), + temp_air_total=resources( + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), output: - transport_demand=resources("transport_demand{weather_year}_s{simpl}_{clusters}.csv"), + transport_demand=resources( + "transport_demand{weather_year}_s{simpl}_{clusters}.csv" + ), transport_data=resources("transport_data{weather_year}_s{simpl}_{clusters}.csv"), avail_profile=resources("avail_profile{weather_year}_s{simpl}_{clusters}.csv"), dsm_profile=resources("dsm_profile{weather_year}_s{simpl}_{clusters}.csv"), @@ -782,7 +890,9 @@ rule build_district_heat_share: sector=config_provider("sector"), input: district_heat_share=resources("district_heat_share.csv"), - clustered_pop_layout=resources("pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv"), + clustered_pop_layout=resources( + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + ), output: district_heat_share=resources( "district_heat_share_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" @@ -791,7 +901,9 @@ rule build_district_heat_share: resources: mem_mb=1000, log: - logs("build_district_heat_share{weather_year}_s{simpl}_{clusters}_{planning_horizons}.log"), + logs( + "build_district_heat_share{weather_year}_s{simpl}_{clusters}_{planning_horizons}.log" + ), conda: "../envs/environment.yaml" script: @@ -805,7 +917,9 @@ rule build_existing_heating_distribution: existing_capacities=config_provider("existing_capacities"), input: existing_heating="data/existing_infrastructure/existing_heating_raw.csv", - clustered_pop_layout=resources("pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv"), + clustered_pop_layout=resources( + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + ), clustered_pop_energy_layout=resources( "pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv" ), @@ -835,7 +949,7 @@ rule build_existing_heating_distribution: def input_profile_offwind(w): return { - f"profile_{tech}": resources(f"profile{{weather_year}}_{tech}.nc") + f"profile_{tech}": resources(f"profile{weather_year}_{tech}.nc") for tech in ["offwind-ac", "offwind-dc"] if (tech in config_provider("electricity", "renewable_carriers")(w)) } @@ -883,13 +997,17 @@ rule prepare_sector_network: else [] ), sequestration_potential=lambda w: ( - resources("co2_sequestration_potential_elec{weather_year}_s{simpl}_{clusters}.csv") + resources( + "co2_sequestration_potential_elec{weather_year}_s{simpl}_{clusters}.csv" + ) if config_provider( "sector", "regional_co2_sequestration_potential", "enable" )(w) else [] ), - network=resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), + network=resources( + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + ), energy_totals_name=resources("energy_totals.csv"), eurostat=input_eurostat, pop_weighted_energy_totals=resources( @@ -898,8 +1016,12 @@ rule prepare_sector_network: pop_weighted_heat_totals=resources( "pop_weighted_heat_totals{weather_year}_s{simpl}_{clusters}.csv" ), - shipping_demand=resources("shipping_demand{weather_year}_s{simpl}_{clusters}.csv"), - transport_demand=resources("transport_demand{weather_year}_s{simpl}_{clusters}.csv"), + shipping_demand=resources( + "shipping_demand{weather_year}_s{simpl}_{clusters}.csv" + ), + transport_demand=resources( + "transport_demand{weather_year}_s{simpl}_{clusters}.csv" + ), transport_data=resources("transport_data{weather_year}_s{simpl}_{clusters}.csv"), avail_profile=resources("avail_profile{weather_year}_s{simpl}_{clusters}.csv"), dsm_profile=resources("dsm_profile{weather_year}_s{simpl}_{clusters}.csv"), @@ -920,10 +1042,14 @@ rule prepare_sector_network: if config_provider("foresight")(w) == "overnight" else resources("costs_{planning_horizons}.csv") ), - h2_cavern=resources("salt_cavern_potentials{weather_year}_s{simpl}_{clusters}.csv"), + h2_cavern=resources( + "salt_cavern_potentials{weather_year}_s{simpl}_{clusters}.csv" + ), busmap_s=resources("busmap_elec{weather_year}_s{simpl}.csv"), busmap=resources("busmap_elec{weather_year}_s{simpl}_{clusters}.csv"), - clustered_pop_layout=resources("pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv"), + clustered_pop_layout=resources( + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + ), simplified_pop_layout=resources("pop_layout_elec{weather_year}_s{simpl}.csv"), industrial_demand=resources( "industrial_energy_demand_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" @@ -934,18 +1060,42 @@ rule prepare_sector_network: district_heat_share=resources( "district_heat_share_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" ), - temp_soil_total=resources("temp_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc"), - temp_soil_rural=resources("temp_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc"), - temp_soil_urban=resources("temp_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc"), - temp_air_total=resources("temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc"), - temp_air_rural=resources("temp_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc"), - temp_air_urban=resources("temp_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc"), - cop_soil_total=resources("cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc"), - cop_soil_rural=resources("cop_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc"), - cop_soil_urban=resources("cop_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc"), - cop_air_total=resources("cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc"), - cop_air_rural=resources("cop_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc"), - cop_air_urban=resources("cop_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc"), + temp_soil_total=resources( + "temp_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + temp_soil_rural=resources( + "temp_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + temp_soil_urban=resources( + "temp_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + temp_air_total=resources( + "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + temp_air_rural=resources( + "temp_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + temp_air_urban=resources( + "temp_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + cop_soil_total=resources( + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + cop_soil_rural=resources( + "cop_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + cop_soil_urban=resources( + "cop_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + cop_air_total=resources( + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + cop_air_rural=resources( + "cop_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + cop_air_urban=resources( + "cop_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + ), solar_thermal_total=lambda w: ( resources("solar_thermal_total_elec{weather_year}_s{simpl}_{clusters}.nc") if config_provider("sector", "solar_thermal")(w) diff --git a/rules/collect.smk b/rules/collect.smk index d04ce9cb..e31434f5 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -34,7 +34,9 @@ rule extra_components_networks: rule prepare_elec_networks: input: expand( - resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), + resources( + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + ), **config["scenario"], run=config["run"]["name"], ), @@ -53,7 +55,8 @@ rule prepare_sector_networks: rule solve_elec_networks: input: expand( - RESULTS + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + RESULTS + + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config["scenario"], run=config["run"]["name"], ), diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 8b968681..cee07ea6 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -23,7 +23,9 @@ if config["foresight"] != "perfect": resources: mem_mb=4000, benchmark: - benchmarks("plot_power_network_clustered/elec{weather_year}_s{simpl}_{clusters}") + benchmarks( + "plot_power_network_clustered/elec{weather_year}_s{simpl}_{clusters}" + ) conda: "../envs/environment.yaml" script: @@ -35,7 +37,9 @@ if config["foresight"] != "perfect": input: network=RESULTS + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - regions=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), output: map=RESULTS + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", @@ -62,7 +66,9 @@ if config["foresight"] != "perfect": input: network=RESULTS + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - regions=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), output: map=RESULTS + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf", @@ -88,7 +94,9 @@ if config["foresight"] != "perfect": input: network=RESULTS + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - regions=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), output: map=RESULTS + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf", @@ -125,7 +133,9 @@ if config["foresight"] == "perfect": input: network=RESULTS + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", - regions=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions=resources( + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + ), output: unpack(output_map_year), threads: 2 @@ -282,11 +292,12 @@ rule plot_elec_statistics: plotting=config_provider("plotting"), barplots=STATISTICS_BARPLOTS, input: - network=RESULTS + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network=RESULTS + + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: **{ f"{plot}_bar": RESULTS - + f"figures/statistics_{plot}_bar_elec{{weather_year}}_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" + + f"figures/statistics_{plot}_bar_elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.pdf" for plot in STATISTICS_BARPLOTS }, barplots_touch=RESULTS diff --git a/rules/retrieve.smk b/rules/retrieve.smk index f2d4f993..000d475e 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -179,7 +179,10 @@ if config["enable"]["retrieve"]: script: "../scripts/retrieve_gas_infrastructure_data.py" -if config["enable"]["retrieve"] and config["enable"].get("retrieve_opsd_load_data", True): + +if config["enable"]["retrieve"] and config["enable"].get( + "retrieve_opsd_load_data", True +): rule retrieve_electricity_demand: params: @@ -197,15 +200,26 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_opsd_load_dat "../scripts/retrieve_electricity_demand.py" -if config["enable"]["retrieve"] and config["enable"].get('retrieve_artificial_load_data', False): +if config["enable"]["retrieve"] and config["enable"].get( + "retrieve_artificial_load_data", False +): rule retrieve_artificial_load_data: - input: HTTP.remote("https://zenodo.org/record/7070438/files/demand_hourly.csv", keep_local=True, static=True) - output: "data/load_artificial_raw.csv" - log: logs("retrieve_artificial_load_data.log") - resources: mem_mb=5000, + input: + HTTP.remote( + "https://zenodo.org/record/7070438/files/demand_hourly.csv", + keep_local=True, + static=True, + ), + output: + "data/load_artificial_raw.csv", + log: + logs("retrieve_artificial_load_data.log"), + resources: + mem_mb=5000, retries: 2 - run: move(input[0], output[0]) + run: + move(input[0], output[0]) if config["enable"]["retrieve"]: diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index b404adf3..c69dfb88 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -13,7 +13,9 @@ rule solve_network: ), custom_extra_functionality=input_custom_extra_functionality, input: - network=resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), + network=resources( + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + ), config=RESULTS + "config.yaml", output: network=RESULTS @@ -26,7 +28,10 @@ rule solve_network: python=RESULTS + "logs/solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", benchmark: - RESULTS + "benchmarks/solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" + ( + RESULTS + + "benchmarks/solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" + ) threads: solver_threads resources: mem_mb=memory, diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 26dbf0ef..da31ac46 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -15,14 +15,20 @@ rule add_existing_baseyear: powerplants=resources("powerplants.csv"), busmap_s=resources("busmap_elec{weather_year}_s{simpl}.csv"), busmap=resources("busmap_elec{weather_year}_s{simpl}_{clusters}.csv"), - clustered_pop_layout=resources("pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv"), + clustered_pop_layout=resources( + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + ), costs=lambda w: resources( "costs_{}.csv".format( config_provider("scenario", "planning_horizons", 0)(w) ) ), - cop_soil_total=resources("cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc"), - cop_air_total=resources("cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc"), + cop_soil_total=resources( + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + cop_air_total=resources( + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), existing_heating_distribution=resources( "existing_heating_distribution_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" ), @@ -79,8 +85,12 @@ rule add_brownfield: + "prenetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", network_p=solved_previous_horizon, #solved network at previous time step costs=resources("costs_{planning_horizons}.csv"), - cop_soil_total=resources("cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc"), - cop_air_total=resources("cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc"), + cop_soil_total=resources( + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), + cop_air_total=resources( + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + ), output: RESULTS + "prenetworks-brownfield/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", From fa88e411d797d9b502a794367650a22b98b1f4ec Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 1 Mar 2024 13:25:38 +0100 Subject: [PATCH 086/173] Automatic formatting --- scripts/prepare_sector_network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 985bfa5c..bc714096 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3515,7 +3515,9 @@ def set_temporal_aggregation(n, resolution, solver_name, drop_leap_day=False): elif "seg" in resolution.lower(): segments = int(resolution[:-3]) logger.info("Use temporal segmentation with %s segments", segments) - n = apply_time_segmentation(n, segments, solver_name=solver_name, drop_leap_day=drop_leap_day) + n = apply_time_segmentation( + n, segments, solver_name=solver_name, drop_leap_day=drop_leap_day + ) # temporal averaging elif "h" in resolution.lower(): From d2527ed0ad1b48281949cb75d6d2b8cad9348da1 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 1 Mar 2024 13:30:01 +0100 Subject: [PATCH 087/173] Add copyright headers --- scripts/build_heat_totals.py | 3 +++ scripts/solve_operations_network_other_year.py | 3 +++ scripts/solve_operations_network_other_year_myopic.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/scripts/build_heat_totals.py b/scripts/build_heat_totals.py index 93e8f9c7..e49ae635 100644 --- a/scripts/build_heat_totals.py +++ b/scripts/build_heat_totals.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT """ Approximate heat demand for all weather years. """ diff --git a/scripts/solve_operations_network_other_year.py b/scripts/solve_operations_network_other_year.py index f7d8f1e8..91f067cc 100644 --- a/scripts/solve_operations_network_other_year.py +++ b/scripts/solve_operations_network_other_year.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT """ Solve operations network. """ diff --git a/scripts/solve_operations_network_other_year_myopic.py b/scripts/solve_operations_network_other_year_myopic.py index 5fc8e6be..5858d79f 100644 --- a/scripts/solve_operations_network_other_year_myopic.py +++ b/scripts/solve_operations_network_other_year_myopic.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT """ Solve myopic operations network. """ From e891305c9dd30fd6d09884f071f2f05ac8a5d023 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 1 Mar 2024 14:37:16 +0100 Subject: [PATCH 088/173] Add selection of cutouts for heat to config --- config/config.default.yaml | 2 ++ rules/build_sector.smk | 45 ++++++++++++++++--------- scripts/build_solar_thermal_profiles.py | 1 + 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 60c492b1..bb714027 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -359,6 +359,7 @@ solar_thermal: orientation: slope: 45. azimuth: 180. + cutout: default # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#existing-capacities existing_capacities: @@ -391,6 +392,7 @@ sector: 2050: 1.0 district_heating_loss: 0.15 cluster_heat_buses: true + heat_demand_cutout: default bev_dsm_restriction_value: 0.75 bev_dsm_restriction_time: 7 transport_heating_deadband_upper: 20. diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 9fb6876b..e8af7f7b 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -151,6 +151,19 @@ rule cluster_gas_network: "../scripts/cluster_gas_network.py" +def heat_demand_cutout(wildcards): + c = config_provider("sector", "heat_demand_cutout")(wildcards) + if c == "default": + return ( + "cutouts/" + + CDIR + + config_provider("atlite", "default_cutout")(wildcards) + + ".nc" + ) + else: + return "cutouts/" + CDIR + c + ".nc" + + rule build_daily_heat_demand: params: snapshots=config_provider("snapshots"), @@ -160,12 +173,7 @@ rule build_daily_heat_demand: regions_onshore=resources( "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" ), - cutout=( - lambda w: "cutouts/" - + CDIR - + config_provider("atlite", "default_cutout")(w) - + ".nc", - ), + cutout=heat_demand_cutout, output: heat_demand=resources( "daily_heat_demand_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc" @@ -217,10 +225,7 @@ rule build_temperature_profiles: regions_onshore=resources( "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" ), - cutout=lambda w: "cutouts/" - + CDIR - + config_provider("atlite", "default_cutout")(w) - + ".nc", + cutout=heat_demand_cutout, output: temp_soil=resources( "temp_soil_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc" @@ -294,6 +299,19 @@ rule build_cop_profiles: "../scripts/build_cop_profiles.py" +def solar_thermal_cutout(wildcards): + c = config_provider("solar_thermal", "cutout")(wildcards) + if c == "default": + return ( + "cutouts/" + + CDIR + + config_provider("atlite", "default_cutout")(wildcards) + + ".nc" + ) + else: + return "cutouts/" + CDIR + c + ".nc" + + rule build_solar_thermal_profiles: params: snapshots=config_provider("snapshots"), @@ -304,10 +322,7 @@ rule build_solar_thermal_profiles: regions_onshore=resources( "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" ), - cutout=lambda w: "cutouts/" - + CDIR - + config_provider("atlite", "default_cutout")(w) - + ".nc", + cutout=solar_thermal_cutout, output: solar_thermal=resources( "solar_thermal_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc" @@ -949,7 +964,7 @@ rule build_existing_heating_distribution: def input_profile_offwind(w): return { - f"profile_{tech}": resources(f"profile{weather_year}_{tech}.nc") + f"profile_{tech}": resources(f"profile{{weather_year}}_{tech}.nc") for tech in ["offwind-ac", "offwind-dc"] if (tech in config_provider("electricity", "renewable_carriers")(w)) } diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index b0896fa7..db56a401 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -31,6 +31,7 @@ if __name__ == "__main__": client = Client(cluster, asynchronous=True) config = snakemake.params.solar_thermal + config.pop("cutout", None) cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year From 0c7c0a8585f2f35bd7ae52e333f1950a979728fd Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 1 Mar 2024 14:37:45 +0100 Subject: [PATCH 089/173] Minor bugfixes in postprocessing code --- rules/postprocess.smk | 2 +- scripts/make_summary.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index cee07ea6..67b71503 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -297,7 +297,7 @@ rule plot_elec_statistics: output: **{ f"{plot}_bar": RESULTS - + f"figures/statistics_{plot}_bar_elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.pdf" + + f"figures/statistics_{plot}_bar_elec{{weather_year}}_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" for plot in STATISTICS_BARPLOTS }, barplots_touch=RESULTS diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 5dc18ec9..67089f92 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -681,7 +681,7 @@ if __name__ == "__main__": networks_dict = { (weather_year, cluster, ll, opt + sector_opt, planning_horizon): "results/" + snakemake.params.RDIR - + f"/postnetworks/elec_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc" + + f"postnetworks/elec{weather_year}_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc" for weather_year in snakemake.params.scenario["weather_year"] for simpl in snakemake.params.scenario["simpl"] for cluster in snakemake.params.scenario["clusters"] From 61d565ec0995498fea28d3bccbb836bb12b16449 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 1 Mar 2024 15:26:16 +0100 Subject: [PATCH 090/173] Correct formatting in build_electricity.smk --- rules/build_electricity.smk | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 3f353dd9..3c8969aa 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -428,7 +428,7 @@ rule build_line_rating: def input_profile_tech(w): return { - f"profile_{tech}": resources(f"profile{weather_year}_{tech}.nc") + f"profile_{tech}": resources(f"profile{{weather_year}}_{tech}.nc") for tech in config_provider("electricity", "renewable_carriers")(w) } @@ -463,7 +463,7 @@ rule add_electricity: else resources("networks/base.nc") ), tech_costs=lambda w: resources( - f"costs_{config_provider('costs' , 'year') (w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), regions=resources("regions_onshore.geojson"), powerplants=resources("powerplants.csv"), @@ -508,7 +508,7 @@ rule simplify_network: input: network=resources("networks/elec{weather_year}.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs' , 'year') (w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), regions_onshore=resources("regions_onshore.geojson"), regions_offshore=resources("regions_offshore.geojson"), @@ -561,7 +561,7 @@ rule cluster_network: else [] ), tech_costs=lambda w: resources( - f"costs_{config_provider('costs' , 'year') (w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), output: network=resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"), @@ -594,7 +594,7 @@ rule add_extra_components: input: network=resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs' , 'year') (w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), output: resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc"), @@ -628,7 +628,7 @@ rule prepare_network: input: resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs' , 'year') (w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), co2_price=lambda w: resources("co2_price.csv") if "Ept" in w.opts else [], output: From 75ff4134fb39a17a26f3c0b86463fd29f0ed856e Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 1 Mar 2024 15:26:47 +0100 Subject: [PATCH 091/173] Select correct snapshots for hourly heating demand --- rules/build_sector.smk | 1 + scripts/build_hourly_heat_demand.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index e8af7f7b..b4e41df8 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -194,6 +194,7 @@ rule build_daily_heat_demand: rule build_hourly_heat_demand: params: snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), input: heat_profile="data/heat_load_profile_BDEW.csv", heat_demand=resources( diff --git a/scripts/build_hourly_heat_demand.py b/scripts/build_hourly_heat_demand.py index a1dacd51..664b4499 100644 --- a/scripts/build_hourly_heat_demand.py +++ b/scripts/build_hourly_heat_demand.py @@ -24,7 +24,16 @@ if __name__ == "__main__": ) set_scenario_config(snakemake) - snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) + year = snakemake.wildcards.weather_year + + if year: + snapshots = dict(start=year, end=str(int(year) + 1), inclusive="left") + else: + snapshots = snakemake.params.snapshots + + snapshots = pd.date_range(freq="h", **snapshots) + if snakemake.params.drop_leap_day: + snapshots = snapshots[~((snapshots.month == 2) & (snapshots.day == 29))] daily_space_heat_demand = ( xr.open_dataarray(snakemake.input.heat_demand) From 9a69f88a36bdeb958e1c763646652a4a98eb2f96 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 1 Mar 2024 15:49:39 +0100 Subject: [PATCH 092/173] Fix bug bug in time segmentation function signature --- scripts/prepare_sector_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index bc714096..6ca06c9a 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3425,7 +3425,7 @@ def cluster_heat_buses(n): def apply_time_segmentation( - n, segments, solver_name="cbc", overwrite_time_dependent=True + n, segments, solver_name="cbc", overwrite_time_dependent=True, drop_leap_day=False ): """ Aggregating time series to segments with different lengths. From e48b25f777a9c53364988d97a87b21673a1425b5 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 1 Mar 2024 15:50:21 +0100 Subject: [PATCH 093/173] Fix unresolved or faulty merge conflicts in documentation --- doc/_static/theme_overrides.css | 74 --------------------------------- doc/configtables/enable.csv | 1 + doc/configtables/hydro.csv | 15 ++----- doc/configtables/toplevel.csv | 28 ------------- doc/configuration.rst | 6 +-- doc/tutorial.rst | 12 ------ 6 files changed, 5 insertions(+), 131 deletions(-) delete mode 100644 doc/_static/theme_overrides.css diff --git a/doc/_static/theme_overrides.css b/doc/_static/theme_overrides.css deleted file mode 100644 index 827dbbad..00000000 --- a/doc/_static/theme_overrides.css +++ /dev/null @@ -1,74 +0,0 @@ -/* SPDX-FileCopyrightText: 2017-2020 The PyPSA-Eur Authors - SPDX-License-Identifier: MIT -*/ - -.wy-side-nav-search { - background-color: #eeeeee; -} - -.wy-side-nav-search .wy-dropdown>a, -.wy-side-nav-search>a { - color: rgb(34, 97, 156) -} - -.wy-side-nav-search>div.version { - color: rgb(34, 97, 156) -} - -.wy-menu-vertical header, -.wy-menu-vertical p.caption, -.rst-versions a { - color: #999999; -} - -.wy-menu-vertical a.reference:hover, -.wy-menu-vertical a.reference.internal:hover { - background: #dddddd; - color: #fff; -} - -.wy-nav-side { - background: #efefef; -} - -.wy-menu-vertical a.reference { - color: #000; -} - -.rst-versions .rst-current-version, -.wy-nav-top, -.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a:hover { - background: #002221; -} - -.wy-nav-content .highlight { - background: #ffffff; -} - -.rst-content code.literal, -.rst-content tt.literal { - color: rgb(34, 97, 156) -} - -.wy-nav-content a.reference { - color: rgb(34, 97, 156); -} - - -/* override table width restrictions */ - -@media screen and (min-width: 767px) { - .wy-table-responsive table td { - /* !important prevents the common CSS stylesheets from overriding - this as on RTD they are loaded after this stylesheet */ - white-space: normal !important; - background: rgb(250, 250, 250) !important; - } - .wy-table-responsive { - max-width: 100%; - overflow: visible !important; - } - .wy-nav-content { - max-width: 910px !important; - } -} diff --git a/doc/configtables/enable.csv b/doc/configtables/enable.csv index 8dd476cb..1e5571b3 100644 --- a/doc/configtables/enable.csv +++ b/doc/configtables/enable.csv @@ -10,3 +10,4 @@ retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`." retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`." custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``." +drop_leap_day,bool,"{true, false}","Switch to drop February 29 from all time-dependent data in leap years" diff --git a/doc/configtables/hydro.csv b/doc/configtables/hydro.csv index f9c8f0d5..790029d1 100644 --- a/doc/configtables/hydro.csv +++ b/doc/configtables/hydro.csv @@ -1,14 +1,3 @@ -<<<<<<< HEAD -,Unit,Values,Description -cutout,--,"Must be 'europe-2013-era5'","Specifies the directory where the relevant weather data ist stored." -carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydro power plants to build per-unit availability time series for. 'ror' stands for run-of-river plants, 'PHS' represents pumped-hydro storage, and 'hydro' stands for hydroelectric dams." -PHS_max_hours,h,float,"Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_." -hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}","Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation `_." -clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero." -eia_norm_year,--,"Year in EIA hydro generation dataset; or False to disable","To specify a specific year by which hydro inflow is normed that deviates from the snapshots' year" -eia_correct_by_capacity,--,boolean,"Correct EIA annual hydro generation data by installed capacity." -eia_approximate_missing,--,boolean,"Approximate hydro generation data for years not included in EIA dataset through a regression based on annual runoff." -======= ,Unit,Values,Description cutout,--,Must be 'europe-2013-era5',Specifies the directory where the relevant weather data ist stored. carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydro power plants to build per-unit availability time series for. 'ror' stands for run-of-river plants, 'PHS' represents pumped-hydro storage, and 'hydro' stands for hydroelectric dams." @@ -17,4 +6,6 @@ hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate flatten_dispatch,bool,"{true, false}",Consider an upper limit for the hydro dispatch. The limit is given by the average capacity factor plus the buffer given in ``flatten_dispatch_buffer`` flatten_dispatch_buffer,--,float,"If ``flatten_dispatch`` is true, specify the value added above the average capacity factor." clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero." ->>>>>>> master +eia_norm_year,--,"Year in EIA hydro generation dataset; or False to disable","To specify a specific year by which hydro inflow is normed that deviates from the snapshots' year" +eia_correct_by_capacity,--,boolean,"Correct EIA annual hydro generation data by installed capacity." +eia_approximate_missing,--,boolean,"Approximate hydro generation data for years not included in EIA dataset through a regression based on annual runoff." diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv index 74cc45d8..418acbab 100644 --- a/doc/configtables/toplevel.csv +++ b/doc/configtables/toplevel.csv @@ -1,40 +1,12 @@ -<<<<<<< HEAD ,Unit,Values,Description version,--,0.x.x,"Version of PyPSA-Eur. Descriptive only." tutorial,bool,"{true, false}","Switch to retrieve the tutorial data set instead of the full data set." logging,,, -- level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only" -- format,--,"","Custom format for log messages. See `LogRecord `_ attributes." -<<<<<<< HEAD -foresight,string,"{overnight, myopic, perfect}","Defaults to overnight scenarios." -countries,--,"Subset of {'AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'}","European countries defined by their `Two-letter country codes (ISO 3166-1) `_ which should be included in the energy system model." -focus_weights,--,"Keys should be two-digit country codes (e.g. DE) and values should range between 0 and 1","Ratio of total clusters for particular countries. the remaining weight is distributed according to mean load. An example: ``focus_weights: 'DE': 0.6 'FR': 0.2``." -<<<<<<< HEAD -enable,,, --- prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia `_" --- retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder." --- build_cutout,bool,"{true, false}","Switch to enable the building of cutouts via the rule :mod:`build_cutout`." --- retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts from zenodo with :mod:`retrieve_cutout`." --- build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`." --- retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`." --- custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec{weather_year}_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec{weather_year}_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``." --- drop_leap_day,bool,"{true, false}","Switch to drop February 29 from all time-dependent data in leap years" -======= -co2_budget,--,"Dictionary with planning horizons as keys.","CO2 budget as a fraction of 1990 emissions. Overwritten if ``CO2Lx`` or ``cb`` are set in ``{sector_opts}`` wildcard" ->>>>>>> master -======= ->>>>>>> origin/master -======= -,Unit,Values,Description -version,--,0.x.x,Version of PyPSA-Eur. Descriptive only. -tutorial,bool,"{true, false}",Switch to retrieve the tutorial data set instead of the full data set. -logging,,, --- level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only" --- format,--,,Custom format for log messages. See `LogRecord `_ attributes. private,,, -- keys,,, -- -- entsoe_api,--,,Optionally specify the ENTSO-E API key. See the guidelines to get `ENTSO-E API key `_ remote,,, -- ssh,--,,Optionally specify the SSH of a remote cluster to be synchronized. -- path,--,,Optionally specify the file path within the remote cluster to be synchronized. ->>>>>>> master diff --git a/doc/configuration.rst b/doc/configuration.rst index 91e6f1ad..9eba4284 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -82,22 +82,18 @@ facilitate running multiple scenarios through a single command # for electricity-only studies snakemake -call solve_elec_networks -<<<<<<< HEAD -For each wildcard, a **list of values** is provided. The rule ``solve_all_networks`` will trigger the rules for creating ``results/networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc`` for **all combinations** of the provided wildcard values as defined by Python's `itertools.product(...) `_ function that snakemake's `expand(...) function `_ uses. -======= # for sector-coupling studies snakemake -call solve_sector_networks For each wildcard, a **list of values** is provided. The rule ``solve_all_elec_networks`` will trigger the rules for creating -``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc`` for **all +``results/networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc`` for **all combinations** of the provided wildcard values as defined by Python's `itertools.product(...) `_ function that snakemake's `expand(...) function `_ uses. ->>>>>>> master An exemplary dependency graph (starting from the simplification rules) then looks like this: diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 9fc91174..4f272292 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -119,21 +119,9 @@ clustered down to 6 buses and every 24 hours aggregated to one snapshot. The com orders ``snakemake`` to run the rule :mod:`solve_network` that produces the solved network and stores it in ``results/networks`` with the name ``elec_s_6_ec_lcopt_Co2L-24H.nc``: -<<<<<<< HEAD -.. code:: - - rule solve_network: - input: "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - [...] - script: "scripts/solve_network.py" - -.. until https://github.com/snakemake/snakemake/issues/46 closed -======= .. literalinclude:: ../rules/solve_electricity.smk :start-at: rule solve_network: :end-before: rule solve_operations_network: ->>>>>>> master This triggers a workflow of multiple preceding jobs that depend on each rule's inputs and outputs: From 1ff8321c5e875ebd37b4b2d17ecc5db980ac1cf8 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 1 Mar 2024 15:54:42 +0100 Subject: [PATCH 094/173] Remove erroneous automatic snakefmt formatting --- rules/build_electricity.smk | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 3c8969aa..a83a9d93 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -463,7 +463,7 @@ rule add_electricity: else resources("networks/base.nc") ), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year') (w)}.csv" + f"costs_{config_provider('costs', 'year')(w)}.csv" ), regions=resources("regions_onshore.geojson"), powerplants=resources("powerplants.csv"), @@ -508,7 +508,7 @@ rule simplify_network: input: network=resources("networks/elec{weather_year}.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year') (w)}.csv" + f"costs_{config_provider('costs', 'year')(w)}.csv" ), regions_onshore=resources("regions_onshore.geojson"), regions_offshore=resources("regions_offshore.geojson"), @@ -561,7 +561,7 @@ rule cluster_network: else [] ), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year') (w)}.csv" + f"costs_{config_provider('costs', 'year')(w)}.csv" ), output: network=resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"), @@ -594,7 +594,7 @@ rule add_extra_components: input: network=resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year') (w)}.csv" + f"costs_{config_provider('costs', 'year')(w)}.csv" ), output: resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc"), @@ -628,7 +628,7 @@ rule prepare_network: input: resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year') (w)}.csv" + f"costs_{config_provider('costs', 'year')(w)}.csv" ), co2_price=lambda w: resources("co2_price.csv") if "Ept" in w.opts else [], output: From 9607eb0e6bd2171914741a68eaf0da1798cb56fa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:16:26 +0000 Subject: [PATCH 095/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_clustered_population_layouts.py | 2 +- scripts/build_energy_totals.py | 140 ++++++++++++------ scripts/build_hydro_profile.py | 2 +- scripts/cluster_gas_network.py | 4 +- 4 files changed, 102 insertions(+), 46 deletions(-) diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index d4cfb96d..619acee8 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -25,7 +25,7 @@ if __name__ == "__main__": ) set_scenario_config(snakemake) - + cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year if year: diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 00e9179c..954769ea 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -35,6 +35,7 @@ def reverse(dictionary): """ return {v: k for k, v in dictionary.items()} + non_EU = ["NO", "CH", "ME", "MK", "RS", "BA", "AL"] idees_rename = {"GR": "EL", "GB": "UK"} @@ -72,7 +73,10 @@ to_ipcc = { def eurostat_per_country(country): country_fn = idees_rename.get(country, country) - fn = snakemake.input.eurostat + f"/{country_fn}-Energy-balance-sheets-June-2021-edition.xlsb" + fn = ( + snakemake.input.eurostat + + f"/{country_fn}-Energy-balance-sheets-June-2021-edition.xlsb" + ) df = pd.read_excel( fn, @@ -93,27 +97,31 @@ def build_eurostat(countries, year=None): """ nprocesses = snakemake.threads - tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries), - desc='Build from eurostat database') + tqdm_kwargs = dict( + ascii=False, + unit=" country", + total=len(countries), + desc="Build from eurostat database", + ) with mp.Pool(processes=nprocesses) as pool: dfs = list(tqdm(pool.imap(eurostat_per_country, countries), **tqdm_kwargs)) - index_names = ['country', 'year', 'lvl1', 'lvl2', 'lvl3'] + index_names = ["country", "year", "lvl1", "lvl2", "lvl3"] df = pd.concat(dfs, keys=countries, names=index_names) - df.dropna(how='all', axis=0, inplace=True) - df.dropna(how='all', axis=1, inplace=True) - df = df[df.index.get_level_values('lvl1') != 'ktoe'] + df.dropna(how="all", axis=0, inplace=True) + df.dropna(how="all", axis=1, inplace=True) + df = df[df.index.get_level_values("lvl1") != "ktoe"] i = df.index.to_frame(index=False) - i.loc[i.lvl2 == 'Primary production', ['lvl1', 'lvl3']] = 'Main' - i.loc[i.lvl2 == 'Gross electricity production', 'lvl1'] = "Gross production" + i.loc[i.lvl2 == "Primary production", ["lvl1", "lvl3"]] = "Main" + i.loc[i.lvl2 == "Gross electricity production", "lvl1"] = "Gross production" i.ffill(inplace=True) df.index = pd.MultiIndex.from_frame(i) df.drop(list(range(1990, 2020)), axis=1, inplace=True) df.drop("Unnamed: 7", axis=1, inplace=True) - df.fillna(0., inplace=True) + df.fillna(0.0, inplace=True) # convert ktoe/a to TWh/a df *= 11.63 / 1e3 @@ -121,7 +129,7 @@ def build_eurostat(countries, year=None): df.index = df.index.set_levels(df.index.levels[1].astype(int), level=1) if year: - df = df.xs(year, level='year') + df = df.xs(year, level="year") return df @@ -133,12 +141,12 @@ def build_swiss(year=None): fn = snakemake.input.swiss - df = pd.read_csv(fn, index_col=[0,1]).stack().unstack('item') + df = pd.read_csv(fn, index_col=[0, 1]).stack().unstack("item") df.index.names = ["country", "year"] df.index = df.index.set_levels(df.index.levels[1].astype(int), level=1) if year: - df = df.xs(year, level='year') + df = df.xs(year, level="year") # convert PJ/a to TWh/a df /= 3.6 @@ -356,14 +364,14 @@ def build_idees(countries, year=None): unit=" country", total=len(countries), desc="Build from IDEES database", - disable=disable_progress + disable=disable_progress, ) with mute_print(): with mp.Pool(processes=nprocesses) as pool: totals_list = list(tqdm(pool.imap(func, countries), **tqdm_kwargs)) - totals = pd.concat(totals_list, keys=countries, names=['country', 'year']) + totals = pd.concat(totals_list, keys=countries, names=["country", "year"]) # convert ktoe to TWh exclude = totals.columns.str.fullmatch("passenger cars") @@ -377,23 +385,19 @@ def build_idees(countries, year=None): def build_energy_totals(countries, eurostat, swiss, idees): - eurostat_fuels = dict( - electricity="Electricity", - total="Total" - ) + eurostat_fuels = dict(electricity="Electricity", total="Total") eurostat_sectors = dict( residential="Households", services="Commercial & public services", road="Road", - rail="Rail" + rail="Rail", ) to_drop = ["passenger cars", "passenger car efficiency"] new_index = pd.MultiIndex.from_product( - [countries, eurostat.index.levels[1]], - names=["country", "year"] + [countries, eurostat.index.levels[1]], names=["country", "year"] ) df = idees.reindex(new_index).drop(to_drop, axis=1) @@ -404,7 +408,7 @@ def build_energy_totals(countries, eurostat, swiss, idees): # add international navigation slicer = idx[in_eurostat, :, :, "International maritime bunkers", :] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0,1]).sum() + fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0, 1]).sum() df.loc[in_eurostat, "total international navigation"] = fill_values # add swiss energy data @@ -414,11 +418,14 @@ def build_energy_totals(countries, eurostat, swiss, idees): # get values for missing countries based on Eurostat EnergyBalances # divide cooking/space/water according to averages in EU28 - to_fill = df.index[df["total residential"].isna() & df.index.get_level_values('country').isin(eurostat_countries)] + to_fill = df.index[ + df["total residential"].isna() + & df.index.get_level_values("country").isin(eurostat_countries) + ] uses = ["space", "cooking", "water"] - c = to_fill.get_level_values('country') - y = to_fill.get_level_values('year') + c = to_fill.get_level_values("country") + y = to_fill.get_level_values("year") for sector in ["residential", "services", "road", "rail"]: @@ -427,7 +434,9 @@ def build_energy_totals(countries, eurostat, swiss, idees): for fuel in ["electricity", "total"]: slicer = idx[c, y, :, :, eurostat_sectors[sector]] - fill_values = eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=[0,1]).sum() + fill_values = ( + eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=[0, 1]).sum() + ) df.loc[to_fill, f"{fuel} {sector}"] = fill_values for sector in ["residential", "services"]: @@ -488,7 +497,9 @@ def build_energy_totals(countries, eurostat, swiss, idees): no_norway[f"total {sector}"] - no_norway[f"electricity {sector}"] ) fraction = nonelectric_use.div(nonelectric).mean() - df.loc["NO", f"total {sector} {use}"] = (total_heating * fraction).values + df.loc["NO", f"total {sector} {use}"] = ( + total_heating * fraction + ).values df.loc["NO", f"electricity {sector} {use}"] = ( total_heating * fraction * elec_fraction ).values @@ -496,17 +507,17 @@ def build_energy_totals(countries, eurostat, swiss, idees): # Missing aviation slicer = idx[c, y, :, :, "Domestic aviation"] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0,1]).sum() + fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0, 1]).sum() df.loc[to_fill, "total domestic aviation"] = fill_values slicer = idx[c, y, :, "International aviation", :] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0,1]).sum() + fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0, 1]).sum() df.loc[to_fill, "total international aviation"] = fill_values # missing domestic navigation slicer = idx[c, y, :, :, "Domestic navigation"] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0,1]).sum() + fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0, 1]).sum() df.loc[to_fill, "total domestic navigation"] = fill_values # split road traffic for non-IDEES @@ -653,13 +664,15 @@ def build_eurostat_co2(countries, eurostat=None, year=1990): if eurostat is None: df = build_eurostat(countries, year) else: - df = eurostat.xs(year, level='year') + df = eurostat.xs(year, level="year") specific_emissions = pd.Series(index=df.columns, dtype=float) # emissions in tCO2_equiv per MWh_th specific_emissions["Solid fossil fuels"] = 0.36 # Approximates coal - specific_emissions["Oil and petroleum products"] = 0.285 # Average of distillate and residue + specific_emissions["Oil and petroleum products"] = ( + 0.285 # Average of distillate and residue + ) specific_emissions["Natural gas"] = 0.2 # For natural gas # oil values from https://www.eia.gov/tools/faqs/faq.cfm?id=74&t=11 @@ -675,19 +688,60 @@ def build_co2_totals(countries, eea_co2, eurostat_co2): for ct in pd.Index(countries).intersection(["BA", "RS", "AL", "ME", "MK"]): mappings = { - "electricity": (ct, "Transformation input", "Electricity & heat generation", "Main"), - "residential non-elec": (ct, "Final energy consumption", "Other sectors", "Households"), - "services non-elec": (ct, "Final energy consumption", "Other sectors", "Commercial & public services"), - "road non-elec": (ct, "Final energy consumption", "Transport sector", "Road"), - "rail non-elec": (ct, "Final energy consumption", "Transport sector", "Rail"), - "domestic navigation": (ct, "Final energy consumption", "Transport sector", "Domestic navigation"), + "electricity": ( + ct, + "Transformation input", + "Electricity & heat generation", + "Main", + ), + "residential non-elec": ( + ct, + "Final energy consumption", + "Other sectors", + "Households", + ), + "services non-elec": ( + ct, + "Final energy consumption", + "Other sectors", + "Commercial & public services", + ), + "road non-elec": ( + ct, + "Final energy consumption", + "Transport sector", + "Road", + ), + "rail non-elec": ( + ct, + "Final energy consumption", + "Transport sector", + "Rail", + ), + "domestic navigation": ( + ct, + "Final energy consumption", + "Transport sector", + "Domestic navigation", + ), "international navigation": (ct, "Main", "International maritime bunkers"), - "domestic aviation": (ct, "Final energy consumption", "Transport sector", "Domestic aviation"), + "domestic aviation": ( + ct, + "Final energy consumption", + "Transport sector", + "Domestic aviation", + ), "international aviation": (ct, "Main", "International aviation"), # does not include industrial process emissions or fuel processing/refining - "industrial non-elec": (ct, "Final energy consumption", "Industry sector", "Non-energy use in industry sector"), + "industrial non-elec": ( + ct, + "Final energy consumption", + "Industry sector", + "Non-energy use in industry sector", + ), # does not include non-energy emissions - "agriculture": (eurostat_co2.index.get_level_values(0) == ct) & eurostat_co2.index.isin(["Agriculture & forestry", "Fishing"], level=3), + "agriculture": (eurostat_co2.index.get_level_values(0) == ct) + & eurostat_co2.index.isin(["Agriculture & forestry", "Fishing"], level=3), } for i, mi in mappings.items(): @@ -771,6 +825,6 @@ if __name__ == "__main__": co2 = build_co2_totals(countries, eea_co2, eurostat_co2) co2.to_csv(snakemake.output.co2_name) - idees_transport = idees.xs(data_year, level='year') + idees_transport = idees.xs(data_year, level="year") transport = build_transport_data(countries, population, idees_transport) transport.to_csv(snakemake.output.transport_name) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 5571e9d7..af5f0ee6 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -65,8 +65,8 @@ import atlite import country_converter as coco import geopandas as gpd import pandas as pd -from numpy.polynomial import Polynomial from _helpers import configure_logging, set_scenario_config +from numpy.polynomial import Polynomial cc = coco.CountryConverter() diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index fd335cd3..8c03665b 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -105,7 +105,9 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("cluster_gas_network", weather_year="", simpl="", clusters="37") + snakemake = mock_snakemake( + "cluster_gas_network", weather_year="", simpl="", clusters="37" + ) configure_logging(snakemake) set_scenario_config(snakemake) From e6347621f9a81825a6fd3b7c0d64c8269dadcdce Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 4 Mar 2024 17:42:35 +0100 Subject: [PATCH 096/173] remove operations rules to reduce amount of simultaneous changes --- config/config.default.yaml | 15 -- rules/solve_overnight.smk | 58 ------ .../solve_operations_network_other_year.py | 147 ---------------- ...ve_operations_network_other_year_myopic.py | 166 ------------------ 4 files changed, 386 deletions(-) delete mode 100644 scripts/solve_operations_network_other_year.py delete mode 100644 scripts/solve_operations_network_other_year_myopic.py diff --git a/config/config.default.yaml b/config/config.default.yaml index bb714027..892a1e74 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -828,21 +828,6 @@ solving: walltime: "12:00:00" -operations: - rolling_horizon: - window: 10 # days - overlap: 8 # days - bidding_prices: # EUR/MW - H2 Electrolysis: -10 - H2 Fuel Cell: 200 - urban central water tanks charger: -10 - urban central water tanks discharger: 10 - hydro: 70 - solid biomass: 150 - biogas: 100 - co2_price: 500 # EUR/t - co2_sequestation_limit: 200 # Mt/a - # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#plotting plotting: map: diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index c190c14f..21fce682 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -41,61 +41,3 @@ rule solve_sector_network: "../envs/environment.yaml" script: "../scripts/solve_network.py" - - -rule solve_operations_network_other_year: - input: - pre=RDIR - + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - post=RDIR - + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - output: - RDIR - + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc", - shadow: - "shallow" - log: - solver=RDIR - + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", - python=RDIR - + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_python.log", - threads: 4 - resources: - mem_mb=10000, - benchmark: - ( - RDIR - + "/benchmarks/solve_operations_network/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}" - ) - script: - "../scripts/solve_operations_network_other_year.py" - - -# rule solve_operations_network_other_year_myopic: -# input: -# overrides="data/override_component_attrs", -# pre=RDIR -# + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", -# post=RDIR -# + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", -# previous=solved_previous_year, -# output: -# RDIR -# + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc", -# shadow: -# "shallow" -# log: -# solver=RDIR -# + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", -# python=RDIR -# + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_python.log", -# threads: 4 -# resources: -# mem_mb=10000, -# benchmark: -# ( -# RDIR -# + "/benchmarks/solve_operations_network_myopic/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}" -# ) -# script: -# "../scripts/solve_operations_network_other_year_myopic.py" diff --git a/scripts/solve_operations_network_other_year.py b/scripts/solve_operations_network_other_year.py deleted file mode 100644 index 91f067cc..00000000 --- a/scripts/solve_operations_network_other_year.py +++ /dev/null @@ -1,147 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors -# -# SPDX-License-Identifier: MIT -""" -Solve operations network. -""" - - -import logging - -import numpy as np -import pypsa -from solve_network import prepare_network, solve_network - -logger = logging.getLogger(__name__) -pypsa.pf.logger.setLevel(logging.WARNING) - - -def set_parameters_from_optimized(n, n_optim): - lines_typed_i = n.lines.index[n.lines.type != ""] - n.lines.loc[lines_typed_i, "num_parallel"] = n_optim.lines["num_parallel"].reindex( - lines_typed_i, fill_value=0.0 - ) - n.lines.loc[lines_typed_i, "s_nom"] = ( - np.sqrt(3) - * n.lines["type"].map(n.line_types.i_nom) - * n.lines.bus0.map(n.buses.v_nom) - * n.lines.num_parallel - ) - - lines_untyped_i = n.lines.index[n.lines.type == ""] - for attr in ("s_nom", "r", "x"): - n.lines.loc[lines_untyped_i, attr] = n_optim.lines[attr].reindex( - lines_untyped_i, fill_value=0.0 - ) - n.lines["s_nom_extendable"] = False - - links_dc_i = n.links.index[n.links.p_nom_extendable] - n.links.loc[links_dc_i, "p_nom"] = n_optim.links["p_nom_opt"].reindex( - links_dc_i, fill_value=0.0 - ) - n.links.loc[links_dc_i, "p_nom_extendable"] = False - - gen_extend_i = n.generators.index[n.generators.p_nom_extendable] - n.generators.loc[gen_extend_i, "p_nom"] = n_optim.generators["p_nom_opt"].reindex( - gen_extend_i, fill_value=0.0 - ) - n.generators.loc[gen_extend_i, "p_nom_extendable"] = False - - stor_units_extend_i = n.storage_units.index[n.storage_units.p_nom_extendable] - n.storage_units.loc[stor_units_extend_i, "p_nom"] = n_optim.storage_units[ - "p_nom_opt" - ].reindex(stor_units_extend_i, fill_value=0.0) - n.storage_units.loc[stor_units_extend_i, "p_nom_extendable"] = False - - stor_extend_i = n.stores.index[n.stores.e_nom_extendable] - n.stores.loc[stor_extend_i, "e_nom"] = n_optim.stores["e_nom_opt"].reindex( - stor_extend_i, fill_value=0.0 - ) - n.stores.loc[stor_extend_i, "e_nom_extendable"] = False - - return n - - -def remove_unused_components(n, threshold=50): - logger.info("Remove assets that are barely used to speed things up.") - - for c in n.iterate_components({"Store", "Link", "Generator"}): - attr = "e_nom" if c.name == "Store" else "p_nom" - to_remove = c.df.loc[c.df[attr] < threshold].index - logger.info(f"Removing barely used {c.name}s:\n{to_remove}") - n.mremove(c.name, to_remove) - - return n - - -def add_load_shedding(n, voll=1e4): - logger.info("Add load shedding to all buses.") - - if "load" in n.generators.carrier.unique(): - to_remove = n.generators.query("carrier == 'load'").index - logger.info(f"Removing pre-existing load shedding:\n{to_remove}") - n.mremove("Generator", to_remove) - - n.madd( - "Generator", - n.buses.index, - suffix=" load", - bus=n.buses.index, - carrier="load", - marginal_cost=voll, - p_nom=1e6, - ) - - return n - - -if __name__ == "__main__": - if "snakemake" not in globals(): - from helper import mock_snakemake - - snakemake = mock_snakemake( - "solve_operations_network", - capacity_year=1952, - simpl="", - opts="", - clusters=37, - lv=2.0, - sector_opts="Co2L0-25H-T-H-B-I-A", - planning_horizons=2030, - weather_year=2013, - ) - - logging.basicConfig( - filename=snakemake.log.python, level=snakemake.config["logging_level"] - ) - - tmpdir = snakemake.config["solving"].get("tmpdir") - if tmpdir is not None: - from pathlib import Path - - Path(tmpdir).mkdir(parents=True, exist_ok=True) - - n = pypsa.Network(snakemake.input.pre) - n_post = pypsa.Network(snakemake.input.post) - n = set_parameters_from_optimized(n, n_post) - del n_post - - n = remove_unused_components(n) - n = add_load_shedding(n) - - opts = snakemake.wildcards.sector_opts.split("-") - solve_opts = snakemake.config["solving"]["options"] - solve_opts["skip_iterations"] = True - - n = prepare_network(n, solve_opts) - - n = solve_network( - n, - config=snakemake.config, - opts=opts, - solver_dir=tmpdir, - solver_logfile=snakemake.log.solver, - ) - - n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/solve_operations_network_other_year_myopic.py b/scripts/solve_operations_network_other_year_myopic.py deleted file mode 100644 index 5858d79f..00000000 --- a/scripts/solve_operations_network_other_year_myopic.py +++ /dev/null @@ -1,166 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors -# -# SPDX-License-Identifier: MIT -""" -Solve myopic operations network. -""" - - -import logging - -import pandas as pd -import pypsa -from solve_network import prepare_network, solve_network -from solve_operations_network import ( - add_load_shedding, - remove_unused_components, - set_parameters_from_optimized, -) - -logger = logging.getLogger(__name__) -pypsa.pf.logger.setLevel(logging.WARNING) - - -def prepare_myopic(n, config, store_soc, storage_unit_soc): - n.stores.e_cyclic = False - n.storage_units.cyclic_state_of_charge = False - - biomass_stores = n.stores.carrier.str.isin(["solid biomass", "biogas"]) - biomass_potential = n.stores.loc[biomass_stores, "e_initial"] - - # storage level contiguity across years - n.stores.e_initial = store_soc - n.storage_units.state_of_charge_initial = storage_unit_soc - - # replace co2 limit with co2 price - n.remove("GlobalConstraint", "CO2Limit") - n.stores.at["co2 atmosphere", "marginal_cost"] = -config["co2_price"] - - # handle co2 sequestration - assert ( - sum(n.stores.carriers == "co2 stored") == 1 - ), "Myopic operation not implemented for spatially resolved CO2 sequestration." - n.stores.at["co2 stored", "e_nom"] = config["co2_sequestration_limit"] * 1e6 # t/a - - # reset co2 emissions - n.stores.loc[n.stores.carrier == "co2 stored", "e_initial"] = 0.0 - n.stores.at["co2 atmosphere", "e_initial"] = 0.0 - - # replenish fossil gas and oil with 1000 TWh each - fossil_stores = n.stores.carrier.str.isin(["gas", "oil"]) - n.stores.loc[fossil_stores, "e_initial"] = 1e9 - n.stores.loc[fossil_stores, "e_nom"] = 10e9 - - # replenish annual solid biomass and biogas potentials - n.stores.loc[biomass_stores, "e_initial"] = biomass_potential - - # set storage bidding prices - bidding_prices = config["bidding_prices"] - for c in n.iterate_components({"Store", "Link", "StorageUnit"}): - c.df.marginal_cost.update(c.df.carrier.map(bidding_prices).dropna()) - - # deduct industry solid biomass - assert ( - sum(n.stores.carriers == "solid biomass") == 1 - ), "Myopic operation not implemented for spatially resolved solid biomass." - n.stores.at["EU solid biomass", "e_initial"] -= ( - n.loads.at["solid biomass for industry", "p_set"] * 8760 - ) - n.remove("Load", "solid biomass for industry") - - return n - - -def solve_network_myopic(n, config, opts="", **kwargs): - rolling_horizon = config["operations"]["rolling_horizon"] - - freq = int(pd.infer_freq(n.snapshots)[:-1]) - window = rolling_horizon["window"] * 24 // freq - overlap = rolling_horizon["overlap"] * 24 // freq - kept = window - overlap - length = len(n.snapshots) - - assert ( - kept > 0 - ), f"Overlap ({overlap} days) must be smaller than windows ({window} days)." - - for i in range(length // kept): - snapshots = n.snapshots[i * kept : (i + 1) * kept + overlap] - logger.info(f"Optimising operations from {snapshots[0]} to {snapshots[-1]}") - - n = solve_network(n, config, opts=opts, snapshots=snapshots, **kwargs) - - last_kept = n.snapshots[(i + 1) * kept - 1] - logger.info(f"Setting initial SOCs from {last_kept} for next iteration.\n") - - n.stores.e_initial = n.stores_t.e.loc[last_kept] - n.storage_units.state_of_charge_initial = n.storage_units_t.state_of_charge.loc[ - last_kept - ] - - # final segment until end of year - snapshots = n.snapshots[(i + 1) * kept :] - n = solve_network(n, config, opts=opts, snapshots=snapshots, **kwargs) - - return n - - -if __name__ == "__main__": - if "snakemake" not in globals(): - from helper import mock_snakemake - - snakemake = mock_snakemake( - "solve_operations_network_myopic", - capacity_year=1952, - simpl="", - opts="", - clusters=37, - lv=2.0, - sector_opts="Co2L0-25H-T-H-B-I-A", - planning_horizons=2030, - weather_year=2013, - ) - - logging.basicConfig( - filename=snakemake.log.python, level=snakemake.config["logging_level"] - ) - - tmpdir = snakemake.config["solving"].get("tmpdir") - if tmpdir is not None: - from pathlib import Path - - Path(tmpdir).mkdir(parents=True, exist_ok=True) - - config = snakemake.config["operations"] - - n = pypsa.Network(snakemake.input.pre) - - n_post = pypsa.Network(snakemake.input.post) - n = set_parameters_from_optimized(n, n_post) - del n_post - - n_previous = pypsa.Network(snakemake.input.previous) - store_soc = n_previous.stores_t.e.iloc[-1] - storage_unit_soc = n_previous.storage_units_t.state_of_charge.iloc[-1] - del n_previous - - n = remove_unused_components(n) - n = add_load_shedding(n) - n = prepare_myopic(n, config, store_soc, storage_unit_soc) - - opts = snakemake.wildcards.sector_opts.split("-") - solve_opts = snakemake.config["solving"]["options"] - solve_opts["skip_iterations"] = True - - n = prepare_network(n, solve_opts) - - n = solve_network_myopic( - n, - config=snakemake.config, - opts=opts, - solver_dir=tmpdir, - solver_logfile=snakemake.log.solver, - ) - - n.export_to_netcdf(snakemake.output[0]) From c2011bce49021a8365954a1a70e6a6f2b0117563 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 4 Mar 2024 17:48:56 +0100 Subject: [PATCH 097/173] remove {weather_year} wildcard --- Snakefile | 1 - config/config.default.yaml | 1 - config/test/config.electricity.yaml | 2 - config/test/config.myopic.yaml | 2 - config/test/config.overnight.yaml | 2 - doc/configuration.rst | 2 +- rules/build_electricity.smk | 90 ++--- rules/build_sector.smk | 330 +++++++++--------- rules/collect.smk | 18 +- rules/common.smk | 2 +- rules/postprocess.smk | 60 ++-- rules/solve_electricity.smk | 20 +- rules/solve_myopic.smk | 46 +-- rules/solve_overnight.smk | 12 +- scripts/add_brownfield.py | 1 - scripts/add_electricity.py | 2 +- scripts/add_existing_baseyear.py | 1 - scripts/add_extra_components.py | 6 +- scripts/build_artificial_load_data.py | 2 +- scripts/build_biomass_potentials.py | 1 - scripts/build_clustered_population_layouts.py | 1 - scripts/build_cop_profiles.py | 1 - scripts/build_daily_heat_demand.py | 1 - scripts/build_electricity_demand.py | 2 +- scripts/build_industrial_distribution_key.py | 1 - ...industrial_energy_demand_per_node_today.py | 1 - scripts/build_population_layouts.py | 1 - ...build_population_weighted_energy_totals.py | 1 - scripts/build_renewable_profiles.py | 2 +- scripts/build_solar_thermal_profiles.py | 1 - scripts/build_transport_demand.py | 1 - scripts/cluster_gas_network.py | 2 +- scripts/cluster_network.py | 22 +- scripts/make_summary.py | 7 +- scripts/prepare_links_p_nom.py | 2 +- scripts/prepare_network.py | 5 +- scripts/prepare_sector_network.py | 1 - scripts/simplify_network.py | 10 +- scripts/solve_network.py | 1 - scripts/solve_operations_network.py | 1 - 40 files changed, 320 insertions(+), 345 deletions(-) diff --git a/Snakefile b/Snakefile index f1bd459f..78a28373 100644 --- a/Snakefile +++ b/Snakefile @@ -56,7 +56,6 @@ localrules: wildcard_constraints: - weather_year="[0-9]{4}|", simpl="[a-zA-Z0-9]*", clusters="[0-9]+(m|c)?|all", ll="(v|c)([0-9\.]+|opt)", diff --git a/config/config.default.yaml b/config/config.default.yaml index 892a1e74..a9166560 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -34,7 +34,6 @@ foresight: overnight # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#scenario # Wildcard docs in https://pypsa-eur.readthedocs.io/en/latest/wildcards.html scenario: - weather_year: [''] # for backwards compatibility simpl: - '' ll: diff --git a/config/test/config.electricity.yaml b/config/test/config.electricity.yaml index 95405878..57964415 100644 --- a/config/test/config.electricity.yaml +++ b/config/test/config.electricity.yaml @@ -12,8 +12,6 @@ run: shared_cutouts: true scenario: - weather_year: - - "" clusters: - 5 opts: diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index 1157a7c6..5abae36d 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -13,8 +13,6 @@ run: foresight: myopic scenario: - weather_year: - - "" ll: - v1.5 clusters: diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index 0e2b6d6b..7fb53e42 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -12,8 +12,6 @@ run: scenario: - weather_year: - - "" ll: - v1.5 clusters: diff --git a/doc/configuration.rst b/doc/configuration.rst index 9eba4284..dae91380 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -87,7 +87,7 @@ facilitate running multiple scenarios through a single command For each wildcard, a **list of values** is provided. The rule ``solve_all_elec_networks`` will trigger the rules for creating -``results/networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc`` for **all +``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc`` for **all combinations** of the provided wildcard values as defined by Python's `itertools.product(...) `_ function diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index a83a9d93..53c2df0b 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -28,9 +28,9 @@ if config["enable"].get("retrieve_opsd_load_data", True): input: ancient("data/electricity_demand_raw.csv"), output: - resources("electricity_demand{weather_year}.csv"), + resources("electricity_demand.csv"), log: - logs("build_electricity_demand{weather_year}.log"), + logs("build_electricity_demand.log"), resources: mem_mb=5000, conda: @@ -45,9 +45,9 @@ if config["enable"].get("retrieve_artificial_load_data", False): input: ancient("data/load_artificial_raw.csv"), output: - resources("electricity_demand{weather_year}.csv"), + resources("electricity_demand.csv"), log: - logs("build_artificial_load_data{weather_year}.log"), + logs("build_artificial_load_data.log"), resources: mem_mb=5000, conda: @@ -192,11 +192,11 @@ if config["enable"].get("build_cutout", False): input: rules.build_cutout.input, output: - protected("cutouts/" + CDIR + "{cutout}-{weather_year}.nc"), + protected("cutouts/" + CDIR + "{cutout}-.nc"), log: - logs(CDIR + "build_cutout/{cutout}-{weather_year}.log"), + logs(CDIR + "build_cutout/{cutout}-.log"), benchmark: - benchmarks(CDIR + "build_cutout_{cutout}-{weather_year}") + benchmarks(CDIR + "build_cutout_{cutout}-") threads: ATLITE_NPROCESSES resources: mem_mb=ATLITE_NPROCESSES * 1000, @@ -342,11 +342,11 @@ rule build_renewable_profiles: + config_provider("renewable", w.technology, "cutout")(w) + ".nc", output: - profile=resources("profile{weather_year}_{technology}.nc"), + profile=resources("profile_{technology}.nc"), log: - logs("build_renewable_profile{weather_year}_{technology}.log"), + logs("build_renewable_profile_{technology}.log"), benchmark: - benchmarks("build_renewable_profiles{weather_year}_{technology}") + benchmarks("build_renewable_profiles_{technology}") threads: config["atlite"].get("nprocesses", 4) resources: mem_mb=config["atlite"].get("nprocesses", 4) * 5000, @@ -390,10 +390,10 @@ rule build_hydro_profile: + config_provider("renewable", "hydro", "cutout")(w) + ".nc", output: - profile=resources("profile{weather_year}_hydro.nc"), - eia_hydro=resources("eia_hydro_stats{weather_year}.csv"), + profile=resources("profile_hydro.nc"), + eia_hydro=resources("eia_hydro_stats.csv"), log: - logs("build_hydro_profile{weather_year}.log"), + logs("build_hydro_profile.log"), resources: mem_mb=5000, conda: @@ -428,7 +428,7 @@ rule build_line_rating: def input_profile_tech(w): return { - f"profile_{tech}": resources(f"profile{{weather_year}}_{tech}.nc") + f"profile_{tech}": resources(f"profile{}_{tech}.nc") for tech in config_provider("electricity", "renewable_carriers")(w) } @@ -475,15 +475,15 @@ rule add_electricity: if config_provider("conventional", "dynamic_fuel_price")(w) else [] ), - load=resources("electricity_demand{weather_year}.csv"), + load=resources("electricity_demand.csv"), nuts3_shapes=resources("nuts3_shapes.geojson"), ua_md_gdp="data/GDP_PPP_30arcsec_v3_mapped_default.csv", output: - resources("networks/elec{weather_year}.nc"), + resources("networks/elec.nc"), log: - logs("add_electricity{weather_year}.log"), + logs("add_electricity.log"), benchmark: - benchmarks("add_electricity{weather_year}") + benchmarks("add_electricity") threads: 1 resources: mem_mb=10000, @@ -506,24 +506,24 @@ rule simplify_network: p_max_pu=config_provider("links", "p_max_pu", default=1.0), costs=config_provider("costs"), input: - network=resources("networks/elec{weather_year}.nc"), + network=resources("networks/elec.nc"), tech_costs=lambda w: resources( f"costs_{config_provider('costs', 'year')(w)}.csv" ), regions_onshore=resources("regions_onshore.geojson"), regions_offshore=resources("regions_offshore.geojson"), output: - network=resources("networks/elec{weather_year}_s{simpl}.nc"), - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}.geojson"), + network=resources("networks/elec_s{simpl}.nc"), + regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"), regions_offshore=resources( - "regions_offshore_elec{weather_year}_s{simpl}.geojson" + "regions_offshore_elec_s{simpl}.geojson" ), - busmap=resources("busmap_elec{weather_year}_s{simpl}.csv"), - connection_costs=resources("connection_costs{weather_year}_s{simpl}.csv"), + busmap=resources("busmap_elec_s{simpl}.csv"), + connection_costs=resources("connection_costs_s{simpl}.csv"), log: - logs("simplify_network/elec{weather_year}_s{simpl}.log"), + logs("simplify_network/elec_s{simpl}.log"), benchmark: - benchmarks("simplify_network/elec{weather_year}_s{simpl}") + benchmarks("simplify_network/elec_s{simpl}") threads: 1 resources: mem_mb=12000, @@ -549,12 +549,12 @@ rule cluster_network: length_factor=config_provider("lines", "length_factor"), costs=config_provider("costs"), input: - network=resources("networks/elec{weather_year}_s{simpl}.nc"), - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}.geojson"), + network=resources("networks/elec_s{simpl}.nc"), + regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"), regions_offshore=resources( - "regions_offshore_elec{weather_year}_s{simpl}.geojson" + "regions_offshore_elec_s{simpl}.geojson" ), - busmap=ancient(resources("busmap_elec{weather_year}_s{simpl}.csv")), + busmap=ancient(resources("busmap_elec_s{simpl}.csv")), custom_busmap=lambda w: ( "data/custom_busmap_elec_s{simpl}_{clusters}.csv" if config_provider("enable", "custom_busmap", default=False)(w) @@ -564,19 +564,19 @@ rule cluster_network: f"costs_{config_provider('costs', 'year')(w)}.csv" ), output: - network=resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"), + network=resources("networks/elec_s{simpl}_{clusters}.nc"), regions_onshore=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), regions_offshore=resources( - "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_offshore_elec_s{simpl}_{clusters}.geojson" ), - busmap=resources("busmap_elec{weather_year}_s{simpl}_{clusters}.csv"), - linemap=resources("linemap_elec{weather_year}_s{simpl}_{clusters}.csv"), + busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), + linemap=resources("linemap_elec_s{simpl}_{clusters}.csv"), log: - logs("cluster_network/elec{weather_year}_s{simpl}_{clusters}.log"), + logs("cluster_network/elec_s{simpl}_{clusters}.log"), benchmark: - benchmarks("cluster_network/elec{weather_year}_s{simpl}_{clusters}") + benchmarks("cluster_network/elec_s{simpl}_{clusters}") threads: 1 resources: mem_mb=10000, @@ -592,16 +592,16 @@ rule add_extra_components: max_hours=config_provider("electricity", "max_hours"), costs=config_provider("costs"), input: - network=resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"), + network=resources("networks/elec_s{simpl}_{clusters}.nc"), tech_costs=lambda w: resources( f"costs_{config_provider('costs', 'year')(w)}.csv" ), output: - resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc"), + resources("networks/elec_s{simpl}_{clusters}_ec.nc"), log: - logs("add_extra_components/elec{weather_year}_s{simpl}_{clusters}.log"), + logs("add_extra_components/elec_s{simpl}_{clusters}.log"), benchmark: - benchmarks("add_extra_components/elec{weather_year}_s{simpl}_{clusters}_ec") + benchmarks("add_extra_components/elec_s{simpl}_{clusters}_ec") threads: 1 resources: mem_mb=4000, @@ -626,21 +626,21 @@ rule prepare_network: adjustments=config_provider("adjustments", "electricity"), autarky=config_provider("electricity", "autarky", default={}), input: - resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc"), + resources("networks/elec_s{simpl}_{clusters}_ec.nc"), tech_costs=lambda w: resources( f"costs_{config_provider('costs', 'year')(w)}.csv" ), co2_price=lambda w: resources("co2_price.csv") if "Ept" in w.opts else [], output: - resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), + resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), log: logs( - "prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" + "prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" ), benchmark: ( benchmarks( - "prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" + "prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" ) ) threads: 1 diff --git a/rules/build_sector.smk b/rules/build_sector.smk index b4e41df8..ab296565 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -12,15 +12,15 @@ rule build_population_layouts: + config_provider("atlite", "default_cutout")(w) + ".nc", output: - pop_layout_total=resources("pop_layout_total{weather_year}.nc"), - pop_layout_urban=resources("pop_layout_urban{weather_year}.nc"), - pop_layout_rural=resources("pop_layout_rural{weather_year}.nc"), + pop_layout_total=resources("pop_layout_total.nc"), + pop_layout_urban=resources("pop_layout_urban.nc"), + pop_layout_rural=resources("pop_layout_rural.nc"), log: - logs("build_population_layouts{weather_year}.log"), + logs("build_population_layouts.log"), resources: mem_mb=20000, benchmark: - benchmarks("build_population_layouts{weather_year}") + benchmarks("build_population_layouts") threads: 8 conda: "../envs/environment.yaml" @@ -30,11 +30,11 @@ rule build_population_layouts: rule build_clustered_population_layouts: input: - pop_layout_total=resources("pop_layout_total{weather_year}.nc"), - pop_layout_urban=resources("pop_layout_urban{weather_year}.nc"), - pop_layout_rural=resources("pop_layout_rural{weather_year}.nc"), + pop_layout_total=resources("pop_layout_total.nc"), + pop_layout_urban=resources("pop_layout_urban.nc"), + pop_layout_rural=resources("pop_layout_rural.nc"), regions_onshore=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), cutout=lambda w: "cutouts/" + CDIR @@ -42,15 +42,15 @@ rule build_clustered_population_layouts: + ".nc", output: clustered_pop_layout=resources( - "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + "pop_layout_elec_s{simpl}_{clusters}.csv" ), log: - logs("build_clustered_population_layouts{weather_year}_{simpl}_{clusters}.log"), + logs("build_clustered_population_layouts_{simpl}_{clusters}.log"), resources: mem_mb=10000, benchmark: benchmarks( - "build_clustered_population_layouts/{weather_year}s{simpl}_{clusters}" + "build_clustered_population_layouts/s{simpl}_{clusters}" ) conda: "../envs/environment.yaml" @@ -60,22 +60,22 @@ rule build_clustered_population_layouts: rule build_simplified_population_layouts: input: - pop_layout_total=resources("pop_layout_total{weather_year}.nc"), - pop_layout_urban=resources("pop_layout_urban{weather_year}.nc"), - pop_layout_rural=resources("pop_layout_rural{weather_year}.nc"), - regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}.geojson"), + pop_layout_total=resources("pop_layout_total.nc"), + pop_layout_urban=resources("pop_layout_urban.nc"), + pop_layout_rural=resources("pop_layout_rural.nc"), + regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"), cutout=lambda w: "cutouts/" + CDIR + config_provider("atlite", "default_cutout")(w) + ".nc", output: - clustered_pop_layout=resources("pop_layout_elec{weather_year}_s{simpl}.csv"), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}.csv"), resources: mem_mb=10000, log: - logs("build_simplified_population_layouts{weather_year}_{simpl}"), + logs("build_simplified_population_layouts_{simpl}"), benchmark: - benchmarks("build_simplified_population_layouts/{weather_year}s{simpl}") + benchmarks("build_simplified_population_layouts/s{simpl}") conda: "../envs/environment.yaml" script: @@ -106,22 +106,22 @@ rule build_gas_input_locations: entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson", storage="data/gas_network/scigrid-gas/data/IGGIELGN_Storages.geojson", regions_onshore=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), regions_offshore=resources( - "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_offshore_elec_s{simpl}_{clusters}.geojson" ), output: gas_input_nodes=resources( - "gas_input_locations{weather_year}_s{simpl}_{clusters}.geojson" + "gas_input_locations_s{simpl}_{clusters}.geojson" ), gas_input_nodes_simplified=resources( - "gas_input_locations{weather_year}_s{simpl}_{clusters}_simplified.csv" + "gas_input_locations_s{simpl}_{clusters}_simplified.csv" ), resources: mem_mb=2000, log: - logs("build_gas_input_locations{weather_year}_s{simpl}_{clusters}.log"), + logs("build_gas_input_locations_s{simpl}_{clusters}.log"), conda: "../envs/environment.yaml" script: @@ -132,19 +132,19 @@ rule cluster_gas_network: input: cleaned_gas_network=resources("gas_network.csv"), regions_onshore=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), regions_offshore=resources( - "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_offshore_elec_s{simpl}_{clusters}.geojson" ), output: clustered_gas_network=resources( - "gas_network_elec{weather_year}_s{simpl}_{clusters}.csv" + "gas_network_elec_s{simpl}_{clusters}.csv" ), resources: mem_mb=4000, log: - logs("cluster_gas_network{weather_year}_s{simpl}_{clusters}.log"), + logs("cluster_gas_network_s{simpl}_{clusters}.log"), conda: "../envs/environment.yaml" script: @@ -169,22 +169,22 @@ rule build_daily_heat_demand: snapshots=config_provider("snapshots"), drop_leap_day=config_provider("enable", "drop_leap_day"), input: - pop_layout=resources("pop_layout_{scope}{weather_year}.nc"), + pop_layout=resources("pop_layout_{scope}.nc"), regions_onshore=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), cutout=heat_demand_cutout, output: heat_demand=resources( - "daily_heat_demand_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc" + "daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc" ), resources: mem_mb=20000, threads: 8 log: - logs("build_daily_heat_demand_{scope}_{weather_year}_{simpl}_{clusters}.loc"), + logs("build_daily_heat_demand_{scope}__{simpl}_{clusters}.loc"), benchmark: - benchmarks("build_daily_heat_demand/{scope}_{weather_year}_s{simpl}_{clusters}") + benchmarks("build_daily_heat_demand/{scope}__s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -198,19 +198,19 @@ rule build_hourly_heat_demand: input: heat_profile="data/heat_load_profile_BDEW.csv", heat_demand=resources( - "daily_heat_demand_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc" + "daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc" ), output: heat_demand=resources( - "hourly_heat_demand_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc" + "hourly_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc" ), resources: mem_mb=2000, threads: 8 log: - logs("build_hourly_heat_demand_{scope}{weather_year}_{simpl}_{clusters}.loc"), + logs("build_hourly_heat_demand_{scope}_{simpl}_{clusters}.loc"), benchmark: - benchmarks("build_hourly_heat_demand/{scope}{weather_year}_s{simpl}_{clusters}") + benchmarks("build_hourly_heat_demand/{scope}_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -222,24 +222,24 @@ rule build_temperature_profiles: snapshots=config_provider("snapshots"), drop_leap_day=config_provider("enable", "drop_leap_day"), input: - pop_layout=resources("pop_layout_{scope}{weather_year}.nc"), + pop_layout=resources("pop_layout_{scope}.nc"), regions_onshore=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), cutout=heat_demand_cutout, output: temp_soil=resources( - "temp_soil_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_soil_{scope}_elec_s{simpl}_{clusters}.nc" ), - temp_air=resources("temp_air_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc"), + temp_air=resources("temp_air_{scope}_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, threads: 8 log: - logs("build_temperature_profiles_{scope}_{weather_year}_{simpl}_{clusters}.log"), + logs("build_temperature_profiles_{scope}__{simpl}_{clusters}.log"), benchmark: benchmarks( - "build_temperature_profiles/{scope}_{weather_year}_s{simpl}_{clusters}" + "build_temperature_profiles/{scope}__s{simpl}_{clusters}" ) conda: "../envs/environment.yaml" @@ -252,48 +252,48 @@ rule build_cop_profiles: heat_pump_sink_T=config_provider("sector", "heat_pump_sink_T"), input: temp_soil_total=resources( - "temp_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_soil_total_elec_s{simpl}_{clusters}.nc" ), temp_soil_rural=resources( - "temp_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_soil_rural_elec_s{simpl}_{clusters}.nc" ), temp_soil_urban=resources( - "temp_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_soil_urban_elec_s{simpl}_{clusters}.nc" ), temp_air_total=resources( - "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_air_total_elec_s{simpl}_{clusters}.nc" ), temp_air_rural=resources( - "temp_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_air_rural_elec_s{simpl}_{clusters}.nc" ), temp_air_urban=resources( - "temp_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_air_urban_elec_s{simpl}_{clusters}.nc" ), output: cop_soil_total=resources( - "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_soil_total_elec_s{simpl}_{clusters}.nc" ), cop_soil_rural=resources( - "cop_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_soil_rural_elec_s{simpl}_{clusters}.nc" ), cop_soil_urban=resources( - "cop_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_soil_urban_elec_s{simpl}_{clusters}.nc" ), cop_air_total=resources( - "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_air_total_elec_s{simpl}_{clusters}.nc" ), cop_air_rural=resources( - "cop_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_air_rural_elec_s{simpl}_{clusters}.nc" ), cop_air_urban=resources( - "cop_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_air_urban_elec_s{simpl}_{clusters}.nc" ), resources: mem_mb=20000, log: - logs("build_cop_profiles{weather_year}_s{simpl}_{clusters}.log"), + logs("build_cop_profiles_s{simpl}_{clusters}.log"), benchmark: - benchmarks("build_cop_profiles/{weather_year}s{simpl}_{clusters}") + benchmarks("build_cop_profiles/s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -319,25 +319,25 @@ rule build_solar_thermal_profiles: drop_leap_day=config_provider("enable", "drop_leap_day"), solar_thermal=config_provider("solar_thermal"), input: - pop_layout=resources("pop_layout_{scope}{weather_year}.nc"), + pop_layout=resources("pop_layout_{scope}.nc"), regions_onshore=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), cutout=solar_thermal_cutout, output: solar_thermal=resources( - "solar_thermal_{scope}_elec{weather_year}_s{simpl}_{clusters}.nc" + "solar_thermal_{scope}_elec_s{simpl}_{clusters}.nc" ), resources: mem_mb=20000, threads: 16 log: logs( - "build_solar_thermal_profiles_{scope}_{weather_year}_s{simpl}_{clusters}.log" + "build_solar_thermal_profiles_{scope}__s{simpl}_{clusters}.log" ), benchmark: benchmarks( - "build_solar_thermal_profiles/{scope}_{weather_year}_s{simpl}_{clusters}" + "build_solar_thermal_profiles/{scope}__s{simpl}_{clusters}" ) conda: "../envs/environment.yaml" @@ -403,7 +403,7 @@ rule build_biomass_potentials: ), nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 regions_onshore=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"), swiss_cantons=ancient("data/bundle/ch_cantons.csv"), @@ -411,21 +411,21 @@ rule build_biomass_potentials: country_shapes=resources("country_shapes.geojson"), output: biomass_potentials_all=resources( - "biomass_potentials_all{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" + "biomass_potentials_all_s{simpl}_{clusters}_{planning_horizons}.csv" ), biomass_potentials=resources( - "biomass_potentials{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" + "biomass_potentials_s{simpl}_{clusters}_{planning_horizons}.csv" ), threads: 1 resources: mem_mb=1000, log: logs( - "build_biomass_potentials{weather_year}_s{simpl}_{clusters}_{planning_horizons}.log" + "build_biomass_potentials_s{simpl}_{clusters}_{planning_horizons}.log" ), benchmark: benchmarks( - "build_biomass_potentials{weather_year}_s{simpl}_{clusters}_{planning_horizons}" + "build_biomass_potentials_s{simpl}_{clusters}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -465,22 +465,22 @@ rule build_sequestration_potentials: keep_local=True, ), regions_onshore=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), regions_offshore=resources( - "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_offshore_elec_s{simpl}_{clusters}.geojson" ), output: sequestration_potential=resources( - "co2_sequestration_potential_elec{weather_year}_s{simpl}_{clusters}.csv" + "co2_sequestration_potential_elec_s{simpl}_{clusters}.csv" ), threads: 1 resources: mem_mb=4000, log: - logs("build_sequestration_potentials{weather_year}_s{simpl}_{clusters}.log"), + logs("build_sequestration_potentials_s{simpl}_{clusters}.log"), benchmark: - benchmarks("build_sequestration_potentials{weather_year}_s{simpl}_{clusters}") + benchmarks("build_sequestration_potentials_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -491,22 +491,22 @@ rule build_salt_cavern_potentials: input: salt_caverns="data/bundle-sector/h2_salt_caverns_GWh_per_sqkm.geojson", regions_onshore=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), regions_offshore=resources( - "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_offshore_elec_s{simpl}_{clusters}.geojson" ), output: h2_cavern_potential=resources( - "salt_cavern_potentials{weather_year}_s{simpl}_{clusters}.csv" + "salt_cavern_potentials_s{simpl}_{clusters}.csv" ), threads: 1 resources: mem_mb=2000, log: - logs("build_salt_cavern_potentials{weather_year}_s{simpl}_{clusters}.log"), + logs("build_salt_cavern_potentials_s{simpl}_{clusters}.log"), benchmark: - benchmarks("build_salt_cavern_potentials{weather_year}_s{simpl}_{clusters}") + benchmarks("build_salt_cavern_potentials_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -642,24 +642,24 @@ rule build_industrial_distribution_key: countries=config_provider("countries"), input: regions_onshore=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), clustered_pop_layout=resources( - "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + "pop_layout_elec_s{simpl}_{clusters}.csv" ), hotmaps_industrial_database="data/bundle-sector/Industrial_Database.csv", output: industrial_distribution_key=resources( - "industrial_distribution_key_elec{weather_year}_s{simpl}_{clusters}.csv" + "industrial_distribution_key_elec_s{simpl}_{clusters}.csv" ), threads: 1 resources: mem_mb=1000, log: - logs("build_industrial_distribution_key{weather_year}_s{simpl}_{clusters}.log"), + logs("build_industrial_distribution_key_s{simpl}_{clusters}.log"), benchmark: benchmarks( - "build_industrial_distribution_key/{weather_year}s{simpl}_{clusters}" + "build_industrial_distribution_key/s{simpl}_{clusters}" ) conda: "../envs/environment.yaml" @@ -670,26 +670,26 @@ rule build_industrial_distribution_key: rule build_industrial_production_per_node: input: industrial_distribution_key=resources( - "industrial_distribution_key_elec{weather_year}_s{simpl}_{clusters}.csv" + "industrial_distribution_key_elec_s{simpl}_{clusters}.csv" ), industrial_production_per_country_tomorrow=resources( "industrial_production_per_country_tomorrow_{planning_horizons}.csv" ), output: industrial_production_per_node=resources( - "industrial_production_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" + "industrial_production_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), threads: 1 resources: mem_mb=1000, log: logs( - "build_industrial_production_per_node{weather_year}_s{simpl}_{clusters}_{planning_horizons}.log" + "build_industrial_production_per_node_s{simpl}_{clusters}_{planning_horizons}.log" ), benchmark: ( benchmarks( - "build_industrial_production_per_node/{weather_year}s{simpl}_{clusters}_{planning_horizons}" + "build_industrial_production_per_node/s{simpl}_{clusters}_{planning_horizons}" ) ) conda: @@ -704,26 +704,26 @@ rule build_industrial_energy_demand_per_node: "industry_sector_ratios_{planning_horizons}.csv" ), industrial_production_per_node=resources( - "industrial_production_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" + "industrial_production_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), industrial_energy_demand_per_node_today=resources( - "industrial_energy_demand_today_elec{weather_year}_s{simpl}_{clusters}.csv" + "industrial_energy_demand_today_elec_s{simpl}_{clusters}.csv" ), output: industrial_energy_demand_per_node=resources( - "industrial_energy_demand_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" + "industrial_energy_demand_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), threads: 1 resources: mem_mb=1000, log: logs( - "build_industrial_energy_demand_per_node{weather_year}_s{simpl}_{clusters}_{planning_horizons}.log" + "build_industrial_energy_demand_per_node_s{simpl}_{clusters}_{planning_horizons}.log" ), benchmark: ( benchmarks( - "build_industrial_energy_demand_per_node/{weather_year}s{simpl}_{clusters}_{planning_horizons}" + "build_industrial_energy_demand_per_node/s{simpl}_{clusters}_{planning_horizons}" ) ) conda: @@ -761,25 +761,25 @@ rule build_industrial_energy_demand_per_country_today: rule build_industrial_energy_demand_per_node_today: input: industrial_distribution_key=resources( - "industrial_distribution_key_elec{weather_year}_s{simpl}_{clusters}.csv" + "industrial_distribution_key_elec_s{simpl}_{clusters}.csv" ), industrial_energy_demand_per_country_today=resources( "industrial_energy_demand_per_country_today.csv" ), output: industrial_energy_demand_per_node_today=resources( - "industrial_energy_demand_today_elec{weather_year}_s{simpl}_{clusters}.csv" + "industrial_energy_demand_today_elec_s{simpl}_{clusters}.csv" ), threads: 1 resources: mem_mb=1000, log: logs( - "build_industrial_energy_demand_per_node_today{weather_year}_s{simpl}_{clusters}.log" + "build_industrial_energy_demand_per_node_today_s{simpl}_{clusters}.log" ), benchmark: benchmarks( - "build_industrial_energy_demand_per_node_today/{weather_year}s{simpl}_{clusters}" + "build_industrial_energy_demand_per_node_today/s{simpl}_{clusters}" ) conda: "../envs/environment.yaml" @@ -795,26 +795,26 @@ rule build_retro_cost: building_stock="data/retro/data_building_stock.csv", data_tabula="data/bundle-sector/retro/tabula-calculator-calcsetbuilding.csv", air_temperature=resources( - "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_air_total_elec_s{simpl}_{clusters}.nc" ), u_values_PL="data/retro/u_values_poland.csv", - tax_w="data/retro/elec{weather_year}tricity_taxes_eu.csv", + tax_w="data/retro/electricity_taxes_eu.csv", construction_index="data/retro/comparative_level_investment.csv", floor_area_missing="data/retro/floor_area_missing.csv", clustered_pop_layout=resources( - "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + "pop_layout_elec_s{simpl}_{clusters}.csv" ), cost_germany="data/retro/retro_cost_germany.csv", window_assumptions="data/retro/window_assumptions.csv", output: - retro_cost=resources("retro_cost_elec{weather_year}_s{simpl}_{clusters}.csv"), - floor_area=resources("floor_area_elec{weather_year}_s{simpl}_{clusters}.csv"), + retro_cost=resources("retro_cost_elec_s{simpl}_{clusters}.csv"), + floor_area=resources("floor_area_elec_s{simpl}_{clusters}.csv"), resources: mem_mb=1000, log: - logs("build_retro_cost{weather_year}_s{simpl}_{clusters}.log"), + logs("build_retro_cost_s{simpl}_{clusters}.log"), benchmark: - benchmarks("build_retro_cost/{weather_year}s{simpl}_{clusters}") + benchmarks("build_retro_cost/s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -825,16 +825,16 @@ rule build_population_weighted_energy_totals: input: energy_totals=resources("{kind}_totals.csv"), clustered_pop_layout=resources( - "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + "pop_layout_elec_s{simpl}_{clusters}.csv" ), output: - resources("pop_weighted_{kind}_totals{weather_year}_s{simpl}_{clusters}.csv"), + resources("pop_weighted_{kind}_totals_s{simpl}_{clusters}.csv"), threads: 1 resources: mem_mb=2000, log: logs( - "build_population_weighted_{kind}_totals{weather_year}_s{simpl}_{clusters}.log" + "build_population_weighted_{kind}_totals_s{simpl}_{clusters}.log" ), conda: "../envs/environment.yaml" @@ -847,18 +847,18 @@ rule build_shipping_demand: ports="data/attributed_ports.json", scope=resources("europe_shape.geojson"), regions=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), demand=resources("energy_totals.csv"), params: energy_totals_year=config_provider("energy", "energy_totals_year"), output: - resources("shipping_demand{weather_year}_s{simpl}_{clusters}.csv"), + resources("shipping_demand_s{simpl}_{clusters}.csv"), threads: 1 resources: mem_mb=2000, log: - logs("build_shipping_demand{weather_year}_s{simpl}_{clusters}.log"), + logs("build_shipping_demand_s{simpl}_{clusters}.log"), conda: "../envs/environment.yaml" script: @@ -872,29 +872,29 @@ rule build_transport_demand: sector=config_provider("sector"), input: clustered_pop_layout=resources( - "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + "pop_layout_elec_s{simpl}_{clusters}.csv" ), pop_weighted_energy_totals=resources( - "pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv" + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv" ), transport_data=resources("transport_data.csv"), traffic_data_KFZ="data/bundle-sector/emobility/KFZ__count", traffic_data_Pkw="data/bundle-sector/emobility/Pkw__count", temp_air_total=resources( - "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_air_total_elec_s{simpl}_{clusters}.nc" ), output: transport_demand=resources( - "transport_demand{weather_year}_s{simpl}_{clusters}.csv" + "transport_demand_s{simpl}_{clusters}.csv" ), - transport_data=resources("transport_data{weather_year}_s{simpl}_{clusters}.csv"), - avail_profile=resources("avail_profile{weather_year}_s{simpl}_{clusters}.csv"), - dsm_profile=resources("dsm_profile{weather_year}_s{simpl}_{clusters}.csv"), + transport_data=resources("transport_data_s{simpl}_{clusters}.csv"), + avail_profile=resources("avail_profile_s{simpl}_{clusters}.csv"), + dsm_profile=resources("dsm_profile_s{simpl}_{clusters}.csv"), threads: 1 resources: mem_mb=2000, log: - logs("build_transport_demand{weather_year}_s{simpl}_{clusters}.log"), + logs("build_transport_demand_s{simpl}_{clusters}.log"), conda: "../envs/environment.yaml" script: @@ -907,18 +907,18 @@ rule build_district_heat_share: input: district_heat_share=resources("district_heat_share.csv"), clustered_pop_layout=resources( - "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + "pop_layout_elec_s{simpl}_{clusters}.csv" ), output: district_heat_share=resources( - "district_heat_share_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" + "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), threads: 1 resources: mem_mb=1000, log: logs( - "build_district_heat_share{weather_year}_s{simpl}_{clusters}_{planning_horizons}.log" + "build_district_heat_share_s{simpl}_{clusters}_{planning_horizons}.log" ), conda: "../envs/environment.yaml" @@ -934,28 +934,28 @@ rule build_existing_heating_distribution: input: existing_heating="data/existing_infrastructure/existing_heating_raw.csv", clustered_pop_layout=resources( - "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + "pop_layout_elec_s{simpl}_{clusters}.csv" ), clustered_pop_energy_layout=resources( - "pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv" + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv" ), district_heat_share=resources( - "district_heat_share_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" + "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), output: existing_heating_distribution=resources( - "existing_heating_distribution_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" + "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), threads: 1 resources: mem_mb=2000, log: logs( - "build_existing_heating_distribution_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.log" + "build_existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.log" ), benchmark: benchmarks( - "build_existing_heating_distribution/elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}" + "build_existing_heating_distribution/elec_s{simpl}_{clusters}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -965,7 +965,7 @@ rule build_existing_heating_distribution: def input_profile_offwind(w): return { - f"profile_{tech}": resources(f"profile{{weather_year}}_{tech}.nc") + f"profile_{tech}": resources(f"profile{}_{tech}.nc") for tech in ["offwind-ac", "offwind-dc"] if (tech in config_provider("electricity", "renewable_carriers")(w)) } @@ -997,12 +997,12 @@ rule prepare_sector_network: **rules.cluster_gas_network.output, **rules.build_gas_input_locations.output, retro_cost=lambda w: ( - resources("retro_cost_elec{weather_year}_s{simpl}_{clusters}.csv") + resources("retro_cost_elec_s{simpl}_{clusters}.csv") if config_provider("sector", "retrofitting", "retro_endogen")(w) else [] ), floor_area=lambda w: ( - resources("floor_area_elec{weather_year}_s{simpl}_{clusters}.csv") + resources("floor_area_elec_s{simpl}_{clusters}.csv") if config_provider("sector", "retrofitting", "retro_endogen")(w) else [] ), @@ -1014,7 +1014,7 @@ rule prepare_sector_network: ), sequestration_potential=lambda w: ( resources( - "co2_sequestration_potential_elec{weather_year}_s{simpl}_{clusters}.csv" + "co2_sequestration_potential_elec_s{simpl}_{clusters}.csv" ) if config_provider( "sector", "regional_co2_sequestration_potential", "enable" @@ -1022,35 +1022,35 @@ rule prepare_sector_network: else [] ), network=resources( - "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" ), energy_totals_name=resources("energy_totals.csv"), eurostat=input_eurostat, pop_weighted_energy_totals=resources( - "pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv" + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv" ), pop_weighted_heat_totals=resources( - "pop_weighted_heat_totals{weather_year}_s{simpl}_{clusters}.csv" + "pop_weighted_heat_totals_s{simpl}_{clusters}.csv" ), shipping_demand=resources( - "shipping_demand{weather_year}_s{simpl}_{clusters}.csv" + "shipping_demand_s{simpl}_{clusters}.csv" ), transport_demand=resources( - "transport_demand{weather_year}_s{simpl}_{clusters}.csv" + "transport_demand_s{simpl}_{clusters}.csv" ), - transport_data=resources("transport_data{weather_year}_s{simpl}_{clusters}.csv"), - avail_profile=resources("avail_profile{weather_year}_s{simpl}_{clusters}.csv"), - dsm_profile=resources("dsm_profile{weather_year}_s{simpl}_{clusters}.csv"), + transport_data=resources("transport_data_s{simpl}_{clusters}.csv"), + avail_profile=resources("avail_profile_s{simpl}_{clusters}.csv"), + dsm_profile=resources("dsm_profile_s{simpl}_{clusters}.csv"), co2_totals_name=resources("co2_totals.csv"), co2="data/bundle-sector/eea/UNFCCC_v23.csv", biomass_potentials=lambda w: ( resources( - "biomass_potentials{weather_year}_s{simpl}_{clusters}_" + "biomass_potentials_s{simpl}_{clusters}_" + "{}.csv".format(config_provider("biomass", "year")(w)) ) if config_provider("foresight")(w) == "overnight" else resources( - "biomass_potentials{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" + "biomass_potentials_s{simpl}_{clusters}_{planning_horizons}.csv" ) ), costs=lambda w: ( @@ -1059,87 +1059,87 @@ rule prepare_sector_network: else resources("costs_{planning_horizons}.csv") ), h2_cavern=resources( - "salt_cavern_potentials{weather_year}_s{simpl}_{clusters}.csv" + "salt_cavern_potentials_s{simpl}_{clusters}.csv" ), - busmap_s=resources("busmap_elec{weather_year}_s{simpl}.csv"), - busmap=resources("busmap_elec{weather_year}_s{simpl}_{clusters}.csv"), + busmap_s=resources("busmap_elec_s{simpl}.csv"), + busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), clustered_pop_layout=resources( - "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + "pop_layout_elec_s{simpl}_{clusters}.csv" ), - simplified_pop_layout=resources("pop_layout_elec{weather_year}_s{simpl}.csv"), + simplified_pop_layout=resources("pop_layout_elec_s{simpl}.csv"), industrial_demand=resources( - "industrial_energy_demand_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" + "industrial_energy_demand_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), hourly_heat_demand_total=resources( - "hourly_heat_demand_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "hourly_heat_demand_total_elec_s{simpl}_{clusters}.nc" ), district_heat_share=resources( - "district_heat_share_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" + "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), temp_soil_total=resources( - "temp_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_soil_total_elec_s{simpl}_{clusters}.nc" ), temp_soil_rural=resources( - "temp_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_soil_rural_elec_s{simpl}_{clusters}.nc" ), temp_soil_urban=resources( - "temp_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_soil_urban_elec_s{simpl}_{clusters}.nc" ), temp_air_total=resources( - "temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_air_total_elec_s{simpl}_{clusters}.nc" ), temp_air_rural=resources( - "temp_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_air_rural_elec_s{simpl}_{clusters}.nc" ), temp_air_urban=resources( - "temp_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + "temp_air_urban_elec_s{simpl}_{clusters}.nc" ), cop_soil_total=resources( - "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_soil_total_elec_s{simpl}_{clusters}.nc" ), cop_soil_rural=resources( - "cop_soil_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_soil_rural_elec_s{simpl}_{clusters}.nc" ), cop_soil_urban=resources( - "cop_soil_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_soil_urban_elec_s{simpl}_{clusters}.nc" ), cop_air_total=resources( - "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_air_total_elec_s{simpl}_{clusters}.nc" ), cop_air_rural=resources( - "cop_air_rural_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_air_rural_elec_s{simpl}_{clusters}.nc" ), cop_air_urban=resources( - "cop_air_urban_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_air_urban_elec_s{simpl}_{clusters}.nc" ), solar_thermal_total=lambda w: ( - resources("solar_thermal_total_elec{weather_year}_s{simpl}_{clusters}.nc") + resources("solar_thermal_total_elec_s{simpl}_{clusters}.nc") if config_provider("sector", "solar_thermal")(w) else [] ), solar_thermal_urban=lambda w: ( - resources("solar_thermal_urban_elec{weather_year}_s{simpl}_{clusters}.nc") + resources("solar_thermal_urban_elec_s{simpl}_{clusters}.nc") if config_provider("sector", "solar_thermal")(w) else [] ), solar_thermal_rural=lambda w: ( - resources("solar_thermal_rural_elec{weather_year}_s{simpl}_{clusters}.nc") + resources("solar_thermal_rural_elec_s{simpl}_{clusters}.nc") if config_provider("sector", "solar_thermal")(w) else [] ), output: RESULTS - + "prenetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", threads: 1 resources: mem_mb=2000, log: RESULTS - + "logs/prepare_sector_network_elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", + + "logs/prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", benchmark: ( RESULTS - + "benchmarks/prepare_sector_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + + "benchmarks/prepare_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" diff --git a/rules/collect.smk b/rules/collect.smk index e31434f5..ef63df0f 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -16,7 +16,7 @@ localrules: rule cluster_networks: input: expand( - resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"), + resources("networks/elec_s{simpl}_{clusters}.nc"), **config["scenario"], run=config["run"]["name"], ), @@ -25,7 +25,7 @@ rule cluster_networks: rule extra_components_networks: input: expand( - resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc"), + resources("networks/elec_s{simpl}_{clusters}_ec.nc"), **config["scenario"], run=config["run"]["name"], ), @@ -35,7 +35,7 @@ rule prepare_elec_networks: input: expand( resources( - "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" ), **config["scenario"], run=config["run"]["name"], @@ -46,7 +46,7 @@ rule prepare_sector_networks: input: expand( RESULTS - + "prenetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", **config["scenario"], run=config["run"]["name"], ), @@ -56,7 +56,7 @@ rule solve_elec_networks: input: expand( RESULTS - + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config["scenario"], run=config["run"]["name"], ), @@ -66,7 +66,7 @@ rule solve_sector_networks: input: expand( RESULTS - + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", **config["scenario"], run=config["run"]["name"], ), @@ -76,7 +76,7 @@ rule solve_sector_networks_perfect: input: expand( RESULTS - + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", **config["scenario"], run=config["run"]["name"], ), @@ -86,13 +86,13 @@ rule validate_elec_networks: input: expand( RESULTS - + "figures/.statistics_plots_elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}", + + "figures/.statistics_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", **config["scenario"], run=config["run"]["name"], ), expand( RESULTS - + "figures/.validation_{kind}_plots_elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}", + + "figures/.validation_{kind}_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", **config["scenario"], run=config["run"]["name"], kind=["production", "prices", "cross_border"], diff --git a/rules/common.smk b/rules/common.smk index b298d743..618d746b 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -143,7 +143,7 @@ def solved_previous_horizon(w): return ( RESULTS - + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" + planning_horizon_p + ".nc" ) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 67b71503..2eba1591 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -13,18 +13,18 @@ if config["foresight"] != "perfect": params: plotting=config_provider("plotting"), input: - network=resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"), + network=resources("networks/elec_s{simpl}_{clusters}.nc"), regions_onshore=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), output: - map=resources("maps/power-network{weather_year}-s{simpl}-{clusters}.pdf"), + map=resources("maps/power-network-s{simpl}-{clusters}.pdf"), threads: 1 resources: mem_mb=4000, benchmark: benchmarks( - "plot_power_network_clustered/elec{weather_year}_s{simpl}_{clusters}" + "plot_power_network_clustered/elec_s{simpl}_{clusters}" ) conda: "../envs/environment.yaml" @@ -36,23 +36,23 @@ if config["foresight"] != "perfect": plotting=config_provider("plotting"), input: network=RESULTS - + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", regions=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), output: map=RESULTS - + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", threads: 2 resources: mem_mb=10000, log: RESULTS - + "logs/plot_power_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", + + "logs/plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", benchmark: ( RESULTS - + "benchmarksplot_power_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + + "benchmarksplot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -65,23 +65,23 @@ if config["foresight"] != "perfect": foresight=config_provider("foresight"), input: network=RESULTS - + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", regions=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), output: map=RESULTS - + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf", + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf", threads: 2 resources: mem_mb=10000, log: RESULTS - + "logs/plot_hydrogen_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", + + "logs/plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", benchmark: ( RESULTS - + "benchmarks/plot_hydrogen_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + + "benchmarks/plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -93,23 +93,23 @@ if config["foresight"] != "perfect": plotting=config_provider("plotting"), input: network=RESULTS - + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", regions=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), output: map=RESULTS - + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf", + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf", threads: 2 resources: mem_mb=10000, log: RESULTS - + "logs/plot_gas_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", + + "logs/plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", benchmark: ( RESULTS - + "benchmarks/plot_gas_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + + "benchmarks/plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -122,7 +122,7 @@ if config["foresight"] == "perfect": def output_map_year(w): return { f"map_{year}": RESULTS - + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_" + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_" + f"{year}.pdf" for year in config_provider("scenario", "planning_horizons")(w) } @@ -132,9 +132,9 @@ if config["foresight"] == "perfect": plotting=config_provider("plotting"), input: network=RESULTS - + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", regions=resources( - "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson" + "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), output: unpack(output_map_year), @@ -171,7 +171,7 @@ rule make_summary: input: networks=expand( RESULTS - + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", **config["scenario"], allow_missing=True, ), @@ -185,20 +185,20 @@ rule make_summary: ) ), ac_plot=expand( - resources("maps/power-network{weather_year}-s{simpl}-{clusters}.pdf"), + resources("maps/power-network-s{simpl}-{clusters}.pdf"), **config["scenario"], allow_missing=True, ), costs_plot=expand( RESULTS - + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", **config["scenario"], allow_missing=True, ), h2_plot=lambda w: expand( ( RESULTS - + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf" + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf" if config_provider("sector", "H2_network")(w) else [] ), @@ -208,7 +208,7 @@ rule make_summary: ch4_plot=lambda w: expand( ( RESULTS - + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf" + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf" if config_provider("sector", "gas_network")(w) else [] ), @@ -293,14 +293,14 @@ rule plot_elec_statistics: barplots=STATISTICS_BARPLOTS, input: network=RESULTS - + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: **{ f"{plot}_bar": RESULTS - + f"figures/statistics_{plot}_bar_elec{{weather_year}}_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" + + f"figures/statistics_{plot}_bar_elec{}_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" for plot in STATISTICS_BARPLOTS }, barplots_touch=RESULTS - + "figures/.statistics_plots_elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}", + + "figures/.statistics_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", script: "../scripts/plot_statistics.py" diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index c69dfb88..5c6777d4 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -14,23 +14,23 @@ rule solve_network: custom_extra_functionality=input_custom_extra_functionality, input: network=resources( - "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" ), config=RESULTS + "config.yaml", output: network=RESULTS - + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", log: solver=normpath( RESULTS - + "logs/solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log" + + "logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log" ), python=RESULTS - + "logs/solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", + + "logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", benchmark: ( RESULTS - + "benchmarks/solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" + + "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" ) threads: solver_threads resources: @@ -49,21 +49,21 @@ rule solve_operations_network: options=config_provider("solving", "options"), input: network=RESULTS - + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: network=RESULTS - + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc", + + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc", log: solver=normpath( RESULTS - + "logs/solve_operations_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log" + + "logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log" ), python=RESULTS - + "logs/solve_operations_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", + + "logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", benchmark: ( RESULTS - + "benchmarks/solve_operations_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}" + + "benchmarks/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" ) threads: 4 resources: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index da31ac46..eae1d990 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -11,12 +11,12 @@ rule add_existing_baseyear: costs=config_provider("costs"), input: network=RESULTS - + "prenetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", powerplants=resources("powerplants.csv"), - busmap_s=resources("busmap_elec{weather_year}_s{simpl}.csv"), - busmap=resources("busmap_elec{weather_year}_s{simpl}_{clusters}.csv"), + busmap_s=resources("busmap_elec_s{simpl}.csv"), + busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), clustered_pop_layout=resources( - "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + "pop_layout_elec_s{simpl}_{clusters}.csv" ), costs=lambda w: resources( "costs_{}.csv".format( @@ -24,20 +24,20 @@ rule add_existing_baseyear: ) ), cop_soil_total=resources( - "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_soil_total_elec_s{simpl}_{clusters}.nc" ), cop_air_total=resources( - "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_air_total_elec_s{simpl}_{clusters}.nc" ), existing_heating_distribution=resources( - "existing_heating_distribution_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv" + "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), existing_solar="data/existing_infrastructure/solar_capacity_IRENA.csv", existing_onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv", existing_offwind="data/existing_infrastructure/offwind_capacity_IRENA.csv", output: RESULTS - + "prenetworks-brownfield/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", wildcard_constraints: # TODO: The first planning_horizon needs to be aligned across scenarios # snakemake does not support passing functions to wildcard_constraints @@ -48,11 +48,11 @@ rule add_existing_baseyear: mem_mb=2000, log: RESULTS - + "logs/add_existing_baseyear_elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", + + "logs/add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", benchmark: ( RESULTS - + "benchmarks/add_existing_baseyear/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + + "benchmarks/add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -79,31 +79,31 @@ rule add_brownfield: carriers=config_provider("electricity", "renewable_carriers"), input: unpack(input_profile_tech_brownfield), - simplify_busmap=resources("busmap_elec{weather_year}_s{simpl}.csv"), - cluster_busmap=resources("busmap_elec{weather_year}_s{simpl}_{clusters}.csv"), + simplify_busmap=resources("busmap_elec_s{simpl}.csv"), + cluster_busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), network=RESULTS - + "prenetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", network_p=solved_previous_horizon, #solved network at previous time step costs=resources("costs_{planning_horizons}.csv"), cop_soil_total=resources( - "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_soil_total_elec_s{simpl}_{clusters}.nc" ), cop_air_total=resources( - "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc" + "cop_air_total_elec_s{simpl}_{clusters}.nc" ), output: RESULTS - + "prenetworks-brownfield/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", threads: 4 resources: mem_mb=10000, log: RESULTS - + "logs/add_brownfield_elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", + + "logs/add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", benchmark: ( RESULTS - + "benchmarks/add_brownfield/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + + "benchmarks/add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -125,19 +125,19 @@ rule solve_sector_network_myopic: custom_extra_functionality=input_custom_extra_functionality, input: network=RESULTS - + "prenetworks-brownfield/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", costs=resources("costs_{planning_horizons}.csv"), config=RESULTS + "config.yaml", output: RESULTS - + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", shadow: "shallow" log: solver=RESULTS - + "logs/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log", + + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log", python=RESULTS - + "logs/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", + + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", threads: solver_threads resources: mem_mb=config_provider("solving", "mem"), @@ -145,7 +145,7 @@ rule solve_sector_network_myopic: benchmark: ( RESULTS - + "benchmarks/solve_sector_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + + "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 21fce682..b212d453 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -14,20 +14,20 @@ rule solve_sector_network: custom_extra_functionality=input_custom_extra_functionality, input: network=RESULTS - + "prenetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", config=RESULTS + "config.yaml", output: RESULTS - + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", shadow: "shallow" log: solver=RESULTS - + "logs/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log", + + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log", memory=RESULTS - + "logs/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log", + + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log", python=RESULTS - + "logs/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", + + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", threads: solver_threads resources: mem_mb=config_provider("solving", "mem"), @@ -35,7 +35,7 @@ rule solve_sector_network: benchmark: ( RESULTS - + "benchmarks/solve_sector_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + + "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index c33012cd..1e175d87 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -206,7 +206,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "add_brownfield", - weather_year="", simpl="", clusters="37", opts="", diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index f997d852..91fc04ea 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -797,7 +797,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("add_electricity", weather_year="") + snakemake = mock_snakemake("add_electricity") configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 43d287cd..de3c10f0 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -553,7 +553,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "add_existing_baseyear", - weather_year="", configfiles="config/test/config.myopic.yaml", simpl="", clusters="37", diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 464b5f9f..9e3f9772 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -38,13 +38,13 @@ Inputs Outputs ------- -- ``networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc``: +- ``networks/elec_s{simpl}_{clusters}_ec.nc``: Description ----------- -The rule :mod:`add_extra_components` attaches additional extendable components to the clustered and simplified network. These can be configured in the ``config/config.yaml`` at ``electricity: extendable_carriers:``. It processes ``networks/elec{weather_year}_s{simpl}_{clusters}.nc`` to build ``networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc``, which in contrast to the former (depending on the configuration) contain with **zero** initial capacity +The rule :mod:`add_extra_components` attaches additional extendable components to the clustered and simplified network. These can be configured in the ``config/config.yaml`` at ``electricity: extendable_carriers:``. It processes ``networks/elec_s{simpl}_{clusters}.nc`` to build ``networks/elec_s{simpl}_{clusters}_ec.nc``, which in contrast to the former (depending on the configuration) contain with **zero** initial capacity - ``StorageUnits`` of carrier 'H2' and/or 'battery'. If this option is chosen, every bus is given an extendable ``StorageUnit`` of the corresponding carrier. The energy and power capacities are linked through a parameter that specifies the energy capacity as maximum hours at full dispatch power and is configured in ``electricity: max_hours:``. This linkage leads to one investment variable per storage unit. The default ``max_hours`` lead to long-term hydrogen and short-term battery storage units. @@ -229,7 +229,7 @@ if __name__ == "__main__": from _helpers import mock_snakemake snakemake = mock_snakemake( - "add_extra_components", weather_year="", simpl="", clusters=5 + "add_extra_components", simpl="", clusters=5 ) configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/build_artificial_load_data.py b/scripts/build_artificial_load_data.py index 7703ff28..51c07291 100755 --- a/scripts/build_artificial_load_data.py +++ b/scripts/build_artificial_load_data.py @@ -16,7 +16,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("build_artificial_load_data", weather_year="") + snakemake = mock_snakemake("build_artificial_load_data") configure_logging(snakemake) diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 3f8c35a8..79e2c203 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -218,7 +218,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_biomass_potentials", - weather_year="", simpl="", clusters="5", planning_horizons=2050, diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index d4cfb96d..5ec60349 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -19,7 +19,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_clustered_population_layouts", - weather_year="", simpl="", clusters=48, ) diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index af678978..16e44c18 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -32,7 +32,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_cop_profiles", - weather_year="", simpl="", clusters=48, ) diff --git a/scripts/build_daily_heat_demand.py b/scripts/build_daily_heat_demand.py index 8fe946a5..0ba47771 100644 --- a/scripts/build_daily_heat_demand.py +++ b/scripts/build_daily_heat_demand.py @@ -21,7 +21,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_daily_heat_demands", scope="total", - weather_year="", simpl="", clusters=48, ) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index 90454700..4328b842 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -258,7 +258,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("build_electricity_demand", weather_year="") + snakemake = mock_snakemake("build_electricity_demand") configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index e2461a7c..7cba0af5 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -146,7 +146,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_industrial_distribution_key", - weather_year="", simpl="", clusters=128, ) diff --git a/scripts/build_industrial_energy_demand_per_node_today.py b/scripts/build_industrial_energy_demand_per_node_today.py index ebfe562f..8b2b70a0 100644 --- a/scripts/build_industrial_energy_demand_per_node_today.py +++ b/scripts/build_industrial_energy_demand_per_node_today.py @@ -73,7 +73,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_industrial_energy_demand_per_node_today", - weather_year="", simpl="", clusters=48, ) diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index 6552eeb2..64e70366 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -23,7 +23,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_population_layouts", - weather_year="", ) configure_logging(snakemake) diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index 0cb3e61e..bf4a6118 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -15,7 +15,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_population_weighted_energy_totals", - weather_year="", simpl="", clusters=48, ) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index d31c13f0..3380a947 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -201,7 +201,7 @@ if __name__ == "__main__": from _helpers import mock_snakemake snakemake = mock_snakemake( - "build_renewable_profiles", technology="offwind-dc", weather_year="" + "build_renewable_profiles", technology="offwind-dc" ) configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index db56a401..dc921ff5 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -20,7 +20,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_solar_thermal_profiles", - weather_year="", simpl="", clusters=48, ) diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index fc13893f..9a64801f 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -167,7 +167,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_transport_demand", - weather_year="", simpl="", clusters=48, ) diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index fd335cd3..19585aa9 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -105,7 +105,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("cluster_gas_network", weather_year="", simpl="", clusters="37") + snakemake = mock_snakemake("cluster_gas_network", simpl="", clusters="37") configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 8ba3e7b3..5682c337 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -32,28 +32,28 @@ Relevant Settings Inputs ------ -- ``resources/regions_onshore_elec{weather_year}_s{simpl}.geojson``: confer :ref:`simplify` -- ``resources/regions_offshore_elec{weather_year}_s{simpl}.geojson``: confer :ref:`simplify` -- ``networks/elec{weather_year}_s{simpl}.nc``: confer :ref:`simplify` -- ``resources/busmap_elec{weather_year}_s{simpl}.csv``: confer :ref:`simplify` -- ``data/custom_busmap_elec{weather_year}_s{simpl}_{clusters}.csv``: optional input +- ``resources/regions_onshore_elec_s{simpl}.geojson``: confer :ref:`simplify` +- ``resources/regions_offshore_elec_s{simpl}.geojson``: confer :ref:`simplify` +- ``networks/elec_s{simpl}.nc``: confer :ref:`simplify` +- ``resources/busmap_elec_s{simpl}.csv``: confer :ref:`simplify` +- ``data/custom_busmap_elec_s{simpl}_{clusters}.csv``: optional input Outputs ------- -- ``resources/regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson``: +- ``resources/regions_onshore_elec_s{simpl}_{clusters}.geojson``: .. image:: img/regions_onshore_elec_s_X.png :scale: 33 % -- ``resources/regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson``: +- ``resources/regions_offshore_elec_s{simpl}_{clusters}.geojson``: .. image:: img/regions_offshore_elec_s_X.png :scale: 33 % -- ``resources/busmap_elec{weather_year}_s{simpl}_{clusters}.csv``: Mapping of buses from ``networks/elec_s{simpl}.nc`` to ``networks/elec_s{simpl}_{clusters}.nc``; -- ``resources/linemap{weather_year}_elec_s{simpl}_{clusters}.csv``: Mapping of lines from ``networks/elec_s{simpl}.nc`` to ``networks/elec_s{simpl}_{clusters}.nc``; -- ``networks/elec{weather_year}_s{simpl}_{clusters}.nc``: +- ``resources/busmap_elec_s{simpl}_{clusters}.csv``: Mapping of buses from ``networks/elec_s{simpl}.nc`` to ``networks/elec_s{simpl}_{clusters}.nc``; +- ``resources/linemap_elec_s{simpl}_{clusters}.csv``: Mapping of lines from ``networks/elec_s{simpl}.nc`` to ``networks/elec_s{simpl}_{clusters}.nc``; +- ``networks/elec_s{simpl}_{clusters}.nc``: .. image:: img/elec_s_X.png :scale: 40 % @@ -455,7 +455,7 @@ if __name__ == "__main__": from _helpers import mock_snakemake snakemake = mock_snakemake( - "cluster_network", simpl="", clusters="5", weather_year="" + "cluster_network", simpl="", clusters="5", ) configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 67089f92..27ce91d3 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -646,7 +646,7 @@ def make_summaries(networks_dict): columns = pd.MultiIndex.from_tuples( networks_dict.keys(), - names=["weather_year", "cluster", "ll", "opt", "planning_horizon"], + names=["cluster", "ll", "opt", "planning_horizon"], ) df = {output: pd.DataFrame(columns=columns, dtype=float) for output in outputs} @@ -679,10 +679,9 @@ if __name__ == "__main__": set_scenario_config(snakemake) networks_dict = { - (weather_year, cluster, ll, opt + sector_opt, planning_horizon): "results/" + (cluster, ll, opt + sector_opt, planning_horizon): "results/" + snakemake.params.RDIR - + f"postnetworks/elec{weather_year}_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc" - for weather_year in snakemake.params.scenario["weather_year"] + + f"postnetworks/elec_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc" for simpl in snakemake.params.scenario["simpl"] for cluster in snakemake.params.scenario["clusters"] for opt in snakemake.params.scenario["opts"] diff --git a/scripts/prepare_links_p_nom.py b/scripts/prepare_links_p_nom.py index 627122e7..7c1ed211 100644 --- a/scripts/prepare_links_p_nom.py +++ b/scripts/prepare_links_p_nom.py @@ -67,7 +67,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake # rule must be enabled in config - snakemake = mock_snakemake("prepare_links_p_nom", simpl="", weather_year="") + snakemake = mock_snakemake("prepare_links_p_nom", simpl="") configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 928fa577..f38a3619 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -41,12 +41,12 @@ Inputs ------ - ``resources/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity. -- ``networks/elec{weather_year}_s{simpl}_{clusters}.nc``: confer :ref:`cluster` +- ``networks/elec_s{simpl}_{clusters}.nc``: confer :ref:`cluster` Outputs ------- -- ``networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: Complete PyPSA network that will be handed to the ``solve_network`` rule. +- ``networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: Complete PyPSA network that will be handed to the ``solve_network`` rule. Description ----------- @@ -305,7 +305,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "prepare_network", - weather_year="", simpl="", clusters="37", ll="v1.0", diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 6ca06c9a..3a1158b9 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3584,7 +3584,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "prepare_sector_network", configfiles="test/config.overnight.yaml", - weather_year="", simpl="", opts="", clusters="37", diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 5a4c792e..4de17704 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -55,18 +55,18 @@ Inputs Outputs ------- -- ``resources/regions_onshore_elec{weather_year}_s{simpl}.geojson``: +- ``resources/regions_onshore_elec_s{simpl}.geojson``: .. image:: img/regions_onshore_elec_s.png :scale: 33 % -- ``resources/regions_offshore_elec{weather_year}_s{simpl}.geojson``: +- ``resources/regions_offshore_elec_s{simpl}.geojson``: .. image:: img/regions_offshore_elec_s .png :scale: 33 % -- ``resources/busmap_elec{weather_year}_s{simpl}.h5``: Mapping of buses from ``networks/elec.nc`` to ``networks/elec{weather_year}_s{simpl}.nc``; has keys ['/busmap_s'] -- ``networks/elec{weather_year}_s{simpl}.nc``: +- ``resources/busmap_elec_s{simpl}.h5``: Mapping of buses from ``networks/elec.nc`` to ``networks/elec_s{simpl}.nc``; has keys ['/busmap_s'] +- ``networks/elec_s{simpl}.nc``: .. image:: img/elec_s.png :scale: 33 % @@ -527,7 +527,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("simplify_network", weather_year="", simpl="") + snakemake = mock_snakemake("simplify_network", simpl="") configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index aed78824..7e53e606 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -929,7 +929,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "solve_sector_network", configfiles="../config/test/config.perfect.yaml", - weather_year="", simpl="", opts="", clusters="37", diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index fdc2feb6..bd4ce7aa 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -29,7 +29,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "solve_operations_network", configfiles="test/config.electricity.yaml", - weather_year="", simpl="", opts="", clusters="5", From 030e5b15ade0adf593609e6698c45038888e6492 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 4 Mar 2024 17:51:32 +0100 Subject: [PATCH 098/173] remove rule build_cutout_year --- rules/build_electricity.smk | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 53c2df0b..ab7b1913 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -165,8 +165,6 @@ rule build_bus_regions: if config["enable"].get("build_cutout", False): - ruleorder: build_cutout_year > build_cutout - rule build_cutout: params: snapshots=config_provider("snapshots"), @@ -188,23 +186,6 @@ if config["enable"].get("build_cutout", False): script: "../scripts/build_cutout.py" - rule build_cutout_year: - input: - rules.build_cutout.input, - output: - protected("cutouts/" + CDIR + "{cutout}-.nc"), - log: - logs(CDIR + "build_cutout/{cutout}-.log"), - benchmark: - benchmarks(CDIR + "build_cutout_{cutout}-") - threads: ATLITE_NPROCESSES - resources: - mem_mb=ATLITE_NPROCESSES * 1000, - conda: - "../envs/environment.yaml" - script: - "../scripts/build_cutout.py" - if config["enable"].get("build_natura_raster", False): From 7b584f16bbb467f24ac44bba7fbdd497581f2394 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:55:57 +0000 Subject: [PATCH 099/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_sector.smk | 296 +++++++--------------------- rules/solve_myopic.smk | 20 +- scripts/add_extra_components.py | 4 +- scripts/build_renewable_profiles.py | 4 +- scripts/cluster_network.py | 4 +- 5 files changed, 84 insertions(+), 244 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index ab296565..686bb9a1 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -33,25 +33,19 @@ rule build_clustered_population_layouts: pop_layout_total=resources("pop_layout_total.nc"), pop_layout_urban=resources("pop_layout_urban.nc"), pop_layout_rural=resources("pop_layout_rural.nc"), - regions_onshore=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), cutout=lambda w: "cutouts/" + CDIR + config_provider("atlite", "default_cutout")(w) + ".nc", output: - clustered_pop_layout=resources( - "pop_layout_elec_s{simpl}_{clusters}.csv" - ), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), log: logs("build_clustered_population_layouts_{simpl}_{clusters}.log"), resources: mem_mb=10000, benchmark: - benchmarks( - "build_clustered_population_layouts/s{simpl}_{clusters}" - ) + benchmarks("build_clustered_population_layouts/s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -105,16 +99,10 @@ rule build_gas_input_locations: ), entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson", storage="data/gas_network/scigrid-gas/data/IGGIELGN_Storages.geojson", - regions_onshore=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - regions_offshore=resources( - "regions_offshore_elec_s{simpl}_{clusters}.geojson" - ), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), + regions_offshore=resources("regions_offshore_elec_s{simpl}_{clusters}.geojson"), output: - gas_input_nodes=resources( - "gas_input_locations_s{simpl}_{clusters}.geojson" - ), + gas_input_nodes=resources("gas_input_locations_s{simpl}_{clusters}.geojson"), gas_input_nodes_simplified=resources( "gas_input_locations_s{simpl}_{clusters}_simplified.csv" ), @@ -131,16 +119,10 @@ rule build_gas_input_locations: rule cluster_gas_network: input: cleaned_gas_network=resources("gas_network.csv"), - regions_onshore=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - regions_offshore=resources( - "regions_offshore_elec_s{simpl}_{clusters}.geojson" - ), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), + regions_offshore=resources("regions_offshore_elec_s{simpl}_{clusters}.geojson"), output: - clustered_gas_network=resources( - "gas_network_elec_s{simpl}_{clusters}.csv" - ), + clustered_gas_network=resources("gas_network_elec_s{simpl}_{clusters}.csv"), resources: mem_mb=4000, log: @@ -170,14 +152,10 @@ rule build_daily_heat_demand: drop_leap_day=config_provider("enable", "drop_leap_day"), input: pop_layout=resources("pop_layout_{scope}.nc"), - regions_onshore=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), cutout=heat_demand_cutout, output: - heat_demand=resources( - "daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc" - ), + heat_demand=resources("daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, threads: 8 @@ -197,13 +175,9 @@ rule build_hourly_heat_demand: drop_leap_day=config_provider("enable", "drop_leap_day"), input: heat_profile="data/heat_load_profile_BDEW.csv", - heat_demand=resources( - "daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc" - ), + heat_demand=resources("daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"), output: - heat_demand=resources( - "hourly_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc" - ), + heat_demand=resources("hourly_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=2000, threads: 8 @@ -223,14 +197,10 @@ rule build_temperature_profiles: drop_leap_day=config_provider("enable", "drop_leap_day"), input: pop_layout=resources("pop_layout_{scope}.nc"), - regions_onshore=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), cutout=heat_demand_cutout, output: - temp_soil=resources( - "temp_soil_{scope}_elec_s{simpl}_{clusters}.nc" - ), + temp_soil=resources("temp_soil_{scope}_elec_s{simpl}_{clusters}.nc"), temp_air=resources("temp_air_{scope}_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, @@ -238,9 +208,7 @@ rule build_temperature_profiles: log: logs("build_temperature_profiles_{scope}__{simpl}_{clusters}.log"), benchmark: - benchmarks( - "build_temperature_profiles/{scope}__s{simpl}_{clusters}" - ) + benchmarks("build_temperature_profiles/{scope}__s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -251,43 +219,19 @@ rule build_cop_profiles: params: heat_pump_sink_T=config_provider("sector", "heat_pump_sink_T"), input: - temp_soil_total=resources( - "temp_soil_total_elec_s{simpl}_{clusters}.nc" - ), - temp_soil_rural=resources( - "temp_soil_rural_elec_s{simpl}_{clusters}.nc" - ), - temp_soil_urban=resources( - "temp_soil_urban_elec_s{simpl}_{clusters}.nc" - ), - temp_air_total=resources( - "temp_air_total_elec_s{simpl}_{clusters}.nc" - ), - temp_air_rural=resources( - "temp_air_rural_elec_s{simpl}_{clusters}.nc" - ), - temp_air_urban=resources( - "temp_air_urban_elec_s{simpl}_{clusters}.nc" - ), + temp_soil_total=resources("temp_soil_total_elec_s{simpl}_{clusters}.nc"), + temp_soil_rural=resources("temp_soil_rural_elec_s{simpl}_{clusters}.nc"), + temp_soil_urban=resources("temp_soil_urban_elec_s{simpl}_{clusters}.nc"), + temp_air_total=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"), + temp_air_rural=resources("temp_air_rural_elec_s{simpl}_{clusters}.nc"), + temp_air_urban=resources("temp_air_urban_elec_s{simpl}_{clusters}.nc"), output: - cop_soil_total=resources( - "cop_soil_total_elec_s{simpl}_{clusters}.nc" - ), - cop_soil_rural=resources( - "cop_soil_rural_elec_s{simpl}_{clusters}.nc" - ), - cop_soil_urban=resources( - "cop_soil_urban_elec_s{simpl}_{clusters}.nc" - ), - cop_air_total=resources( - "cop_air_total_elec_s{simpl}_{clusters}.nc" - ), - cop_air_rural=resources( - "cop_air_rural_elec_s{simpl}_{clusters}.nc" - ), - cop_air_urban=resources( - "cop_air_urban_elec_s{simpl}_{clusters}.nc" - ), + cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), + cop_soil_rural=resources("cop_soil_rural_elec_s{simpl}_{clusters}.nc"), + cop_soil_urban=resources("cop_soil_urban_elec_s{simpl}_{clusters}.nc"), + cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), + cop_air_rural=resources("cop_air_rural_elec_s{simpl}_{clusters}.nc"), + cop_air_urban=resources("cop_air_urban_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, log: @@ -320,25 +264,17 @@ rule build_solar_thermal_profiles: solar_thermal=config_provider("solar_thermal"), input: pop_layout=resources("pop_layout_{scope}.nc"), - regions_onshore=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), cutout=solar_thermal_cutout, output: - solar_thermal=resources( - "solar_thermal_{scope}_elec_s{simpl}_{clusters}.nc" - ), + solar_thermal=resources("solar_thermal_{scope}_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, threads: 16 log: - logs( - "build_solar_thermal_profiles_{scope}__s{simpl}_{clusters}.log" - ), + logs("build_solar_thermal_profiles_{scope}__s{simpl}_{clusters}.log"), benchmark: - benchmarks( - "build_solar_thermal_profiles/{scope}__s{simpl}_{clusters}" - ) + benchmarks("build_solar_thermal_profiles/{scope}__s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -402,9 +338,7 @@ rule build_biomass_potentials: keep_local=True, ), nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 - regions_onshore=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"), swiss_cantons=ancient("data/bundle/ch_cantons.csv"), swiss_population=ancient("data/bundle/je-e-21.03.02.xls"), @@ -420,13 +354,9 @@ rule build_biomass_potentials: resources: mem_mb=1000, log: - logs( - "build_biomass_potentials_s{simpl}_{clusters}_{planning_horizons}.log" - ), + logs("build_biomass_potentials_s{simpl}_{clusters}_{planning_horizons}.log"), benchmark: - benchmarks( - "build_biomass_potentials_s{simpl}_{clusters}_{planning_horizons}" - ) + benchmarks("build_biomass_potentials_s{simpl}_{clusters}_{planning_horizons}") conda: "../envs/environment.yaml" script: @@ -464,12 +394,8 @@ rule build_sequestration_potentials: "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", keep_local=True, ), - regions_onshore=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - regions_offshore=resources( - "regions_offshore_elec_s{simpl}_{clusters}.geojson" - ), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), + regions_offshore=resources("regions_offshore_elec_s{simpl}_{clusters}.geojson"), output: sequestration_potential=resources( "co2_sequestration_potential_elec_s{simpl}_{clusters}.csv" @@ -490,16 +416,10 @@ rule build_sequestration_potentials: rule build_salt_cavern_potentials: input: salt_caverns="data/bundle-sector/h2_salt_caverns_GWh_per_sqkm.geojson", - regions_onshore=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - regions_offshore=resources( - "regions_offshore_elec_s{simpl}_{clusters}.geojson" - ), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), + regions_offshore=resources("regions_offshore_elec_s{simpl}_{clusters}.geojson"), output: - h2_cavern_potential=resources( - "salt_cavern_potentials_s{simpl}_{clusters}.csv" - ), + h2_cavern_potential=resources("salt_cavern_potentials_s{simpl}_{clusters}.csv"), threads: 1 resources: mem_mb=2000, @@ -641,12 +561,8 @@ rule build_industrial_distribution_key: ), countries=config_provider("countries"), input: - regions_onshore=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - clustered_pop_layout=resources( - "pop_layout_elec_s{simpl}_{clusters}.csv" - ), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), hotmaps_industrial_database="data/bundle-sector/Industrial_Database.csv", output: industrial_distribution_key=resources( @@ -658,9 +574,7 @@ rule build_industrial_distribution_key: log: logs("build_industrial_distribution_key_s{simpl}_{clusters}.log"), benchmark: - benchmarks( - "build_industrial_distribution_key/s{simpl}_{clusters}" - ) + benchmarks("build_industrial_distribution_key/s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -774,13 +688,9 @@ rule build_industrial_energy_demand_per_node_today: resources: mem_mb=1000, log: - logs( - "build_industrial_energy_demand_per_node_today_s{simpl}_{clusters}.log" - ), + logs("build_industrial_energy_demand_per_node_today_s{simpl}_{clusters}.log"), benchmark: - benchmarks( - "build_industrial_energy_demand_per_node_today/s{simpl}_{clusters}" - ) + benchmarks("build_industrial_energy_demand_per_node_today/s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -794,16 +704,12 @@ rule build_retro_cost: input: building_stock="data/retro/data_building_stock.csv", data_tabula="data/bundle-sector/retro/tabula-calculator-calcsetbuilding.csv", - air_temperature=resources( - "temp_air_total_elec_s{simpl}_{clusters}.nc" - ), + air_temperature=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"), u_values_PL="data/retro/u_values_poland.csv", tax_w="data/retro/electricity_taxes_eu.csv", construction_index="data/retro/comparative_level_investment.csv", floor_area_missing="data/retro/floor_area_missing.csv", - clustered_pop_layout=resources( - "pop_layout_elec_s{simpl}_{clusters}.csv" - ), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), cost_germany="data/retro/retro_cost_germany.csv", window_assumptions="data/retro/window_assumptions.csv", output: @@ -824,18 +730,14 @@ rule build_retro_cost: rule build_population_weighted_energy_totals: input: energy_totals=resources("{kind}_totals.csv"), - clustered_pop_layout=resources( - "pop_layout_elec_s{simpl}_{clusters}.csv" - ), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), output: resources("pop_weighted_{kind}_totals_s{simpl}_{clusters}.csv"), threads: 1 resources: mem_mb=2000, log: - logs( - "build_population_weighted_{kind}_totals_s{simpl}_{clusters}.log" - ), + logs("build_population_weighted_{kind}_totals_s{simpl}_{clusters}.log"), conda: "../envs/environment.yaml" script: @@ -846,9 +748,7 @@ rule build_shipping_demand: input: ports="data/attributed_ports.json", scope=resources("europe_shape.geojson"), - regions=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), demand=resources("energy_totals.csv"), params: energy_totals_year=config_provider("energy", "energy_totals_year"), @@ -871,22 +771,16 @@ rule build_transport_demand: drop_leap_day=config_provider("enable", "drop_leap_day"), sector=config_provider("sector"), input: - clustered_pop_layout=resources( - "pop_layout_elec_s{simpl}_{clusters}.csv" - ), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), pop_weighted_energy_totals=resources( "pop_weighted_energy_totals_s{simpl}_{clusters}.csv" ), transport_data=resources("transport_data.csv"), traffic_data_KFZ="data/bundle-sector/emobility/KFZ__count", traffic_data_Pkw="data/bundle-sector/emobility/Pkw__count", - temp_air_total=resources( - "temp_air_total_elec_s{simpl}_{clusters}.nc" - ), + temp_air_total=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"), output: - transport_demand=resources( - "transport_demand_s{simpl}_{clusters}.csv" - ), + transport_demand=resources("transport_demand_s{simpl}_{clusters}.csv"), transport_data=resources("transport_data_s{simpl}_{clusters}.csv"), avail_profile=resources("avail_profile_s{simpl}_{clusters}.csv"), dsm_profile=resources("dsm_profile_s{simpl}_{clusters}.csv"), @@ -906,9 +800,7 @@ rule build_district_heat_share: sector=config_provider("sector"), input: district_heat_share=resources("district_heat_share.csv"), - clustered_pop_layout=resources( - "pop_layout_elec_s{simpl}_{clusters}.csv" - ), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), output: district_heat_share=resources( "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv" @@ -917,9 +809,7 @@ rule build_district_heat_share: resources: mem_mb=1000, log: - logs( - "build_district_heat_share_s{simpl}_{clusters}_{planning_horizons}.log" - ), + logs("build_district_heat_share_s{simpl}_{clusters}_{planning_horizons}.log"), conda: "../envs/environment.yaml" script: @@ -933,9 +823,7 @@ rule build_existing_heating_distribution: existing_capacities=config_provider("existing_capacities"), input: existing_heating="data/existing_infrastructure/existing_heating_raw.csv", - clustered_pop_layout=resources( - "pop_layout_elec_s{simpl}_{clusters}.csv" - ), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), clustered_pop_energy_layout=resources( "pop_weighted_energy_totals_s{simpl}_{clusters}.csv" ), @@ -1013,17 +901,13 @@ rule prepare_sector_network: else [] ), sequestration_potential=lambda w: ( - resources( - "co2_sequestration_potential_elec_s{simpl}_{clusters}.csv" - ) + resources("co2_sequestration_potential_elec_s{simpl}_{clusters}.csv") if config_provider( "sector", "regional_co2_sequestration_potential", "enable" )(w) else [] ), - network=resources( - "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - ), + network=resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), energy_totals_name=resources("energy_totals.csv"), eurostat=input_eurostat, pop_weighted_energy_totals=resources( @@ -1032,12 +916,8 @@ rule prepare_sector_network: pop_weighted_heat_totals=resources( "pop_weighted_heat_totals_s{simpl}_{clusters}.csv" ), - shipping_demand=resources( - "shipping_demand_s{simpl}_{clusters}.csv" - ), - transport_demand=resources( - "transport_demand_s{simpl}_{clusters}.csv" - ), + shipping_demand=resources("shipping_demand_s{simpl}_{clusters}.csv"), + transport_demand=resources("transport_demand_s{simpl}_{clusters}.csv"), transport_data=resources("transport_data_s{simpl}_{clusters}.csv"), avail_profile=resources("avail_profile_s{simpl}_{clusters}.csv"), dsm_profile=resources("dsm_profile_s{simpl}_{clusters}.csv"), @@ -1058,14 +938,10 @@ rule prepare_sector_network: if config_provider("foresight")(w) == "overnight" else resources("costs_{planning_horizons}.csv") ), - h2_cavern=resources( - "salt_cavern_potentials_s{simpl}_{clusters}.csv" - ), + h2_cavern=resources("salt_cavern_potentials_s{simpl}_{clusters}.csv"), busmap_s=resources("busmap_elec_s{simpl}.csv"), busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), - clustered_pop_layout=resources( - "pop_layout_elec_s{simpl}_{clusters}.csv" - ), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), simplified_pop_layout=resources("pop_layout_elec_s{simpl}.csv"), industrial_demand=resources( "industrial_energy_demand_elec_s{simpl}_{clusters}_{planning_horizons}.csv" @@ -1076,42 +952,18 @@ rule prepare_sector_network: district_heat_share=resources( "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), - temp_soil_total=resources( - "temp_soil_total_elec_s{simpl}_{clusters}.nc" - ), - temp_soil_rural=resources( - "temp_soil_rural_elec_s{simpl}_{clusters}.nc" - ), - temp_soil_urban=resources( - "temp_soil_urban_elec_s{simpl}_{clusters}.nc" - ), - temp_air_total=resources( - "temp_air_total_elec_s{simpl}_{clusters}.nc" - ), - temp_air_rural=resources( - "temp_air_rural_elec_s{simpl}_{clusters}.nc" - ), - temp_air_urban=resources( - "temp_air_urban_elec_s{simpl}_{clusters}.nc" - ), - cop_soil_total=resources( - "cop_soil_total_elec_s{simpl}_{clusters}.nc" - ), - cop_soil_rural=resources( - "cop_soil_rural_elec_s{simpl}_{clusters}.nc" - ), - cop_soil_urban=resources( - "cop_soil_urban_elec_s{simpl}_{clusters}.nc" - ), - cop_air_total=resources( - "cop_air_total_elec_s{simpl}_{clusters}.nc" - ), - cop_air_rural=resources( - "cop_air_rural_elec_s{simpl}_{clusters}.nc" - ), - cop_air_urban=resources( - "cop_air_urban_elec_s{simpl}_{clusters}.nc" - ), + temp_soil_total=resources("temp_soil_total_elec_s{simpl}_{clusters}.nc"), + temp_soil_rural=resources("temp_soil_rural_elec_s{simpl}_{clusters}.nc"), + temp_soil_urban=resources("temp_soil_urban_elec_s{simpl}_{clusters}.nc"), + temp_air_total=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"), + temp_air_rural=resources("temp_air_rural_elec_s{simpl}_{clusters}.nc"), + temp_air_urban=resources("temp_air_urban_elec_s{simpl}_{clusters}.nc"), + cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), + cop_soil_rural=resources("cop_soil_rural_elec_s{simpl}_{clusters}.nc"), + cop_soil_urban=resources("cop_soil_urban_elec_s{simpl}_{clusters}.nc"), + cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), + cop_air_rural=resources("cop_air_rural_elec_s{simpl}_{clusters}.nc"), + cop_air_urban=resources("cop_air_urban_elec_s{simpl}_{clusters}.nc"), solar_thermal_total=lambda w: ( resources("solar_thermal_total_elec_s{simpl}_{clusters}.nc") if config_provider("sector", "solar_thermal")(w) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index eae1d990..5eb30233 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -15,20 +15,14 @@ rule add_existing_baseyear: powerplants=resources("powerplants.csv"), busmap_s=resources("busmap_elec_s{simpl}.csv"), busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), - clustered_pop_layout=resources( - "pop_layout_elec_s{simpl}_{clusters}.csv" - ), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), costs=lambda w: resources( "costs_{}.csv".format( config_provider("scenario", "planning_horizons", 0)(w) ) ), - cop_soil_total=resources( - "cop_soil_total_elec_s{simpl}_{clusters}.nc" - ), - cop_air_total=resources( - "cop_air_total_elec_s{simpl}_{clusters}.nc" - ), + cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), + cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), existing_heating_distribution=resources( "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), @@ -85,12 +79,8 @@ rule add_brownfield: + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", network_p=solved_previous_horizon, #solved network at previous time step costs=resources("costs_{planning_horizons}.csv"), - cop_soil_total=resources( - "cop_soil_total_elec_s{simpl}_{clusters}.nc" - ), - cop_air_total=resources( - "cop_air_total_elec_s{simpl}_{clusters}.nc" - ), + cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), + cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), output: RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 9e3f9772..eb14436e 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -228,9 +228,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake( - "add_extra_components", simpl="", clusters=5 - ) + snakemake = mock_snakemake("add_extra_components", simpl="", clusters=5) configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 3380a947..a4822921 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -200,9 +200,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake( - "build_renewable_profiles", technology="offwind-dc" - ) + snakemake = mock_snakemake("build_renewable_profiles", technology="offwind-dc") configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 5682c337..47b65359 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -455,7 +455,9 @@ if __name__ == "__main__": from _helpers import mock_snakemake snakemake = mock_snakemake( - "cluster_network", simpl="", clusters="5", + "cluster_network", + simpl="", + clusters="5", ) configure_logging(snakemake) set_scenario_config(snakemake) From 249cfab4292f11e2efdaa70e6ca104d297a48570 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 4 Mar 2024 18:24:01 +0100 Subject: [PATCH 100/173] remove remaining {weather_year} wildcard usages --- scripts/add_electricity.py | 9 +-------- scripts/build_artificial_load_data.py | 9 +-------- scripts/build_clustered_population_layouts.py | 6 +----- scripts/build_cutout.py | 4 ---- scripts/build_daily_heat_demand.py | 11 ++--------- scripts/build_electricity_demand.py | 9 +-------- scripts/build_hourly_heat_demand.py | 9 +-------- scripts/build_hydro_profile.py | 12 ++++++------ scripts/build_population_layouts.py | 6 +----- scripts/build_population_weighted_energy_totals.py | 10 +++++++--- scripts/build_renewable_profiles.py | 9 +-------- scripts/build_solar_thermal_profiles.py | 13 ++----------- scripts/build_temperature_profiles.py | 13 ++----------- scripts/build_transport_demand.py | 8 +------- 14 files changed, 27 insertions(+), 101 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 91fc04ea..c0076293 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -805,14 +805,7 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.base_network) - weather_year = snakemake.wildcards.weather_year - if weather_year: - snapshots = dict( - start=weather_year, end=str(int(weather_year) + 1), inclusive="left" - ) - else: - snapshots = snakemake.config["snapshots"] - n.set_snapshots(pd.date_range(freq="h", **snapshots)) + n.set_snapshots(pd.date_range(freq="h", **snakemake.params.snapshots)) Nyears = n.snapshot_weightings.objective.sum() / 8760.0 diff --git a/scripts/build_artificial_load_data.py b/scripts/build_artificial_load_data.py index 51c07291..a12ac917 100755 --- a/scripts/build_artificial_load_data.py +++ b/scripts/build_artificial_load_data.py @@ -20,14 +20,7 @@ if __name__ == "__main__": configure_logging(snakemake) - weather_year = snakemake.wildcards.weather_year - if weather_year: - snapshots = dict( - start=weather_year, end=str(int(weather_year) + 1), inclusive="left" - ) - else: - snapshots = snakemake.config["snapshots"] - snapshots = pd.date_range(freq="h", **snapshots) + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) fixed_year = snakemake.config["load"].get("fixed_year", False) years = ( diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index 21ff02be..2d9c6acb 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -25,11 +25,7 @@ if __name__ == "__main__": set_scenario_config(snakemake) - cutout_name = snakemake.input.cutout - year = snakemake.wildcards.weather_year - if year: - cutout_name = cutout_name.format(weather_year=year) - cutout = atlite.Cutout(cutout_name) + cutout = atlite.Cutout(snakemake.input.cutout) clustered_regions = ( gpd.read_file(snakemake.input.regions_onshore).set_index("name").buffer(0) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 039f645c..80ddae34 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -109,10 +109,6 @@ if __name__ == "__main__": cutout_params = snakemake.params.cutouts[snakemake.wildcards.cutout] - if hasattr(snakemake.wildcards, "weather_year"): - time = snakemake.wildcards.weather_year - cutout_params["time"] = [time, time] - if "time" not in cutout_params: snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) cutout_params["time"] = [snapshots[0], snapshots[-1]] diff --git a/scripts/build_daily_heat_demand.py b/scripts/build_daily_heat_demand.py index 0ba47771..c08cfe33 100644 --- a/scripts/build_daily_heat_demand.py +++ b/scripts/build_daily_heat_demand.py @@ -31,16 +31,9 @@ if __name__ == "__main__": client = Client(cluster, asynchronous=True) cutout_name = snakemake.input.cutout - year = snakemake.wildcards.weather_year - if year: - snapshots = dict(start=year, end=str(int(year) + 1), inclusive="left") - cutout_name = cutout_name.format(weather_year=year) - else: - snapshots = snakemake.params.snapshots - - time = pd.date_range(freq="h", **snapshots) - daily = pd.date_range(freq="D", **snapshots) + time = pd.date_range(freq="h", **snakemake.params.snapshots) + daily = pd.date_range(freq="D", **snakemake.params.snapshots) if snakemake.params.drop_leap_day: time = time[~((time.month == 2) & (time.day == 29))] daily = daily[~((daily.month == 2) & (daily.day == 29))] diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index 4328b842..f5bd744a 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -263,14 +263,7 @@ if __name__ == "__main__": configure_logging(snakemake) set_scenario_config(snakemake) - weather_year = snakemake.wildcards.weather_year - if weather_year: - snapshots = dict( - start=weather_year, end=str(int(weather_year) + 1), inclusive="left" - ) - else: - snapshots = snakemake.params.snapshots - snapshots = pd.date_range(freq="h", **snapshots) + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) fixed_year = snakemake.config["load"].get("fixed_year", False) years = ( diff --git a/scripts/build_hourly_heat_demand.py b/scripts/build_hourly_heat_demand.py index 664b4499..c3916b54 100644 --- a/scripts/build_hourly_heat_demand.py +++ b/scripts/build_hourly_heat_demand.py @@ -24,14 +24,7 @@ if __name__ == "__main__": ) set_scenario_config(snakemake) - year = snakemake.wildcards.weather_year - - if year: - snapshots = dict(start=year, end=str(int(year) + 1), inclusive="left") - else: - snapshots = snakemake.params.snapshots - - snapshots = pd.date_range(freq="h", **snapshots) + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) if snakemake.params.drop_leap_day: snapshots = snapshots[~((snapshots.month == 2) & (snapshots.day == 29))] diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index af5f0ee6..a732cf45 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -65,8 +65,8 @@ import atlite import country_converter as coco import geopandas as gpd import pandas as pd -from _helpers import configure_logging, set_scenario_config from numpy.polynomial import Polynomial +from _helpers import configure_logging, set_scenario_config cc = coco.CountryConverter() @@ -167,7 +167,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("build_hydro_profile", weather_year="") + snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) set_scenario_config(snakemake) @@ -197,12 +197,12 @@ if __name__ == "__main__": eia_stats.to_csv(snakemake.output.eia_hydro) - weather_year = snakemake.wildcards.weather_year + contained_years = pd.date_range(freq="YE", **snakemake.params.snapshots).year norm_year = config_hydro.get("eia_norm_year") if norm_year: - eia_stats.loc[weather_year] = eia_stats.loc[norm_year] - elif weather_year and weather_year not in eia_stats.index: - eia_stats.loc[weather_year] = eia_stats.median() + eia_stats.loc[contained_years] = eia_stats.loc[norm_year] + elif missing_years := eia_stats.index.difference(contained_years): + eia_stats.loc[missing_years] = eia_stats.median() inflow = cutout.runoff( shapes=country_shapes, diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index 64e70366..dc4cf2f8 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -28,11 +28,7 @@ if __name__ == "__main__": configure_logging(snakemake) set_scenario_config(snakemake) - cutout_name = snakemake.input.cutout - year = snakemake.wildcards.weather_year - if year: - cutout_name = cutout_name.format(weather_year=year) - cutout = atlite.Cutout(cutout_name) + cutout = atlite.Cutout(snakemake.input.cutout) grid_cells = cutout.grid.geometry diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index bf4a6118..e055a4c2 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -21,9 +21,13 @@ if __name__ == "__main__": set_scenario_config(snakemake) config = snakemake.config["energy"] - data_year = int(config["energy_totals_year"]) - if snakemake.wildcards.weather_year and snakemake.wildcards.kind == "heat": - data_year = int(snakemake.wildcards.weather_year) + + if snakemake.wildcards.kind == "heat": + years = pd.date_range(freq="YE", **snakemake.config["snapshots"]).year + assert len(years) == 1, "Currently only works for single year." + data_year = years[0] + else: + data_year = int(config["energy_totals_year"]) pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 3380a947..ae7ba2c2 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -229,14 +229,7 @@ if __name__ == "__main__": else: client = None - weather_year = snakemake.wildcards.weather_year - if weather_year: - snapshots = dict( - start=weather_year, end=str(int(weather_year) + 1), inclusive="left" - ) - else: - snapshots = snakemake.params.snapshots - sns = pd.date_range(freq="h", **snapshots) + sns = pd.date_range(freq="h", **snakemake.params.snapshots) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=sns) regions = gpd.read_file(snakemake.input.regions) diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index dc921ff5..c9eaffcc 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -32,20 +32,11 @@ if __name__ == "__main__": config = snakemake.params.solar_thermal config.pop("cutout", None) - cutout_name = snakemake.input.cutout - year = snakemake.wildcards.weather_year - - if year: - snapshots = dict(start=year, end=str(int(year) + 1), inclusive="left") - cutout_name = cutout_name.format(weather_year=year) - else: - snapshots = snakemake.params.snapshots - - time = pd.date_range(freq="h", **snapshots) + time = pd.date_range(freq="h", **snakemake.params.snapshots) if snakemake.params.drop_leap_day: time = time[~((time.month == 2) & (time.day == 29))] - cutout = atlite.Cutout(cutout_name).sel(time=time) + cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( gpd.read_file(snakemake.input.regions_onshore).set_index("name").buffer(0) diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index eefc2616..e5fa0b38 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -20,7 +20,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_temperature_profiles", - weather_year="", simpl="", clusters=48, ) @@ -29,20 +28,12 @@ if __name__ == "__main__": nprocesses = int(snakemake.threads) cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - cutout_name = snakemake.input.cutout - year = snakemake.wildcards.weather_year - if year: - snapshots = dict(start=year, end=str(int(year) + 1), inclusive="left") - cutout_name = cutout_name.format(weather_year=year) - else: - snapshots = snakemake.params.snapshots - - time = pd.date_range(freq="h", **snapshots) + time = pd.date_range(freq="h", **snakemake.params.snapshots) if snakemake.params.drop_leap_day: time = time[~((time.month == 2) & (time.day == 29))] - cutout = atlite.Cutout(cutout_name).sel(time=time) + cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( gpd.read_file(snakemake.input.regions_onshore).set_index("name").buffer(0) diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index 9a64801f..d3c740be 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -183,13 +183,7 @@ if __name__ == "__main__": options = snakemake.params.sector - year = snakemake.wildcards.weather_year - snapshots = ( - dict(start=year, end=str(int(year) + 1), inclusive="left") - if year - else snakemake.params.snapshots - ) - snapshots = pd.date_range(freq="h", **snapshots, tz="UTC") + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots, tz="UTC") if snakemake.params.drop_leap_day: leap_day = (snapshots.month == 2) & (snapshots.day == 29) snapshots = snapshots[~leap_day] From f4f4c1bf375eff2cedf15027afd889db1041a527 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 17:26:39 +0000 Subject: [PATCH 101/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_electricity.smk | 26 ++++++-------------------- rules/collect.smk | 7 ++----- rules/solve_electricity.smk | 18 +++++------------- scripts/build_hydro_profile.py | 2 +- 4 files changed, 14 insertions(+), 39 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index ab7b1913..a9499428 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -496,9 +496,7 @@ rule simplify_network: output: network=resources("networks/elec_s{simpl}.nc"), regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"), - regions_offshore=resources( - "regions_offshore_elec_s{simpl}.geojson" - ), + regions_offshore=resources("regions_offshore_elec_s{simpl}.geojson"), busmap=resources("busmap_elec_s{simpl}.csv"), connection_costs=resources("connection_costs_s{simpl}.csv"), log: @@ -532,9 +530,7 @@ rule cluster_network: input: network=resources("networks/elec_s{simpl}.nc"), regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"), - regions_offshore=resources( - "regions_offshore_elec_s{simpl}.geojson" - ), + regions_offshore=resources("regions_offshore_elec_s{simpl}.geojson"), busmap=ancient(resources("busmap_elec_s{simpl}.csv")), custom_busmap=lambda w: ( "data/custom_busmap_elec_s{simpl}_{clusters}.csv" @@ -546,12 +542,8 @@ rule cluster_network: ), output: network=resources("networks/elec_s{simpl}_{clusters}.nc"), - regions_onshore=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - regions_offshore=resources( - "regions_offshore_elec_s{simpl}_{clusters}.geojson" - ), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), + regions_offshore=resources("regions_offshore_elec_s{simpl}_{clusters}.geojson"), busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), linemap=resources("linemap_elec_s{simpl}_{clusters}.csv"), log: @@ -615,15 +607,9 @@ rule prepare_network: output: resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), log: - logs( - "prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" - ), + logs("prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log"), benchmark: - ( - benchmarks( - "prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" - ) - ) + (benchmarks("prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}")) threads: 1 resources: mem_mb=4000, diff --git a/rules/collect.smk b/rules/collect.smk index ef63df0f..214b8102 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -34,9 +34,7 @@ rule extra_components_networks: rule prepare_elec_networks: input: expand( - resources( - "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - ), + resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), **config["scenario"], run=config["run"]["name"], ), @@ -55,8 +53,7 @@ rule prepare_sector_networks: rule solve_elec_networks: input: expand( - RESULTS - + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config["scenario"], run=config["run"]["name"], ), diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 5c6777d4..81d07993 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -13,13 +13,10 @@ rule solve_network: ), custom_extra_functionality=input_custom_extra_functionality, input: - network=resources( - "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - ), + network=resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), config=RESULTS + "config.yaml", output: - network=RESULTS - + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", log: solver=normpath( RESULTS @@ -28,10 +25,7 @@ rule solve_network: python=RESULTS + "logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", benchmark: - ( - RESULTS - + "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" - ) + (RESULTS + "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}") threads: solver_threads resources: mem_mb=memory, @@ -48,11 +42,9 @@ rule solve_operations_network: params: options=config_provider("solving", "options"), input: - network=RESULTS - + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: - network=RESULTS - + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc", + network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc", log: solver=normpath( RESULTS diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index a732cf45..ce8f3a4c 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -65,8 +65,8 @@ import atlite import country_converter as coco import geopandas as gpd import pandas as pd -from numpy.polynomial import Polynomial from _helpers import configure_logging, set_scenario_config +from numpy.polynomial import Polynomial cc = coco.CountryConverter() From 9d913bcb38aba2165021296523649f4921df7cc2 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 4 Mar 2024 19:36:41 +0100 Subject: [PATCH 102/173] smaller fixes and reduction of diff --- doc/configtables/load.csv | 1 - rules/build_electricity.smk | 28 +++++-------------- rules/build_sector.smk | 14 +++++----- rules/postprocess.smk | 2 +- scripts/build_district_heat_share.py | 1 - ...build_industrial_energy_demand_per_node.py | 1 - ...build_population_weighted_energy_totals.py | 2 +- scripts/cluster_network.py | 8 ++---- scripts/make_summary.py | 2 +- scripts/simplify_network.py | 2 +- 10 files changed, 20 insertions(+), 41 deletions(-) diff --git a/doc/configtables/load.csv b/doc/configtables/load.csv index 204b79d8..a4de8323 100644 --- a/doc/configtables/load.csv +++ b/doc/configtables/load.csv @@ -1,5 +1,4 @@ ,Unit,Values,Description -url,--,string,"Link to open power system data time series data." power_statistics,bool,"{true, false}",Whether to load the electricity consumption data of the ENTSOE power statistics (only for files from 2019 and before) or from the ENTSOE transparency data (only has load data from 2015 onwards). interpolate_limit,hours,integer,"Maximum gap size (consecutive nans) which interpolated linearly." time_shift_for_large_gaps,string,string,"Periods which are used for copying time-slices in order to fill large gaps of nans. Have to be valid ``pandas`` period strings." diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index ab7b1913..b2b5b3d5 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -409,7 +409,7 @@ rule build_line_rating: def input_profile_tech(w): return { - f"profile_{tech}": resources(f"profile{}_{tech}.nc") + f"profile_{tech}": resources(f"profile_{tech}.nc") for tech in config_provider("electricity", "renewable_carriers")(w) } @@ -496,9 +496,7 @@ rule simplify_network: output: network=resources("networks/elec_s{simpl}.nc"), regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"), - regions_offshore=resources( - "regions_offshore_elec_s{simpl}.geojson" - ), + regions_offshore=resources("regions_offshore_elec_s{simpl}.geojson"), busmap=resources("busmap_elec_s{simpl}.csv"), connection_costs=resources("connection_costs_s{simpl}.csv"), log: @@ -532,9 +530,7 @@ rule cluster_network: input: network=resources("networks/elec_s{simpl}.nc"), regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"), - regions_offshore=resources( - "regions_offshore_elec_s{simpl}.geojson" - ), + regions_offshore=resources("regions_offshore_elec_s{simpl}.geojson"), busmap=ancient(resources("busmap_elec_s{simpl}.csv")), custom_busmap=lambda w: ( "data/custom_busmap_elec_s{simpl}_{clusters}.csv" @@ -546,12 +542,8 @@ rule cluster_network: ), output: network=resources("networks/elec_s{simpl}_{clusters}.nc"), - regions_onshore=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - regions_offshore=resources( - "regions_offshore_elec_s{simpl}_{clusters}.geojson" - ), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), + regions_offshore=resources("regions_offshore_elec_s{simpl}_{clusters}.geojson"), busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), linemap=resources("linemap_elec_s{simpl}_{clusters}.csv"), log: @@ -615,15 +607,9 @@ rule prepare_network: output: resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), log: - logs( - "prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" - ), + logs("prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log"), benchmark: - ( - benchmarks( - "prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" - ) - ) + (benchmarks("prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}")) threads: 1 resources: mem_mb=4000, diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 686bb9a1..eecd0fce 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -160,9 +160,9 @@ rule build_daily_heat_demand: mem_mb=20000, threads: 8 log: - logs("build_daily_heat_demand_{scope}__{simpl}_{clusters}.loc"), + logs("build_daily_heat_demand_{scope}_{simpl}_{clusters}.loc"), benchmark: - benchmarks("build_daily_heat_demand/{scope}__s{simpl}_{clusters}") + benchmarks("build_daily_heat_demand/{scope}_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -206,9 +206,9 @@ rule build_temperature_profiles: mem_mb=20000, threads: 8 log: - logs("build_temperature_profiles_{scope}__{simpl}_{clusters}.log"), + logs("build_temperature_profiles_{scope}_{simpl}_{clusters}.log"), benchmark: - benchmarks("build_temperature_profiles/{scope}__s{simpl}_{clusters}") + benchmarks("build_temperature_profiles/{scope}_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -272,9 +272,9 @@ rule build_solar_thermal_profiles: mem_mb=20000, threads: 16 log: - logs("build_solar_thermal_profiles_{scope}__s{simpl}_{clusters}.log"), + logs("build_solar_thermal_profiles_{scope}_s{simpl}_{clusters}.log"), benchmark: - benchmarks("build_solar_thermal_profiles/{scope}__s{simpl}_{clusters}") + benchmarks("build_solar_thermal_profiles/{scope}_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -853,7 +853,7 @@ rule build_existing_heating_distribution: def input_profile_offwind(w): return { - f"profile_{tech}": resources(f"profile{}_{tech}.nc") + f"profile_{tech}": resources(f"profile_{tech}.nc") for tech in ["offwind-ac", "offwind-dc"] if (tech in config_provider("electricity", "renewable_carriers")(w)) } diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 2eba1591..ac7bf983 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -297,7 +297,7 @@ rule plot_elec_statistics: output: **{ f"{plot}_bar": RESULTS - + f"figures/statistics_{plot}_bar_elec{}_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" + + f"figures/statistics_{plot}_bar_elec_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" for plot in STATISTICS_BARPLOTS }, barplots_touch=RESULTS diff --git a/scripts/build_district_heat_share.py b/scripts/build_district_heat_share.py index 0adf81ed..3cf88466 100644 --- a/scripts/build_district_heat_share.py +++ b/scripts/build_district_heat_share.py @@ -35,7 +35,6 @@ if __name__ == "__main__": district_heat_share = pd.read_csv(snakemake.input.district_heat_share, index_col=0)[ "district heat share" ] - district_heat_share = district_heat_share.groupby("country").max() # make ct-based share nodal district_heat_share = district_heat_share.reindex(pop_layout.ct).fillna(0) diff --git a/scripts/build_industrial_energy_demand_per_node.py b/scripts/build_industrial_energy_demand_per_node.py index 3987c975..ce72ea7a 100644 --- a/scripts/build_industrial_energy_demand_per_node.py +++ b/scripts/build_industrial_energy_demand_per_node.py @@ -15,7 +15,6 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_industrial_energy_demand_per_node", - weather_year="", simpl="", clusters=48, planning_horizons=2030, diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index e055a4c2..b26f289d 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -23,7 +23,7 @@ if __name__ == "__main__": config = snakemake.config["energy"] if snakemake.wildcards.kind == "heat": - years = pd.date_range(freq="YE", **snakemake.config["snapshots"]).year + years = pd.date_range(freq="YE", **snakemake.params.snapshots).year assert len(years) == 1, "Currently only works for single year." data_year = years[0] else: diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 47b65359..270774af 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -34,8 +34,8 @@ Inputs - ``resources/regions_onshore_elec_s{simpl}.geojson``: confer :ref:`simplify` - ``resources/regions_offshore_elec_s{simpl}.geojson``: confer :ref:`simplify` -- ``networks/elec_s{simpl}.nc``: confer :ref:`simplify` - ``resources/busmap_elec_s{simpl}.csv``: confer :ref:`simplify` +- ``networks/elec_s{simpl}.nc``: confer :ref:`simplify` - ``data/custom_busmap_elec_s{simpl}_{clusters}.csv``: optional input Outputs @@ -454,11 +454,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake( - "cluster_network", - simpl="", - clusters="5", - ) + snakemake = mock_snakemake("cluster_network", simpl="", clusters="5") configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 27ce91d3..b9e1a8af 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -681,7 +681,7 @@ if __name__ == "__main__": networks_dict = { (cluster, ll, opt + sector_opt, planning_horizon): "results/" + snakemake.params.RDIR - + f"postnetworks/elec_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc" + + f"/postnetworks/elec_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc" for simpl in snakemake.params.scenario["simpl"] for cluster in snakemake.params.scenario["clusters"] for opt in snakemake.params.scenario["opts"] diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 4de17704..eac026a4 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -65,7 +65,7 @@ Outputs .. image:: img/regions_offshore_elec_s .png :scale: 33 % -- ``resources/busmap_elec_s{simpl}.h5``: Mapping of buses from ``networks/elec.nc`` to ``networks/elec_s{simpl}.nc``; has keys ['/busmap_s'] +- ``resources/busmap_elec_s{simpl}.csv``: Mapping of buses from ``networks/elec.nc`` to ``networks/elec_s{simpl}.nc``; - ``networks/elec_s{simpl}.nc``: .. image:: img/elec_s.png From 96f71d7e9c7cb8d2d4d92e9f738ee6ff8231b926 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 4 Mar 2024 19:37:02 +0100 Subject: [PATCH 103/173] outsource drop_leap_day in temporal aggregation --- scripts/prepare_sector_network.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 3a1158b9..802066b9 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -813,7 +813,7 @@ def add_co2limit(n, options, nyears=1.0, limit=0.0): # TODO PyPSA-Eur merge issue -def average_every_nhours(n, offset, drop_leap_day=False): +def average_every_nhours(n, offset): logger.info(f"Resampling the network to {offset}") m = n.copy(with_time=False) @@ -832,10 +832,6 @@ def average_every_nhours(n, offset, drop_leap_day=False): else: pnl[k] = df.resample(offset).mean() - if drop_leap_day: - sns = m.snapshots[~((m.snapshots.month == 2) & (m.snapshots.day == 29))] - m.set_snapshots(sns) - return m @@ -3425,7 +3421,7 @@ def cluster_heat_buses(n): def apply_time_segmentation( - n, segments, solver_name="cbc", overwrite_time_dependent=True, drop_leap_day=False + n, segments, solver_name="cbc", overwrite_time_dependent=True ): """ Aggregating time series to segments with different lengths. @@ -3489,11 +3485,6 @@ def apply_time_segmentation( for component, key in values_t.columns.droplevel(2).unique(): n.pnl(component)[key] = values_t[component, key] - # TODO: need to check that this actually works as intended. - if drop_leap_day: - sns = n.snapshots[~((n.snapshots.month == 2) & (n.snapshots.day == 29))] - n.set_snapshots(sns) - return n @@ -3515,14 +3506,16 @@ def set_temporal_aggregation(n, resolution, solver_name, drop_leap_day=False): elif "seg" in resolution.lower(): segments = int(resolution[:-3]) logger.info("Use temporal segmentation with %s segments", segments) - n = apply_time_segmentation( - n, segments, solver_name=solver_name, drop_leap_day=drop_leap_day - ) + n = apply_time_segmentation(n, segments, solver_name=solver_name) # temporal averaging elif "h" in resolution.lower(): logger.info("Aggregate to frequency %s", resolution) - n = average_every_nhours(n, resolution, drop_leap_day) + n = average_every_nhours(n, resolution) + + if drop_leap_day: + sns = n.snapshots[~((n.snapshots.month == 2) & (n.snapshots.day == 29))] + n.set_snapshots(sns) return n From 8904943a87addc393cb23e7c88638323e12d3093 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 4 Mar 2024 19:37:52 +0100 Subject: [PATCH 104/173] integrate artificial load data supplement into build_electricity_demand --- config/config.default.yaml | 3 +- rules/build_electricity.smk | 59 +++++++++------------------ rules/retrieve.smk | 9 ++-- scripts/build_artificial_load_data.py | 42 ------------------- scripts/build_electricity_demand.py | 16 +++++--- 5 files changed, 35 insertions(+), 94 deletions(-) delete mode 100755 scripts/build_artificial_load_data.py diff --git a/config/config.default.yaml b/config/config.default.yaml index a9166560..d19b7f0e 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -73,8 +73,6 @@ enable: build_cutout: false retrieve_irena: false retrieve_cutout: true - retrieve_opsd_load_data: true - retrieve_artificial_load_data: false build_natura_raster: false retrieve_natura_raster: true custom_busmap: false @@ -295,6 +293,7 @@ load: manual_adjustments: true # false scaling_factor: 1.0 fixed_year: false # false or year (e.g. 2013) + supplement_missing_data_artificially: true # docs # TODO: PyPSA-Eur merge issue in prepare_sector_network.py diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index b2b5b3d5..e0517933 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -18,45 +18,26 @@ if config["enable"].get("prepare_links_p_nom", False): "../scripts/prepare_links_p_nom.py" -if config["enable"].get("retrieve_opsd_load_data", True): - - rule build_electricity_demand: - params: - snapshots=config_provider("snapshots"), - countries=config_provider("countries"), - load=config_provider("load"), - input: - ancient("data/electricity_demand_raw.csv"), - output: - resources("electricity_demand.csv"), - log: - logs("build_electricity_demand.log"), - resources: - mem_mb=5000, - conda: - "../envs/environment.yaml" - script: - "../scripts/build_electricity_demand.py" - - -if config["enable"].get("retrieve_artificial_load_data", False): - - rule build_artificial_load_data: - input: - ancient("data/load_artificial_raw.csv"), - output: - resources("electricity_demand.csv"), - log: - logs("build_artificial_load_data.log"), - resources: - mem_mb=5000, - conda: - "../envs/environment.yaml" - script: - "../scripts/build_artificial_load_data.py" - - -ruleorder: build_artificial_load_data > build_electricity_demand +rule build_electricity_demand: + params: + snapshots=config_provider("snapshots"), + countries=config_provider("countries"), + load=config_provider("load"), + input: + reported=ancient("data/electricity_demand_raw.csv"), + artificial=lambda w: ancient("data/load_artificial_raw.csv") + if config_provider("load", "supplement_missing_data_artificially")(w) + else [], + output: + resources("electricity_demand.csv"), + log: + logs("build_electricity_demand.log"), + resources: + mem_mb=5000, + conda: + "../envs/environment.yaml" + script: + "../scripts/build_electricity_demand.py" rule build_powerplants: diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 000d475e..24093670 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -180,9 +180,7 @@ if config["enable"]["retrieve"]: "../scripts/retrieve_gas_infrastructure_data.py" -if config["enable"]["retrieve"] and config["enable"].get( - "retrieve_opsd_load_data", True -): +if config["enable"]["retrieve"]: rule retrieve_electricity_demand: params: @@ -200,9 +198,7 @@ if config["enable"]["retrieve"] and config["enable"].get( "../scripts/retrieve_electricity_demand.py" -if config["enable"]["retrieve"] and config["enable"].get( - "retrieve_artificial_load_data", False -): +if config["enable"]["retrieve"]: rule retrieve_artificial_load_data: input: @@ -220,6 +216,7 @@ if config["enable"]["retrieve"] and config["enable"].get( retries: 2 run: move(input[0], output[0]) + validate_checksum(output[0], input[0]) if config["enable"]["retrieve"]: diff --git a/scripts/build_artificial_load_data.py b/scripts/build_artificial_load_data.py deleted file mode 100755 index a12ac917..00000000 --- a/scripts/build_artificial_load_data.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: 2022 The PyPSA-Eur Authors -# -# SPDX-License-Identifier: MIT -""" -This rule downloads the load data. -""" - -import logging - -logger = logging.getLogger(__name__) -import pandas as pd -from _helpers import configure_logging - -if __name__ == "__main__": - if "snakemake" not in globals(): - from _helpers import mock_snakemake - - snakemake = mock_snakemake("build_artificial_load_data") - - configure_logging(snakemake) - - snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) - - fixed_year = snakemake.config["load"].get("fixed_year", False) - years = ( - slice(str(fixed_year), str(fixed_year)) - if fixed_year - else slice(snapshots[0], snapshots[-1]) - ) - countries = snakemake.config["countries"] - - load = pd.read_csv(snakemake.input[0], index_col=0, parse_dates=True).loc[ - snapshots, countries - ] - - assert not load.isna().any().any(), "Load data contains nans." - - if fixed_year: - load.index = load.index.map(lambda t: t.replace(year=snapshots.year[0])) - - load.to_csv(snakemake.output[0]) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index f5bd744a..68df4bac 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -277,16 +277,15 @@ if __name__ == "__main__": time_shift = snakemake.params.load["time_shift_for_large_gaps"] - load = load_timeseries(snakemake.input[0], years, countries) + load = load_timeseries(snakemake.input.reported, years, countries) if "UA" in countries: # attach load of UA (best data only for entsoe transparency) - load_ua = load_timeseries(snakemake.input[0], "2018", ["UA"], False) + load_ua = load_timeseries(snakemake.input.reported, "2018", ["UA"]) snapshot_year = str(snapshots.year.unique().item()) time_diff = pd.Timestamp("2018") - pd.Timestamp(snapshot_year) - load_ua.index -= ( - time_diff # hack indices (currently, UA is manually set to 2018) - ) + # hack indices (currently, UA is manually set to 2018) + load_ua.index -= time_diff load["UA"] = load_ua # attach load of MD (no time-series available, use 2020-totals and distribute according to UA): # https://www.iea.org/data-and-statistics/data-browser/?country=MOLDOVA&fuel=Energy%20consumption&indicator=TotElecCons @@ -307,6 +306,13 @@ if __name__ == "__main__": ) load = load.apply(fill_large_gaps, shift=time_shift) + if snakemake.params.load["supplement_missing_data_artificially"]: + logger.info("Supplement missing data with artificial data.") + fn = snakemake.input.artificial + artificial_load = pd.read_csv(fn, index_col=0, parse_dates=True) + artificial_load = artificial_load.loc[snapshots, countries] + load = load.combine_first(artificial_load) + assert not load.isna().any().any(), ( "Load data contains nans. Adjust the parameters " "`time_shift_for_large_gaps` or modify the `manual_adjustment` function " From 8aa6d98f5635064262b43bac526674b3fe4ac631 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 18:38:29 +0000 Subject: [PATCH 105/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_electricity.smk | 8 +++++--- rules/postprocess.smk | 23 ++++++----------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index e0517933..a9e111ea 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -25,9 +25,11 @@ rule build_electricity_demand: load=config_provider("load"), input: reported=ancient("data/electricity_demand_raw.csv"), - artificial=lambda w: ancient("data/load_artificial_raw.csv") - if config_provider("load", "supplement_missing_data_artificially")(w) - else [], + artificial=lambda w: ( + ancient("data/load_artificial_raw.csv") + if config_provider("load", "supplement_missing_data_artificially")(w) + else [] + ), output: resources("electricity_demand.csv"), log: diff --git a/rules/postprocess.smk b/rules/postprocess.smk index ac7bf983..dc08699f 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -23,9 +23,7 @@ if config["foresight"] != "perfect": resources: mem_mb=4000, benchmark: - benchmarks( - "plot_power_network_clustered/elec_s{simpl}_{clusters}" - ) + benchmarks("plot_power_network_clustered/elec_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -37,9 +35,7 @@ if config["foresight"] != "perfect": input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - regions=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), output: map=RESULTS + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", @@ -66,9 +62,7 @@ if config["foresight"] != "perfect": input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - regions=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), output: map=RESULTS + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf", @@ -94,9 +88,7 @@ if config["foresight"] != "perfect": input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - regions=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), output: map=RESULTS + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf", @@ -133,9 +125,7 @@ if config["foresight"] == "perfect": input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", - regions=resources( - "regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), + regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), output: unpack(output_map_year), threads: 2 @@ -292,8 +282,7 @@ rule plot_elec_statistics: plotting=config_provider("plotting"), barplots=STATISTICS_BARPLOTS, input: - network=RESULTS - + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: **{ f"{plot}_bar": RESULTS From 3af878d69c9ba3968b828f1270383cf7ca6072a1 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 5 Mar 2024 10:33:41 +0100 Subject: [PATCH 106/173] remove power statistics references --- config/config.default.yaml | 1 - config/config.validation.yaml | 3 --- doc/configtables/load.csv | 1 - 3 files changed, 5 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index d19b7f0e..99ebae88 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -287,7 +287,6 @@ transformers: # docs-load in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#load load: - power_statistics: true interpolate_limit: 3 time_shift_for_large_gaps: 1w manual_adjustments: true # false diff --git a/config/config.validation.yaml b/config/config.validation.yaml index c8fb2fa5..062e82f5 100644 --- a/config/config.validation.yaml +++ b/config/config.validation.yaml @@ -69,9 +69,6 @@ conventional: biomass: p_max_pu: 0.65 -load: - power_statistics: false - lines: s_max_pu: 0.23 under_construction: 'remove' diff --git a/doc/configtables/load.csv b/doc/configtables/load.csv index a4de8323..0a62aeef 100644 --- a/doc/configtables/load.csv +++ b/doc/configtables/load.csv @@ -1,5 +1,4 @@ ,Unit,Values,Description -power_statistics,bool,"{true, false}",Whether to load the electricity consumption data of the ENTSOE power statistics (only for files from 2019 and before) or from the ENTSOE transparency data (only has load data from 2015 onwards). interpolate_limit,hours,integer,"Maximum gap size (consecutive nans) which interpolated linearly." time_shift_for_large_gaps,string,string,"Periods which are used for copying time-slices in order to fill large gaps of nans. Have to be valid ``pandas`` period strings." manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`." From 7abbb47efda6a51d1d1ef1b6f66e5a04bb4919e6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 6 Mar 2024 16:42:33 +0100 Subject: [PATCH 107/173] revert to master version of build_energy_totals --- scripts/build_energy_totals.py | 208 +++++++++++++-------------------- 1 file changed, 83 insertions(+), 125 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index b14644d5..1ffc4ae2 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -122,27 +122,16 @@ def build_eurostat(input_eurostat, countries, year): # convert to TWh/a from ktoe/a df *= 11.63 / 1e3 - df.index = df.index.set_levels(df.index.levels[1].astype(int), level=1) - - if year: - df = df.xs(year, level="year") - return df -def build_swiss(year=None): +def build_swiss(year): """ - Return a pd.DataFrame of Swiss energy data in TWh/a. + Return a pd.Series of Swiss energy data in TWh/a. """ - fn = snakemake.input.swiss - df = pd.read_csv(fn, index_col=[0, 1]).stack().unstack("item") - df.index.names = ["country", "year"] - df.index = df.index.set_levels(df.index.levels[1].astype(int), level=1) - - if year: - df = df.xs(year, level="year") + df = pd.read_csv(fn, index_col=[0, 1]).loc["CH", str(year)] # convert PJ/a to TWh/a df /= 3.6 @@ -151,7 +140,6 @@ def build_swiss(year=None): def idees_per_country(ct, year, base_dir): - ct_totals = {} ct_idees = idees_rename.get(ct, ct) fn_residential = f"{base_dir}/JRC-IDEES-2015_Residential_{ct_idees}.xlsx" fn_tertiary = f"{base_dir}/JRC-IDEES-2015_Tertiary_{ct_idees}.xlsx" @@ -159,27 +147,27 @@ def idees_per_country(ct, year, base_dir): # residential - df = pd.read_excel(fn_residential, "RES_hh_fec", index_col=0) - - ct_totals["total residential space"] = df.loc["Space heating"] + df = pd.read_excel(fn_residential, "RES_hh_fec", index_col=0)[year] rows = ["Advanced electric heating", "Conventional electric heating"] - ct_totals["electricity residential space"] = df.loc[rows].sum() - - ct_totals["total residential water"] = df.loc["Water heating"] + ct_totals = { + "total residential space": df["Space heating"], + "electricity residential space": df[rows].sum(), + } + ct_totals["total residential water"] = df.at["Water heating"] assert df.index[23] == "Electricity" ct_totals["electricity residential water"] = df.iloc[23] - ct_totals["total residential cooking"] = df.loc["Cooking"] + ct_totals["total residential cooking"] = df["Cooking"] assert df.index[30] == "Electricity" ct_totals["electricity residential cooking"] = df.iloc[30] - df = pd.read_excel(fn_residential, "RES_summary", index_col=0) + df = pd.read_excel(fn_residential, "RES_summary", index_col=0)[year] row = "Energy consumption by fuel - Eurostat structure (ktoe)" - ct_totals["total residential"] = df.loc[row] + ct_totals["total residential"] = df[row] assert df.index[47] == "Electricity" ct_totals["electricity residential"] = df.iloc[47] @@ -192,27 +180,27 @@ def idees_per_country(ct, year, base_dir): # services - df = pd.read_excel(fn_tertiary, "SER_hh_fec", index_col=0) + df = pd.read_excel(fn_tertiary, "SER_hh_fec", index_col=0)[year] - ct_totals["total services space"] = df.loc["Space heating"] + ct_totals["total services space"] = df["Space heating"] rows = ["Advanced electric heating", "Conventional electric heating"] - ct_totals["electricity services space"] = df.loc[rows].sum() + ct_totals["electricity services space"] = df[rows].sum() - ct_totals["total services water"] = df.loc["Hot water"] + ct_totals["total services water"] = df["Hot water"] assert df.index[24] == "Electricity" ct_totals["electricity services water"] = df.iloc[24] - ct_totals["total services cooking"] = df.loc["Catering"] + ct_totals["total services cooking"] = df["Catering"] assert df.index[31] == "Electricity" ct_totals["electricity services cooking"] = df.iloc[31] - df = pd.read_excel(fn_tertiary, "SER_summary", index_col=0) + df = pd.read_excel(fn_tertiary, "SER_summary", index_col=0)[year] row = "Energy consumption by fuel - Eurostat structure (ktoe)" - ct_totals["total services"] = df.loc[row] + ct_totals["total services"] = df[row] assert df.index[50] == "Electricity" ct_totals["electricity services"] = df.iloc[50] @@ -228,7 +216,7 @@ def idees_per_country(ct, year, base_dir): start = "Detailed split of energy consumption (ktoe)" end = "Market shares of energy uses (%)" - df = pd.read_excel(fn_tertiary, "AGR_fec", index_col=0).loc[start:end] + df = pd.read_excel(fn_tertiary, "AGR_fec", index_col=0).loc[start:end, year] rows = [ "Lighting", @@ -236,30 +224,30 @@ def idees_per_country(ct, year, base_dir): "Specific electricity uses", "Pumping devices (electric)", ] - ct_totals["total agriculture electricity"] = df.loc[rows].sum() + ct_totals["total agriculture electricity"] = df[rows].sum() rows = ["Specific heat uses", "Low enthalpy heat"] - ct_totals["total agriculture heat"] = df.loc[rows].sum() + ct_totals["total agriculture heat"] = df[rows].sum() rows = [ "Motor drives", "Farming machine drives (diesel oil incl. biofuels)", "Pumping devices (diesel oil incl. biofuels)", ] - ct_totals["total agriculture machinery"] = df.loc[rows].sum() + ct_totals["total agriculture machinery"] = df[rows].sum() row = "Agriculture, forestry and fishing" - ct_totals["total agriculture"] = df.loc[row] + ct_totals["total agriculture"] = df[row] # transport - df = pd.read_excel(fn_transport, "TrRoad_ene", index_col=0) + df = pd.read_excel(fn_transport, "TrRoad_ene", index_col=0)[year] - ct_totals["total road"] = df.loc["by fuel (EUROSTAT DATA)"] + ct_totals["total road"] = df["by fuel (EUROSTAT DATA)"] - ct_totals["electricity road"] = df.loc["Electricity"] + ct_totals["electricity road"] = df["Electricity"] - ct_totals["total two-wheel"] = df.loc["Powered 2-wheelers (Gasoline)"] + ct_totals["total two-wheel"] = df["Powered 2-wheelers (Gasoline)"] assert df.index[19] == "Passenger cars" ct_totals["total passenger cars"] = df.iloc[19] @@ -280,16 +268,16 @@ def idees_per_country(ct, year, base_dir): ct_totals["electricity light duty road freight"] = df.iloc[49] row = "Heavy duty vehicles (Diesel oil incl. biofuels)" - ct_totals["total heavy duty road freight"] = df.loc[row] + ct_totals["total heavy duty road freight"] = df[row] assert df.index[61] == "Passenger cars" ct_totals["passenger car efficiency"] = df.iloc[61] - df = pd.read_excel(fn_transport, "TrRail_ene", index_col=0) + df = pd.read_excel(fn_transport, "TrRail_ene", index_col=0)[year] - ct_totals["total rail"] = df.loc["by fuel (EUROSTAT DATA)"] + ct_totals["total rail"] = df["by fuel (EUROSTAT DATA)"] - ct_totals["electricity rail"] = df.loc["Electricity"] + ct_totals["electricity rail"] = df["Electricity"] assert df.index[15] == "Passenger transport" ct_totals["total rail passenger"] = df.iloc[15] @@ -305,7 +293,7 @@ def idees_per_country(ct, year, base_dir): assert df.index[23] == "Electric" ct_totals["electricity rail freight"] = df.iloc[23] - df = pd.read_excel(fn_transport, "TrAvia_ene", index_col=0) + df = pd.read_excel(fn_transport, "TrAvia_ene", index_col=0)[year] assert df.index[6] == "Passenger transport" ct_totals["total aviation passenger"] = df.iloc[6] @@ -336,21 +324,20 @@ def idees_per_country(ct, year, base_dir): + ct_totals["total international aviation passenger"] ) - df = pd.read_excel(fn_transport, "TrNavi_ene", index_col=0) + df = pd.read_excel(fn_transport, "TrNavi_ene", index_col=0)[year] # coastal and inland - ct_totals["total domestic navigation"] = df.loc["by fuel (EUROSTAT DATA)"] + ct_totals["total domestic navigation"] = df["by fuel (EUROSTAT DATA)"] - df = pd.read_excel(fn_transport, "TrRoad_act", index_col=0) + df = pd.read_excel(fn_transport, "TrRoad_act", index_col=0)[year] assert df.index[85] == "Passenger cars" ct_totals["passenger cars"] = df.iloc[85] - return pd.DataFrame(ct_totals) + return pd.Series(ct_totals, name=ct) -def build_idees(countries, year=None): - +def build_idees(countries, year): nprocesses = snakemake.threads disable_progress = snakemake.config["run"].get("disable_progressbar", False) @@ -362,49 +349,35 @@ def build_idees(countries, year=None): desc="Build from IDEES database", disable=disable_progress, ) - with mute_print(): with mp.Pool(processes=nprocesses) as pool: totals_list = list(tqdm(pool.imap(func, countries), **tqdm_kwargs)) - totals = pd.concat(totals_list, keys=countries, names=["country", "year"]) + totals = pd.concat(totals_list, axis=1) # convert ktoe to TWh - exclude = totals.columns.str.fullmatch("passenger cars") - totals.loc[:, ~exclude] *= 11.63 / 1e3 + exclude = totals.index.str.fullmatch("passenger cars") + totals.loc[~exclude] *= 11.63 / 1e3 # convert TWh/100km to kWh/km - totals.loc[:, "passenger car efficiency"] *= 10 + totals.loc["passenger car efficiency"] *= 10 - return totals + return totals.T def build_energy_totals(countries, eurostat, swiss, idees): - - eurostat_fuels = dict(electricity="Electricity", total="Total") - - eurostat_sectors = dict( - residential="Households", - services="Commercial & public services", - road="Road", - rail="Rail", - ) + eurostat_fuels = {"electricity": "Electricity", "total": "Total all products"} to_drop = ["passenger cars", "passenger car efficiency"] - - new_index = pd.MultiIndex.from_product( - [countries, eurostat.index.levels[1]], names=["country", "year"] - ) - - df = idees.reindex(new_index).drop(to_drop, axis=1) + df = idees.reindex(countries).drop(to_drop, axis=1) eurostat_countries = eurostat.index.levels[0] - in_eurostat = df.index.levels[0].intersection(eurostat_countries) + in_eurostat = df.index.intersection(eurostat_countries) # add international navigation - slicer = idx[in_eurostat, :, :, "International maritime bunkers", :] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0, 1]).sum() + slicer = idx[in_eurostat, :, "Bunkers", :] + fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() df.loc[in_eurostat, "total international navigation"] = fill_values # add swiss energy data @@ -414,24 +387,19 @@ def build_energy_totals(countries, eurostat, swiss, idees): # get values for missing countries based on Eurostat EnergyBalances # divide cooking/space/water according to averages in EU28 - to_fill = df.index[ - df["total residential"].isna() - & df.index.get_level_values("country").isin(eurostat_countries) - ] + missing = df.index[df["total residential"].isna()] + to_fill = missing.intersection(eurostat_countries) uses = ["space", "cooking", "water"] - c = to_fill.get_level_values("country") - y = to_fill.get_level_values("year") - for sector in ["residential", "services", "road", "rail"]: + eurostat_sector = sector.capitalize() # fuel use for fuel in ["electricity", "total"]: - - slicer = idx[c, y, :, :, eurostat_sectors[sector]] + slicer = idx[to_fill, :, :, eurostat_sector] fill_values = ( - eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=[0, 1]).sum() + eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=0).sum() ) df.loc[to_fill, f"{fuel} {sector}"] = fill_values @@ -493,32 +461,30 @@ def build_energy_totals(countries, eurostat, swiss, idees): no_norway[f"total {sector}"] - no_norway[f"electricity {sector}"] ) fraction = nonelectric_use.div(nonelectric).mean() - df.loc["NO", f"total {sector} {use}"] = ( - total_heating * fraction - ).values + df.loc["NO", f"total {sector} {use}"] = total_heating * fraction df.loc["NO", f"electricity {sector} {use}"] = ( total_heating * fraction * elec_fraction - ).values + ) # Missing aviation - slicer = idx[c, y, :, :, "Domestic aviation"] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0, 1]).sum() + slicer = idx[to_fill, :, :, "Domestic aviation"] + fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() df.loc[to_fill, "total domestic aviation"] = fill_values - slicer = idx[c, y, :, "International aviation", :] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0, 1]).sum() + slicer = idx[to_fill, :, :, "International aviation"] + fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() df.loc[to_fill, "total international aviation"] = fill_values # missing domestic navigation - slicer = idx[c, y, :, :, "Domestic navigation"] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0, 1]).sum() + slicer = idx[to_fill, :, :, "Domestic Navigation"] + fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() df.loc[to_fill, "total domestic navigation"] = fill_values # split road traffic for non-IDEES missing = df.index[df["total passenger cars"].isna()] - for fuel in ["electricity", "total"]: + for fuel in ["total", "electricity"]: selection = [ f"{fuel} passenger cars", f"{fuel} other road passenger", @@ -564,9 +530,11 @@ def build_energy_totals(countries, eurostat, swiss, idees): ) if "BA" in df.index: - # fill missing data for BA proportional to RS - ratio = (df.loc["BA"].loc[2014:2020] / df.loc["RS"].loc[2014:2020]).mean() - df.loc["BA"] = (ratio * df.loc["RS"]).values + # fill missing data for BA (services and road energy data) + # proportional to RS with ratio of total residential demand + missing = df.loc["BA"] == 0.0 + ratio = df.at["BA", "total residential"] / df.at["RS", "total residential"] + df.loc["BA", missing] = ratio * df.loc["RS", missing] return df @@ -582,21 +550,18 @@ def build_district_heat_share(countries, idees): district_heat_share = district_heat / total_heat - district_heat_share = district_heat_share.reindex(countries, level="country") + district_heat_share = district_heat_share.reindex(countries) # Missing district heating share - dh_share = pd.read_csv( - snakemake.input.district_heat_share, index_col=0, usecols=[0, 1] + dh_share = ( + pd.read_csv(snakemake.input.district_heat_share, index_col=0, usecols=[0, 1]) + .div(100) + .squeeze() ) - - dh_share = pd.concat( - {y: dh_share for y in range(1990, 2021)}, names=["year", "country"] - ).swaplevel() - dh_share = dh_share.div(100) - dh_share = dh_share.reindex(district_heat_share.index) - # make conservative assumption and take minimum from both data sets - district_heat_share = pd.concat([district_heat_share, dh_share], axis=1).min(axis=1) + district_heat_share = pd.concat( + [district_heat_share, dh_share.reindex_like(district_heat_share)], axis=1 + ).min(axis=1) district_heat_share.name = "district heat share" @@ -613,6 +578,8 @@ def build_eea_co2(input_co2, year=1990, emissions_scope="CO2"): index_col = ["Country_code", "Pollutant_name", "Year", "Sector_name"] df = df.set_index(index_col).sort_index() + emissions_scope = emissions_scope + cts = ["CH", "EUA", "NO"] + eu28_eea slicer = idx[cts, emissions_scope, year, to_ipcc.values()] @@ -658,26 +625,19 @@ def build_eea_co2(input_co2, year=1990, emissions_scope="CO2"): def build_eurostat_co2(input_eurostat, countries, year=1990): eurostat = build_eurostat(input_eurostat, countries, year) - if eurostat is None: - df = build_eurostat(countries, year) - else: - df = eurostat.xs(year, level="year") - - specific_emissions = pd.Series(index=df.columns, dtype=float) + specific_emissions = pd.Series(index=eurostat.columns, dtype=float) # emissions in tCO2_equiv per MWh_th - specific_emissions["Solid fossil fuels"] = 0.36 # Approximates coal - specific_emissions["Oil and petroleum products"] = ( - 0.285 # Average of distillate and residue - ) - specific_emissions["Natural gas"] = 0.2 # For natural gas + specific_emissions["Solid fuels"] = 0.36 # Approximates coal + specific_emissions["Oil (total)"] = 0.285 # Average of distillate and residue + specific_emissions["Gas"] = 0.2 # For natural gas # oil values from https://www.eia.gov/tools/faqs/faq.cfm?id=74&t=11 # Distillate oil (No. 2) 0.276 # Residual oil (No. 6) 0.298 # https://www.eia.gov/electricity/annual/html/epa_a_03.html - return df.multiply(specific_emissions).sum(axis=1) + return eurostat.multiply(specific_emissions).sum(axis=1) def build_co2_totals(countries, eea_co2, eurostat_co2): @@ -884,7 +844,6 @@ if __name__ == "__main__": countries = snakemake.params.countries idees_countries = pd.Index(countries).intersection(eu28) - countries_without_ch = pd.Index(countries).difference(["CH"]) data_year = params["energy_totals_year"] input_eurostat = snakemake.input.eurostat @@ -917,6 +876,5 @@ if __name__ == "__main__": co2 = build_co2_totals(countries, eea_co2, eurostat_co2) co2.to_csv(snakemake.output.co2_name) - idees_transport = idees.xs(data_year, level="year") - transport = build_transport_data(countries, population, idees_transport) + transport = build_transport_data(countries, population, idees) transport.to_csv(snakemake.output.transport_name) From a8682db4fc14c0f6688cccf1de53d6555063c71d Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 6 Mar 2024 20:42:45 +0100 Subject: [PATCH 108/173] build_energy_totals: pre-process data for all years first [incomplete] --- scripts/build_energy_totals.py | 303 ++++++++++++++++++++------------- 1 file changed, 186 insertions(+), 117 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 1ffc4ae2..8e9abb69 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -69,38 +69,58 @@ to_ipcc = { } -def build_eurostat(input_eurostat, countries, year): +def eurostat_per_country(input_eurostat, country): + filename = ( + f"{input_eurostat}/{country}-Energy-balance-sheets-April-2023-edition.xlsb" + ) + sheet = pd.read_excel( + filename, + engine="pyxlsb", + sheet_name=None, + skiprows=4, + index_col=list(range(4)), + ) + sheet.pop("Cover") + return pd.concat(sheet) + + +def build_eurostat(input_eurostat, countries): """ Return multi-index for all countries' energy data in TWh/a. """ - df = {} countries = {idees_rename.get(country, country) for country in countries} - {"CH"} - for country in countries: - filename = ( - f"{input_eurostat}/{country}-Energy-balance-sheets-April-2023-edition.xlsb" - ) - sheet = pd.read_excel( - filename, - engine="pyxlsb", - sheet_name=str(year), - skiprows=4, - index_col=list(range(4)), - ) - df[country] = sheet - df = pd.concat(df, axis=0) + + nprocesses = snakemake.threads + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + + func = partial(eurostat_per_country, input_eurostat) + tqdm_kwargs = dict( + ascii=False, + unit=" country", + total=len(countries), + desc="Build from eurostat database", + disable=disable_progress, + ) + with mute_print(): + with mp.Pool(processes=nprocesses) as pool: + dfs = list(tqdm(pool.imap(func, countries), **tqdm_kwargs)) + + index_names = ["country", "year", "lvl1", "lvl2", "lvl3", "lvl4"] + df = pd.concat(dfs, keys=countries, names=index_names) + df.index = df.index.set_levels(df.index.levels[1].astype(int), level=1) # drop columns with all NaNs unnamed_cols = df.columns[df.columns.astype(str).str.startswith("Unnamed")] df.drop(unnamed_cols, axis=1, inplace=True) - df.drop(year, axis=1, inplace=True) + df.drop(list(range(1990, 2022)), axis=1, inplace=True, errors="ignore") # make numeric values where possible df.replace("Z", 0, inplace=True) df = df.apply(pd.to_numeric, errors="coerce") df = df.select_dtypes(include=[np.number]) - # write 'International aviation' to the 2nd level of the multiindex - int_avia = df.index.get_level_values(2) == "International aviation" + # write 'International aviation' to the lower level of the multiindex + int_avia = df.index.get_level_values(3) == "International aviation" temp = df.loc[int_avia] temp.index = pd.MultiIndex.from_frame( temp.index.to_frame().fillna("International aviation") @@ -113,11 +133,11 @@ def build_eurostat(input_eurostat, countries, year): "Commercial & public services": "Services", "Domestic navigation": "Domestic Navigation", "International maritime bunkers": "Bunkers", + "UK": "GB", } - columns_rename = {"Total": "Total all products", "UK": "GB"} + columns_rename = {"Total": "Total all products"} df.rename(index=index_rename, columns=columns_rename, inplace=True) df.sort_index(inplace=True) - df.index.names = [None] * len(df.index.names) # convert to TWh/a from ktoe/a df *= 11.63 / 1e3 @@ -125,13 +145,21 @@ def build_eurostat(input_eurostat, countries, year): return df -def build_swiss(year): +def build_swiss(): """ - Return a pd.Series of Swiss energy data in TWh/a. + Return a pd.DataFrame of Swiss energy data in TWh/a. """ fn = snakemake.input.swiss - df = pd.read_csv(fn, index_col=[0, 1]).loc["CH", str(year)] + df = pd.read_csv(fn, index_col=[0, 1]) + + df.columns = df.columns.astype(int) + + df.columns.name = "year" + + df = df.stack().unstack("item") + + df.columns.name = None # convert PJ/a to TWh/a df /= 3.6 @@ -139,35 +167,35 @@ def build_swiss(year): return df -def idees_per_country(ct, year, base_dir): +def idees_per_country(ct, base_dir): ct_idees = idees_rename.get(ct, ct) fn_residential = f"{base_dir}/JRC-IDEES-2015_Residential_{ct_idees}.xlsx" fn_tertiary = f"{base_dir}/JRC-IDEES-2015_Tertiary_{ct_idees}.xlsx" fn_transport = f"{base_dir}/JRC-IDEES-2015_Transport_{ct_idees}.xlsx" + ct_totals = {} + # residential - df = pd.read_excel(fn_residential, "RES_hh_fec", index_col=0)[year] + df = pd.read_excel(fn_residential, "RES_hh_fec", index_col=0) rows = ["Advanced electric heating", "Conventional electric heating"] - ct_totals = { - "total residential space": df["Space heating"], - "electricity residential space": df[rows].sum(), - } - ct_totals["total residential water"] = df.at["Water heating"] + ct_totals["electricity residential space"] = df.loc[rows].sum() + ct_totals["total residential space"] = df.loc["Space heating"] + ct_totals["total residential water"] = df.loc["Water heating"] assert df.index[23] == "Electricity" ct_totals["electricity residential water"] = df.iloc[23] - ct_totals["total residential cooking"] = df["Cooking"] + ct_totals["total residential cooking"] = df.loc["Cooking"] assert df.index[30] == "Electricity" ct_totals["electricity residential cooking"] = df.iloc[30] - df = pd.read_excel(fn_residential, "RES_summary", index_col=0)[year] + df = pd.read_excel(fn_residential, "RES_summary", index_col=0) row = "Energy consumption by fuel - Eurostat structure (ktoe)" - ct_totals["total residential"] = df[row] + ct_totals["total residential"] = df.loc[row] assert df.index[47] == "Electricity" ct_totals["electricity residential"] = df.iloc[47] @@ -180,27 +208,27 @@ def idees_per_country(ct, year, base_dir): # services - df = pd.read_excel(fn_tertiary, "SER_hh_fec", index_col=0)[year] + df = pd.read_excel(fn_tertiary, "SER_hh_fec", index_col=0) - ct_totals["total services space"] = df["Space heating"] + ct_totals["total services space"] = df.loc["Space heating"] rows = ["Advanced electric heating", "Conventional electric heating"] - ct_totals["electricity services space"] = df[rows].sum() + ct_totals["electricity services space"] = df.loc[rows].sum() - ct_totals["total services water"] = df["Hot water"] + ct_totals["total services water"] = df.loc["Hot water"] assert df.index[24] == "Electricity" ct_totals["electricity services water"] = df.iloc[24] - ct_totals["total services cooking"] = df["Catering"] + ct_totals["total services cooking"] = df.loc["Catering"] assert df.index[31] == "Electricity" ct_totals["electricity services cooking"] = df.iloc[31] - df = pd.read_excel(fn_tertiary, "SER_summary", index_col=0)[year] + df = pd.read_excel(fn_tertiary, "SER_summary", index_col=0) row = "Energy consumption by fuel - Eurostat structure (ktoe)" - ct_totals["total services"] = df[row] + ct_totals["total services"] = df.loc[row] assert df.index[50] == "Electricity" ct_totals["electricity services"] = df.iloc[50] @@ -216,7 +244,7 @@ def idees_per_country(ct, year, base_dir): start = "Detailed split of energy consumption (ktoe)" end = "Market shares of energy uses (%)" - df = pd.read_excel(fn_tertiary, "AGR_fec", index_col=0).loc[start:end, year] + df = pd.read_excel(fn_tertiary, "AGR_fec", index_col=0).loc[start:end] rows = [ "Lighting", @@ -224,30 +252,30 @@ def idees_per_country(ct, year, base_dir): "Specific electricity uses", "Pumping devices (electric)", ] - ct_totals["total agriculture electricity"] = df[rows].sum() + ct_totals["total agriculture electricity"] = df.loc[rows].sum() rows = ["Specific heat uses", "Low enthalpy heat"] - ct_totals["total agriculture heat"] = df[rows].sum() + ct_totals["total agriculture heat"] = df.loc[rows].sum() rows = [ "Motor drives", "Farming machine drives (diesel oil incl. biofuels)", "Pumping devices (diesel oil incl. biofuels)", ] - ct_totals["total agriculture machinery"] = df[rows].sum() + ct_totals["total agriculture machinery"] = df.loc[rows].sum() row = "Agriculture, forestry and fishing" - ct_totals["total agriculture"] = df[row] + ct_totals["total agriculture"] = df.loc[row] # transport - df = pd.read_excel(fn_transport, "TrRoad_ene", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrRoad_ene", index_col=0) - ct_totals["total road"] = df["by fuel (EUROSTAT DATA)"] + ct_totals["total road"] = df.loc["by fuel (EUROSTAT DATA)"] - ct_totals["electricity road"] = df["Electricity"] + ct_totals["electricity road"] = df.loc["Electricity"] - ct_totals["total two-wheel"] = df["Powered 2-wheelers (Gasoline)"] + ct_totals["total two-wheel"] = df.loc["Powered 2-wheelers (Gasoline)"] assert df.index[19] == "Passenger cars" ct_totals["total passenger cars"] = df.iloc[19] @@ -268,16 +296,16 @@ def idees_per_country(ct, year, base_dir): ct_totals["electricity light duty road freight"] = df.iloc[49] row = "Heavy duty vehicles (Diesel oil incl. biofuels)" - ct_totals["total heavy duty road freight"] = df[row] + ct_totals["total heavy duty road freight"] = df.loc[row] assert df.index[61] == "Passenger cars" ct_totals["passenger car efficiency"] = df.iloc[61] - df = pd.read_excel(fn_transport, "TrRail_ene", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrRail_ene", index_col=0) - ct_totals["total rail"] = df["by fuel (EUROSTAT DATA)"] + ct_totals["total rail"] = df.loc["by fuel (EUROSTAT DATA)"] - ct_totals["electricity rail"] = df["Electricity"] + ct_totals["electricity rail"] = df.loc["Electricity"] assert df.index[15] == "Passenger transport" ct_totals["total rail passenger"] = df.iloc[15] @@ -293,7 +321,7 @@ def idees_per_country(ct, year, base_dir): assert df.index[23] == "Electric" ct_totals["electricity rail freight"] = df.iloc[23] - df = pd.read_excel(fn_transport, "TrAvia_ene", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrAvia_ene", index_col=0) assert df.index[6] == "Passenger transport" ct_totals["total aviation passenger"] = df.iloc[6] @@ -324,24 +352,24 @@ def idees_per_country(ct, year, base_dir): + ct_totals["total international aviation passenger"] ) - df = pd.read_excel(fn_transport, "TrNavi_ene", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrNavi_ene", index_col=0) # coastal and inland - ct_totals["total domestic navigation"] = df["by fuel (EUROSTAT DATA)"] + ct_totals["total domestic navigation"] = df.loc["by fuel (EUROSTAT DATA)"] - df = pd.read_excel(fn_transport, "TrRoad_act", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrRoad_act", index_col=0) assert df.index[85] == "Passenger cars" ct_totals["passenger cars"] = df.iloc[85] - return pd.Series(ct_totals, name=ct) + return pd.DataFrame(ct_totals) -def build_idees(countries, year): +def build_idees(countries): nprocesses = snakemake.threads disable_progress = snakemake.config["run"].get("disable_progressbar", False) - func = partial(idees_per_country, year=year, base_dir=snakemake.input.idees) + func = partial(idees_per_country, base_dir=snakemake.input.idees) tqdm_kwargs = dict( ascii=False, unit=" country", @@ -353,53 +381,67 @@ def build_idees(countries, year): with mp.Pool(processes=nprocesses) as pool: totals_list = list(tqdm(pool.imap(func, countries), **tqdm_kwargs)) - totals = pd.concat(totals_list, axis=1) + totals = pd.concat( + totals_list, + keys=countries, + names=["country", "year"], + ) # convert ktoe to TWh - exclude = totals.index.str.fullmatch("passenger cars") - totals.loc[~exclude] *= 11.63 / 1e3 + exclude = totals.columns.str.fullmatch("passenger cars") + totals.loc[:, ~exclude] *= 11.63 / 1e3 # convert TWh/100km to kWh/km - totals.loc["passenger car efficiency"] *= 10 + totals.loc[:, "passenger car efficiency"] *= 10 - return totals.T + return totals def build_energy_totals(countries, eurostat, swiss, idees): eurostat_fuels = {"electricity": "Electricity", "total": "Total all products"} + eurostat_countries = eurostat.index.levels[0] + eurostat_years = eurostat.index.levels[1] to_drop = ["passenger cars", "passenger car efficiency"] - df = idees.reindex(countries).drop(to_drop, axis=1) + new_index = pd.MultiIndex.from_product( + [countries, eurostat_years], names=["country", "year"] + ) - eurostat_countries = eurostat.index.levels[0] - in_eurostat = df.index.intersection(eurostat_countries) + df = idees.reindex(new_index).drop(to_drop, axis=1) + + in_eurostat = df.index.levels[0].intersection(eurostat_countries) # add international navigation - slicer = idx[in_eurostat, :, "Bunkers", :] - fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() + slicer = idx[in_eurostat, :, :, "Bunkers", :] + fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=[0, 1]).sum() df.loc[in_eurostat, "total international navigation"] = fill_values # add swiss energy data - df.loc["CH"] = swiss + df = pd.concat([df.drop("CH"), swiss]).sort_index() # get values for missing countries based on Eurostat EnergyBalances # divide cooking/space/water according to averages in EU28 - missing = df.index[df["total residential"].isna()] - to_fill = missing.intersection(eurostat_countries) uses = ["space", "cooking", "water"] + to_fill = df.index[ + df["total residential"].isna() + & df.index.get_level_values("country").isin(eurostat_countries) + ] + c = to_fill.get_level_values("country") + y = to_fill.get_level_values("year") + for sector in ["residential", "services", "road", "rail"]: eurostat_sector = sector.capitalize() # fuel use for fuel in ["electricity", "total"]: - slicer = idx[to_fill, :, :, eurostat_sector] + slicer = idx[c, y, :, :, eurostat_sector] fill_values = ( - eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=0).sum() + eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=[0, 1]).sum() ) df.loc[to_fill, f"{fuel} {sector}"] = fill_values @@ -461,25 +503,27 @@ def build_energy_totals(countries, eurostat, swiss, idees): no_norway[f"total {sector}"] - no_norway[f"electricity {sector}"] ) fraction = nonelectric_use.div(nonelectric).mean() - df.loc["NO", f"total {sector} {use}"] = total_heating * fraction + df.loc["NO", f"total {sector} {use}"] = ( + total_heating * fraction + ).values df.loc["NO", f"electricity {sector} {use}"] = ( total_heating * fraction * elec_fraction - ) + ).values # Missing aviation - slicer = idx[to_fill, :, :, "Domestic aviation"] - fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() + slicer = idx[c, y, :, :, "Domestic aviation"] + fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=[0, 1]).sum() df.loc[to_fill, "total domestic aviation"] = fill_values - slicer = idx[to_fill, :, :, "International aviation"] - fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() + slicer = idx[c, y, :, :, "International aviation"] + fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=[0, 1]).sum() df.loc[to_fill, "total international aviation"] = fill_values # missing domestic navigation - slicer = idx[to_fill, :, :, "Domestic Navigation"] - fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() + slicer = idx[c, y, :, :, "Domestic Navigation"] + fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=[0, 1]).sum() df.loc[to_fill, "total domestic navigation"] = fill_values # split road traffic for non-IDEES @@ -532,9 +576,11 @@ def build_energy_totals(countries, eurostat, swiss, idees): if "BA" in df.index: # fill missing data for BA (services and road energy data) # proportional to RS with ratio of total residential demand - missing = df.loc["BA"] == 0.0 - ratio = df.at["BA", "total residential"] / df.at["RS", "total residential"] - df.loc["BA", missing] = ratio * df.loc["RS", missing] + mean_BA = df.loc["BA"].loc[2014:2021, "total residential"].mean() + mean_RS = df.loc["RS"].loc[2014:2021, "total residential"].mean() + ratio = mean_BA / mean_RS + df.loc["BA"] = df.loc["BA"].replace(0.0, np.nan).values + df.loc["BA"] = df.loc["BA"].combine_first(ratio * df.loc["RS"]).values return df @@ -550,7 +596,7 @@ def build_district_heat_share(countries, idees): district_heat_share = district_heat / total_heat - district_heat_share = district_heat_share.reindex(countries) + district_heat_share = district_heat_share.reindex(countries, level="country") # Missing district heating share dh_share = ( @@ -578,8 +624,6 @@ def build_eea_co2(input_co2, year=1990, emissions_scope="CO2"): index_col = ["Country_code", "Pollutant_name", "Year", "Sector_name"] df = df.set_index(index_col).sort_index() - emissions_scope = emissions_scope - cts = ["CH", "EUA", "NO"] + eu28_eea slicer = idx[cts, emissions_scope, year, to_ipcc.values()] @@ -622,8 +666,8 @@ def build_eea_co2(input_co2, year=1990, emissions_scope="CO2"): return emissions / 1e3 -def build_eurostat_co2(input_eurostat, countries, year=1990): - eurostat = build_eurostat(input_eurostat, countries, year) +def build_eurostat_co2(eurostat, year=1990): + eurostat_year = eurostat.xs(year, level="year") specific_emissions = pd.Series(index=eurostat.columns, dtype=float) @@ -637,7 +681,7 @@ def build_eurostat_co2(input_eurostat, countries, year=1990): # Residual oil (No. 6) 0.298 # https://www.eia.gov/electricity/annual/html/epa_a_03.html - return eurostat.multiply(specific_emissions).sum(axis=1) + return eurostat_year.multiply(specific_emissions).sum(axis=1) def build_co2_totals(countries, eea_co2, eurostat_co2): @@ -704,7 +748,9 @@ def build_transport_data(countries, population, idees): def rescale_idees_from_eurostat( - idees_countries, energy, eurostat, input_eurostat, countries + idees_countries, + energy, + eurostat, ): """ Takes JRC IDEES data from 2015 and rescales it by the ratio of the eurostat @@ -714,11 +760,10 @@ def rescale_idees_from_eurostat( """ main_cols = ["Total all products", "Electricity"] # read in the eurostat data for 2015 - eurostat_2015 = build_eurostat(input_eurostat, countries, 2015)[main_cols] - eurostat_year = eurostat[main_cols] + eurostat_2015 = eurostat.xs(2015, level="year")[main_cols] # calculate the ratio of the two data sets - ratio = eurostat_year / eurostat_2015 - ratio = ratio.droplevel([1, 4]) + ratio = eurostat[main_cols] / eurostat_2015 + ratio = ratio.droplevel([2, 5]) cols_rename = {"Total all products": "total", "Electricity": "ele"} index_rename = {v: k for k, v in idees_rename.items()} ratio.rename(columns=cols_rename, index=index_rename, inplace=True) @@ -811,19 +856,45 @@ def rescale_idees_from_eurostat( ] for country in idees_countries: + slicer_target = idx[country, 2016:2021, :, :] + slicer_source = idx[country, 2015, :, :] for sector, mapping in mappings.items(): - sector_ratio = ratio.loc[(country, slice(None), sector)] + sector_ratio = ratio.loc[(country, slice(None), slice(None), sector)].droplevel("lvl2") - energy.loc[country, mapping["total"]] *= sector_ratio["total"].iloc[0] - energy.loc[country, mapping["elec"]] *= sector_ratio["ele"].iloc[0] + energy.loc[slicer_target, mapping["total"]] = cartesian( + sector_ratio.loc[2016:2021, "total"], + energy.loc[slicer_source, mapping["total"]].squeeze() + ).values + energy.loc[slicer_target, mapping["elec"]] = cartesian( + sector_ratio.loc[2016:2021, "ele"], + energy.loc[slicer_source, mapping["elec"]].squeeze() + ).values - avi_d = ratio.loc[(country, slice(None), "Domestic aviation"), "total"] - avi_i = ratio.loc[(country, "International aviation", slice(None)), "total"] - energy.loc[country, avia_inter] *= avi_i.iloc[0] - energy.loc[country, avia_domestic] *= avi_d.iloc[0] + level_drops = ["country", "lvl2", "lvl3"] - nav = ratio.loc[(country, slice(None), "Domestic Navigation"), "total"] - energy.loc[country, navigation] *= nav.iloc[0] + slicer = idx[country, :, :, "Domestic aviation"] + avi_d = ratio.loc[slicer, "total"].droplevel(level_drops) + + slicer = idx[country, :, :, "International aviation"] + avi_i = ratio.loc[slicer, "total"].droplevel(level_drops) + + slicer = idx[country, :, :, "Domestic Navigation"] + nav = ratio.loc[slicer, "total"].droplevel(level_drops) + + energy.loc[slicer_target, avia_inter] = cartesian( + avi_i.loc[2016:2021], + energy.loc[slicer_source, avia_inter].squeeze() + ).values + + energy.loc[slicer_target, avia_domestic] = cartesian( + avi_d.loc[2016:2021], + energy.loc[slicer_source, avia_domestic].squeeze() + ).values + + energy.loc[slicer_target, navigation] = cartesian( + nav.loc[2016:2021], + energy.loc[slicer_source, navigation] + ).values return energy @@ -845,20 +916,18 @@ if __name__ == "__main__": countries = snakemake.params.countries idees_countries = pd.Index(countries).intersection(eu28) - data_year = params["energy_totals_year"] input_eurostat = snakemake.input.eurostat - eurostat = build_eurostat(input_eurostat, countries, data_year) - swiss = build_swiss(data_year) - # data from idees only exists from 2000-2015. read in latest data and rescale later - idees = build_idees(idees_countries, min(2015, data_year)) + eurostat = build_eurostat(input_eurostat, countries) + swiss = build_swiss() + idees = build_idees(idees_countries) energy = build_energy_totals(countries, eurostat, swiss, idees) - if data_year > 2015: - logger.info("Data year is after 2015. Rescaling IDEES data based on eurostat.") - energy = rescale_idees_from_eurostat( - idees_countries, energy, eurostat, input_eurostat, countries - ) + # Data from IDEES only exists from 2000-2015. + logger.info("Extrapolate IDEES data based on eurostat for years 2015-2021.") + energy = rescale_idees_from_eurostat( + idees_countries, energy, eurostat + ) energy.to_csv(snakemake.output.energy_name) @@ -871,7 +940,7 @@ if __name__ == "__main__": base_year_emissions = params["base_emissions_year"] emissions_scope = snakemake.params.energy["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope) - eurostat_co2 = build_eurostat_co2(input_eurostat, countries, base_year_emissions) + eurostat_co2 = build_eurostat_co2(eurostat, base_year_emissions) co2 = build_co2_totals(countries, eea_co2, eurostat_co2) co2.to_csv(snakemake.output.co2_name) From d2968230bbdfa07324c37b383f2a16211fdb7780 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 19:43:30 +0000 Subject: [PATCH 109/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_energy_totals.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 8e9abb69..368d7206 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -859,15 +859,17 @@ def rescale_idees_from_eurostat( slicer_target = idx[country, 2016:2021, :, :] slicer_source = idx[country, 2015, :, :] for sector, mapping in mappings.items(): - sector_ratio = ratio.loc[(country, slice(None), slice(None), sector)].droplevel("lvl2") + sector_ratio = ratio.loc[ + (country, slice(None), slice(None), sector) + ].droplevel("lvl2") energy.loc[slicer_target, mapping["total"]] = cartesian( sector_ratio.loc[2016:2021, "total"], - energy.loc[slicer_source, mapping["total"]].squeeze() + energy.loc[slicer_source, mapping["total"]].squeeze(), ).values energy.loc[slicer_target, mapping["elec"]] = cartesian( sector_ratio.loc[2016:2021, "ele"], - energy.loc[slicer_source, mapping["elec"]].squeeze() + energy.loc[slicer_source, mapping["elec"]].squeeze(), ).values level_drops = ["country", "lvl2", "lvl3"] @@ -882,18 +884,15 @@ def rescale_idees_from_eurostat( nav = ratio.loc[slicer, "total"].droplevel(level_drops) energy.loc[slicer_target, avia_inter] = cartesian( - avi_i.loc[2016:2021], - energy.loc[slicer_source, avia_inter].squeeze() + avi_i.loc[2016:2021], energy.loc[slicer_source, avia_inter].squeeze() ).values energy.loc[slicer_target, avia_domestic] = cartesian( - avi_d.loc[2016:2021], - energy.loc[slicer_source, avia_domestic].squeeze() + avi_d.loc[2016:2021], energy.loc[slicer_source, avia_domestic].squeeze() ).values energy.loc[slicer_target, navigation] = cartesian( - nav.loc[2016:2021], - energy.loc[slicer_source, navigation] + nav.loc[2016:2021], energy.loc[slicer_source, navigation] ).values return energy @@ -925,9 +924,7 @@ if __name__ == "__main__": # Data from IDEES only exists from 2000-2015. logger.info("Extrapolate IDEES data based on eurostat for years 2015-2021.") - energy = rescale_idees_from_eurostat( - idees_countries, energy, eurostat - ) + energy = rescale_idees_from_eurostat(idees_countries, energy, eurostat) energy.to_csv(snakemake.output.energy_name) From 06fa9c92dad45eca5f0211a9f44c736b7c337282 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 13 Mar 2024 08:57:57 +0100 Subject: [PATCH 110/173] remove copy_config rule and write config for each solved network --- doc/release_notes.rst | 3 +++ doc/sector.rst | 5 ----- rules/postprocess.smk | 18 ------------------ rules/solve_electricity.smk | 2 +- rules/solve_myopic.smk | 5 +++-- rules/solve_overnight.smk | 5 +++-- rules/solve_perfect.smk | 4 ++-- scripts/copy_config.py | 28 ---------------------------- scripts/solve_network.py | 12 +++++++++++- 9 files changed, 23 insertions(+), 59 deletions(-) delete mode 100644 scripts/copy_config.py diff --git a/doc/release_notes.rst b/doc/release_notes.rst index ad1bbf4f..7edc7ac6 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,9 @@ Release Notes Upcoming Release ================ +* Removed rule ``copy_config``. Instead, a config file is created for each + network output of the ``solve_*`` rules, with the same content as ``n.meta``. + * Upgrade to Snakemake v8.5+. This version is the new minimum version required. To upgrade an existing environment, run ``conda install -c bioconda snakemake-minimal">=8.5"`` and ``pip install snakemake-storage-plugin-http`` diff --git a/doc/sector.rst b/doc/sector.rst index 84a5fdc9..71a2acb8 100644 --- a/doc/sector.rst +++ b/doc/sector.rst @@ -171,11 +171,6 @@ Rule ``cluster_gas_network`` .. automodule:: cluster_gas_network -Rule ``copy_config`` -============================================================================== - -.. automodule:: copy_config - Rule ``prepare_sector_network`` ============================================================================== diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 1b188829..30de78bb 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -3,10 +3,6 @@ # SPDX-License-Identifier: MIT -localrules: - copy_config, - - if config["foresight"] != "perfect": rule plot_power_network_clustered: @@ -137,20 +133,6 @@ if config["foresight"] == "perfect": "../scripts/plot_power_network_perfect.py" -rule copy_config: - params: - RDIR=RDIR, - output: - RESULTS + "config.yaml", - threads: 1 - resources: - mem_mb=1000, - conda: - "../envs/environment.yaml" - script: - "../scripts/copy_config.py" - - rule make_summary: params: foresight=config_provider("foresight"), diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 8481c75b..06d2f4aa 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -14,9 +14,9 @@ rule solve_network: custom_extra_functionality=input_custom_extra_functionality, input: network=resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), - config=RESULTS + "config.yaml", output: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + config=RESULTS + "configs/config.elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.yaml", log: solver=normpath( RESULTS diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 57b8a9d3..28c0313f 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -117,10 +117,11 @@ rule solve_sector_network_myopic: network=RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", costs=resources("costs_{planning_horizons}.csv"), - config=RESULTS + "config.yaml", output: - RESULTS + network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + config=RESULTS + + "configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.yaml", shadow: "shallow" log: diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 26e6e220..26dee7a6 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -15,10 +15,11 @@ rule solve_sector_network: input: network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - config=RESULTS + "config.yaml", output: - RESULTS + network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + config=RESULTS + + "configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.yaml", shadow: "shallow" log: diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index a565d978..215757fe 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -107,10 +107,10 @@ rule solve_sector_network_perfect: network=RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", costs=resources("costs_2030.csv"), - config=RESULTS + "config.yaml", output: - RESULTS + network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", + config="configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.yaml", threads: solver_threads resources: mem_mb=config_provider("solving", "mem"), diff --git a/scripts/copy_config.py b/scripts/copy_config.py deleted file mode 100644 index 0a5fe10b..00000000 --- a/scripts/copy_config.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: : 2020-2024 The PyPSA-Eur Authors -# -# SPDX-License-Identifier: MIT -""" -Copy used configuration files and important scripts for archiving. -""" - - -import yaml -from _helpers import set_scenario_config - -if __name__ == "__main__": - if "snakemake" not in globals(): - from _helpers import mock_snakemake - - snakemake = mock_snakemake("copy_config") - - set_scenario_config(snakemake) - - with open(snakemake.output[0], "w") as yaml_file: - yaml.dump( - snakemake.config, - yaml_file, - default_flow_style=False, - allow_unicode=True, - sort_keys=False, - ) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 7e53e606..6cb9a054 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -36,6 +36,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr +import yaml from _benchmark import memory_logger from _helpers import ( configure_logging, @@ -968,4 +969,13 @@ if __name__ == "__main__": logger.info(f"Maximum memory usage: {mem.mem_usage}") n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) - n.export_to_netcdf(snakemake.output[0]) + n.export_to_netcdf(snakemake.output.network) + + with open(snakemake.output.config, "w") as file: + yaml.dump( + n.meta, + file, + default_flow_style=False, + allow_unicode=True, + sort_keys=False, + ) From a429b488746115ffe9279223f87df64460af9e7d Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 13 Mar 2024 09:19:10 +0100 Subject: [PATCH 111/173] update storage provider for snakemake 8 --- rules/retrieve.smk | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 30f35aa8..76069c6f 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -198,10 +198,9 @@ if config["enable"]["retrieve"]: rule retrieve_artificial_load_data: input: - HTTP.remote( + storage( "https://zenodo.org/record/7070438/files/demand_hourly.csv", keep_local=True, - static=True, ), output: "data/load_artificial_raw.csv", From 2eb159af7ca5605d9c786360a7b59e56c09a679c Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 13 Mar 2024 13:40:23 +0100 Subject: [PATCH 112/173] idees eurostat rescaling also for period 1990-1999 --- scripts/build_energy_totals.py | 76 +++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 368d7206..1b36ec1b 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -856,44 +856,52 @@ def rescale_idees_from_eurostat( ] for country in idees_countries: - slicer_target = idx[country, 2016:2021, :, :] - slicer_source = idx[country, 2015, :, :] - for sector, mapping in mappings.items(): - sector_ratio = ratio.loc[ - (country, slice(None), slice(None), sector) - ].droplevel("lvl2") + filling_years = [(2015, slice(2016, 2021)), (2000, slice(1990, 1999))] - energy.loc[slicer_target, mapping["total"]] = cartesian( - sector_ratio.loc[2016:2021, "total"], - energy.loc[slicer_source, mapping["total"]].squeeze(), - ).values - energy.loc[slicer_target, mapping["elec"]] = cartesian( - sector_ratio.loc[2016:2021, "ele"], - energy.loc[slicer_source, mapping["elec"]].squeeze(), + for source_year, target_years in filling_years: + + slicer_source = idx[country, source_year, :, :] + slicer_target = idx[country, target_years, :, :] + + for sector, mapping in mappings.items(): + sector_ratio = ratio.loc[ + (country, slice(None), slice(None), sector) + ].droplevel("lvl2") + + energy.loc[slicer_target, mapping["total"]] = cartesian( + sector_ratio.loc[target_years, "total"], + energy.loc[slicer_source, mapping["total"]].squeeze(axis=0), + ).values + energy.loc[slicer_target, mapping["elec"]] = cartesian( + sector_ratio.loc[target_years, "ele"], + energy.loc[slicer_source, mapping["elec"]].squeeze(axis=0), + ).values + + level_drops = ["country", "lvl2", "lvl3"] + + slicer = idx[country, :, :, "Domestic aviation"] + avi_d = ratio.loc[slicer, "total"].droplevel(level_drops) + + slicer = idx[country, :, :, "International aviation"] + avi_i = ratio.loc[slicer, "total"].droplevel(level_drops) + + slicer = idx[country, :, :, "Domestic Navigation"] + nav = ratio.loc[slicer, "total"].droplevel(level_drops) + + energy.loc[slicer_target, avia_inter] = cartesian( + avi_i.loc[target_years], + energy.loc[slicer_source, avia_inter].squeeze(axis=0), ).values - level_drops = ["country", "lvl2", "lvl3"] + energy.loc[slicer_target, avia_domestic] = cartesian( + avi_d.loc[target_years], + energy.loc[slicer_source, avia_domestic].squeeze(axis=0), + ).values - slicer = idx[country, :, :, "Domestic aviation"] - avi_d = ratio.loc[slicer, "total"].droplevel(level_drops) - - slicer = idx[country, :, :, "International aviation"] - avi_i = ratio.loc[slicer, "total"].droplevel(level_drops) - - slicer = idx[country, :, :, "Domestic Navigation"] - nav = ratio.loc[slicer, "total"].droplevel(level_drops) - - energy.loc[slicer_target, avia_inter] = cartesian( - avi_i.loc[2016:2021], energy.loc[slicer_source, avia_inter].squeeze() - ).values - - energy.loc[slicer_target, avia_domestic] = cartesian( - avi_d.loc[2016:2021], energy.loc[slicer_source, avia_domestic].squeeze() - ).values - - energy.loc[slicer_target, navigation] = cartesian( - nav.loc[2016:2021], energy.loc[slicer_source, navigation] - ).values + energy.loc[slicer_target, navigation] = cartesian( + nav.loc[target_years], + energy.loc[slicer_source, navigation].squeeze(axis=0), + ).values return energy From 5b5d308bf70f15dd0b107d8a19c121093dcdd5bd Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 13 Mar 2024 13:42:03 +0100 Subject: [PATCH 113/173] adapt transport data to multiyear setup with new swiss data --- data/gr-e-11.03.02.01.01-cc.csv | 45 +++++++++++++++++++++++++++++++++ rules/build_sector.smk | 1 + scripts/build_energy_totals.py | 22 +++++++++++----- 3 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 data/gr-e-11.03.02.01.01-cc.csv diff --git a/data/gr-e-11.03.02.01.01-cc.csv b/data/gr-e-11.03.02.01.01-cc.csv new file mode 100644 index 00000000..47567275 --- /dev/null +++ b/data/gr-e-11.03.02.01.01-cc.csv @@ -0,0 +1,45 @@ +year,passenger cars,passenger vehicles,goods vehicles,agricultural vehicles,industrial vehicles,motorcycles,mopeds (incl. fast e-bikes)¹ +1980,2246752,11087,169402,137685,0,137340,671473 +1981,2394455,11122,167846,151238,0,152508,687517 +1982,2473318,11341,178313,156631,0,178398,656102 +1983,2520610,11255,189920,165332,0,187090,674710 +1984,2552132,10853,192708,164078,0,199302,647391 +1985,2617164,10771,200537,175161,0,217974,644175 +1986,2678911,10800,207014,183689,0,225676,627523 +1987,2732720,11027,217750,189984,0,240102,613093 +1988,2819548,26869,236649,152693,43519,219987,581270 +1989,2895842,29270,241488,157867,44326,261715,551808 +1990,2985397,31180,252136,162932,45920,299264,464609 +1991,3057798,32968,257646,165571,46938,319779,418251 +1992,3091228,34136,256611,169277,47281,336448,381236 +1993,3109523,34852,253461,171414,47229,348159,358732 +1994,3165042,35676,256285,172300,47373,357252,336367 +1995,3229176,36517,262352,174026,47693,370700,317783 +1996,3268093,37662,263020,174247,47622,381986,301009 +1997,3323455,38508,264200,175689,47743,410750,280467 +1998,3383307,39012,267380,176712,47754,435042,265422 +1999,3467311,39692,273954,177148,48265,464357,246018 +2000,3545247,40260,278518,177963,48949,493781,218932 +2001,3629713,41342,285246,179321,49549,521390,199033 +2002,3700951,42401,290142,180063,50227,545132,186811 +2003,3753890,43629,292329,180295,50795,567358,173486 +2004,3811351,44784,298193,180898,50957,583010,165000 +2005,3863807,45785,307264,182093,51860,592194,156095 +2006,3899917,46445,314020,185450,53437,608648,150563 +2007,3955787,48026,324153,184062,55149,619166,144704 +2008,3989811,48536,326232,188218,55808,636540,141549 +2009,4009602,50675,327808,185902,56533,642777,139220 +2010,4075825,52751,335200,186485,58492,651202,139548 +2011,4163003,55422,348553,187130,60324,665870,142834 +2012,4254725,58278,361926,188358,62219,679822,145984 +2013,4320885,60151,371361,189305,63950,687990,147247 +2014,4384490,62436,382281,190095,65563,699219,152962 +2015,4458069,65720,393598,191132,67101,710022,161292 +2016,4524029,69676,405566,192139,68721,720381,176030 +2017,4570823,73814,416501,192858,70113,729149,188053 +2018,4602688,77985,428808,193283,71683,739344,201423 +2019,4623952,83054,440795,193834,74085,744542,211480 +2020,4658335,88293,452186,195082,75659,771586,229421 +2021,4709366,97805,466857,196530,77672,791323,244572 +2022,4721280,105158,475714,196942,79691,789794,257753 +2023,4760948,114299,485303,197678,81241,805653, \ No newline at end of file diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 3a0fda21..e41d0224 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -289,6 +289,7 @@ rule build_energy_totals: nuts3_shapes=resources("nuts3_shapes.geojson"), co2="data/bundle-sector/eea/UNFCCC_v23.csv", swiss="data/switzerland-new_format-all_years.csv", + swiss_transport="data/gr-e-11.03.02.01.01-cc.csv", idees="data/bundle-sector/jrc-idees-2015", district_heat_share="data/district_heat_share.csv", eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition", diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 1b36ec1b..29cf8182 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -712,15 +712,25 @@ def build_co2_totals(countries, eea_co2, eurostat_co2): def build_transport_data(countries, population, idees): - transport_data = pd.DataFrame(index=countries) + # first collect number of cars - # collect number of cars + transport_data = pd.DataFrame(idees["passenger cars"]) - transport_data["number cars"] = idees["passenger cars"] - - # CH from http://ec.europa.eu/eurostat/statistics-explained/index.php/Passenger_cars_in_the_EU#Luxembourg_has_the_highest_number_of_passenger_cars_per_inhabitant +# https://www.bfs.admin.ch/bfs/en/home/statistics/mobility-transport/transport-infrastructure-vehicles/vehicles/road-vehicles-stock-level-motorisation.html if "CH" in countries: - transport_data.at["CH", "number cars"] = 4.136e6 + fn = snakemake.input.swiss_transport + swiss_cars = pd.read_csv(fn, index_col=0).loc[1990:2021, ["passenger cars"]] + + swiss_cars.index = pd.MultiIndex.from_product( + [["CH"], swiss_cars.index], + names=["country", "year"] + ) + + transport_data = pd.concat([transport_data, swiss_cars]).sort_index() + + transport_data.rename( + columns={"passenger cars": "number cars"}, inplace=True + ) missing = transport_data.index[transport_data["number cars"].isna()] if not missing.empty: From 955d6837fda2e9b2142e291c28078067f5b9752c Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 13 Mar 2024 13:42:29 +0100 Subject: [PATCH 114/173] district heating: handle missing GB data for 2020-2021 --- scripts/build_energy_totals.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 29cf8182..9282c0e7 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -611,6 +611,11 @@ def build_district_heat_share(countries, idees): district_heat_share.name = "district heat share" + # restrict to available years + district_heat_share = ( + district_heat_share.unstack().dropna(how="all", axis=1).ffill(axis=1) + ) + return district_heat_share From 721c64a5bda180301e0bead5e5b8b21c4cf8dbb0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 12:42:57 +0000 Subject: [PATCH 115/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- data/gr-e-11.03.02.01.01-cc.csv | 2 +- scripts/build_energy_totals.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/data/gr-e-11.03.02.01.01-cc.csv b/data/gr-e-11.03.02.01.01-cc.csv index 47567275..0ba695de 100644 --- a/data/gr-e-11.03.02.01.01-cc.csv +++ b/data/gr-e-11.03.02.01.01-cc.csv @@ -42,4 +42,4 @@ 2020,4658335,88293,452186,195082,75659,771586,229421 2021,4709366,97805,466857,196530,77672,791323,244572 2022,4721280,105158,475714,196942,79691,789794,257753 -2023,4760948,114299,485303,197678,81241,805653, \ No newline at end of file +2023,4760948,114299,485303,197678,81241,805653, diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 9282c0e7..48df4625 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -721,21 +721,18 @@ def build_transport_data(countries, population, idees): transport_data = pd.DataFrame(idees["passenger cars"]) -# https://www.bfs.admin.ch/bfs/en/home/statistics/mobility-transport/transport-infrastructure-vehicles/vehicles/road-vehicles-stock-level-motorisation.html + # https://www.bfs.admin.ch/bfs/en/home/statistics/mobility-transport/transport-infrastructure-vehicles/vehicles/road-vehicles-stock-level-motorisation.html if "CH" in countries: fn = snakemake.input.swiss_transport swiss_cars = pd.read_csv(fn, index_col=0).loc[1990:2021, ["passenger cars"]] swiss_cars.index = pd.MultiIndex.from_product( - [["CH"], swiss_cars.index], - names=["country", "year"] + [["CH"], swiss_cars.index], names=["country", "year"] ) transport_data = pd.concat([transport_data, swiss_cars]).sort_index() - transport_data.rename( - columns={"passenger cars": "number cars"}, inplace=True - ) + transport_data.rename(columns={"passenger cars": "number cars"}, inplace=True) missing = transport_data.index[transport_data["number cars"].isna()] if not missing.empty: From 74339b5648d7244360164ac403e422e909545064 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 12:01:57 +0100 Subject: [PATCH 116/173] update data/era5-annual-{runoff,HDD}-per-country.csv --- data/era5-annual-HDD-per-country.csv | 68 ++++++++++++------------- data/era5-annual-runoff-per-country.csv | 68 ++++++++++++------------- 2 files changed, 68 insertions(+), 68 deletions(-) diff --git a/data/era5-annual-HDD-per-country.csv b/data/era5-annual-HDD-per-country.csv index 472257c6..5f6ad45d 100644 --- a/data/era5-annual-HDD-per-country.csv +++ b/data/era5-annual-HDD-per-country.csv @@ -1,34 +1,34 @@ -name,1951,1952,1953,1954,1955,1956,1957,1958,1959,1960,1961,1962,1963,1964,1965,1966,1967,1968,1969,1970,1971,1972,1973,1974,1975,1976,1977,1978,1979,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021 -AL,56,58,65,67,55,70,58,59,61,54,54,62,62,62,68,59,60,57,57,58,61,57,64,60,61,61,54,63,57,65,63,58,61,57,57,58,61,62,56,53,67,62,62,53,61,61,60,60,56,58,57,53,61,58,65,64,55,55,54,53,58,58,50,46,54,50,54,47,49,50,54 -AT,374,416,379,422,419,449,387,393,378,389,367,437,425,406,424,376,385,400,409,413,394,407,408,379,384,400,380,414,405,428,397,384,377,411,420,406,409,378,369,372,415,373,383,334,380,416,371,373,368,343,371,337,370,370,384,362,333,340,351,388,337,353,358,302,324,336,347,315,320,322,358 -BE,95,105,94,102,109,113,91,97,88,89,84,111,114,100,100,91,94,101,102,101,96,102,101,90,97,97,92,103,108,105,99,94,96,99,112,107,108,85,84,83,101,92,94,82,89,110,90,89,84,79,89,78,89,90,87,86,76,87,87,107,75,90,99,69,81,88,82,81,80,71,89 -BG,283,294,334,367,295,363,306,283,320,268,277,303,324,313,323,259,297,292,320,286,300,306,323,303,296,314,287,310,293,326,301,312,295,306,326,313,333,320,276,272,332,303,318,266,307,335,325,303,273,275,284,272,322,277,298,296,259,266,266,273,313,295,258,250,261,265,281,261,232,250,277 -BA,144,164,167,183,158,194,157,157,152,143,141,173,180,176,176,155,162,157,165,165,163,154,170,156,153,168,145,170,153,176,163,154,157,159,167,160,166,158,146,139,174,152,162,143,157,171,156,158,147,132,145,130,156,146,164,148,134,132,139,147,144,147,133,113,137,134,144,128,125,127,141 -CH,214,225,210,226,221,240,211,215,203,214,200,234,231,214,229,213,217,219,225,227,217,221,224,213,216,220,208,219,220,229,216,208,208,227,224,217,217,200,202,200,217,204,208,183,206,217,195,205,203,192,200,189,199,204,209,193,187,194,193,213,179,196,204,178,181,189,191,174,184,176,197 -CZ,297,344,307,351,350,377,313,322,301,314,294,358,361,345,351,297,296,320,342,336,319,326,322,282,296,315,299,330,328,347,316,308,298,323,350,326,346,297,280,283,333,299,316,279,312,364,319,296,288,265,309,285,308,302,311,300,269,275,289,338,281,298,306,242,261,283,287,262,255,259,302 -DE,1254,1428,1237,1416,1484,1567,1282,1342,1229,1266,1183,1486,1528,1390,1399,1269,1211,1341,1447,1429,1300,1388,1356,1184,1258,1339,1221,1377,1433,1433,1348,1288,1273,1384,1492,1425,1491,1201,1156,1135,1367,1241,1312,1155,1295,1548,1288,1220,1172,1085,1250,1168,1269,1236,1239,1198,1072,1159,1202,1459,1112,1232,1305,992,1095,1188,1148,1092,1070,1019,1230 -DK,160,176,152,175,186,190,163,178,154,169,154,184,194,176,182,181,153,170,182,186,159,166,159,147,147,172,161,171,190,177,175,164,155,162,191,181,190,151,137,132,160,147,166,154,162,191,162,154,147,134,161,148,158,151,150,144,134,137,150,193,144,159,160,125,138,145,139,142,134,125,150 -ES,962,911,927,948,785,1092,934,848,821,890,742,952,981,967,949,878,890,839,933,899,971,961,943,931,928,928,797,906,887,911,793,787,807,920,860,891,781,783,684,767,911,852,889,724,627,759,605,735,796,750,757,677,746,813,856,714,776,785,736,856,673,810,830,649,684,722,688,768,708,672,728 -EE,297,310,283,297,321,336,278,310,281,305,253,298,323,292,306,324,283,310,333,317,285,285,289,256,245,327,302,324,299,313,289,278,260,271,328,294,332,281,232,240,257,257,285,288,265,306,281,278,262,235,273,271,278,270,270,260,250,230,268,315,253,286,259,254,225,261,255,254,235,209,270 -FI,3285,3442,3085,3179,3880,3819,3296,3626,3180,3386,3065,3513,3467,3329,3526,3873,3284,3693,3640,3494,3572,3178,3458,2963,3074,3691,3490,3743,3419,3607,3572,3331,3296,3192,3894,3516,3793,3402,2888,3075,3209,3163,3319,3341,3213,3343,3302,3468,3259,2915,3303,3307,3231,3159,2998,3101,3018,3009,3219,3604,2961,3351,2985,2983,2795,3030,3127,3061,3116,2726,3260 -FR,1430,1480,1422,1489,1396,1633,1338,1378,1240,1332,1196,1584,1630,1499,1471,1331,1408,1412,1473,1469,1475,1453,1545,1356,1462,1445,1312,1454,1502,1545,1396,1315,1424,1484,1599,1518,1520,1269,1237,1227,1501,1375,1389,1168,1280,1454,1222,1337,1263,1190,1299,1132,1294,1346,1376,1263,1199,1301,1277,1522,1060,1300,1408,1034,1143,1266,1236,1150,1165,1043,1279 -GB,906,934,809,901,925,935,813,890,787,849,825,1000,1046,912,970,917,839,874,894,869,805,880,862,842,838,851,863,873,972,880,896,823,824,838,923,944,903,801,764,746,859,824,857,793,794,886,738,752,745,771,816,726,751,743,748,734,706,790,777,930,707,832,841,683,759,769,720,764,750,725,764 -GR,166,162,210,211,160,215,184,169,196,164,172,182,184,192,199,162,185,176,171,164,184,185,195,186,188,190,160,186,172,191,184,192,190,179,174,180,194,193,176,165,206,197,195,161,179,191,192,180,167,177,174,164,191,172,184,189,161,158,155,143,188,179,148,136,165,150,164,139,146,148,156 -HR,116,140,140,155,137,166,131,132,122,119,114,149,158,152,144,124,133,129,143,138,137,131,144,124,124,139,121,140,130,149,133,132,130,133,148,141,144,128,118,114,143,123,138,115,126,145,132,129,124,106,120,109,135,127,140,123,109,110,116,130,123,123,118,92,114,116,119,111,102,105,119 -HU,236,291,282,321,289,333,272,273,246,248,235,288,321,309,298,244,272,268,293,283,276,264,283,244,255,281,267,287,271,311,272,282,265,274,313,294,305,274,243,238,299,262,288,240,271,310,291,272,267,232,269,243,300,267,289,265,229,230,243,275,267,259,250,198,235,251,255,234,211,228,259 -IE,216,215,191,208,213,211,190,204,184,205,195,234,250,208,232,210,216,213,222,214,190,226,204,213,198,211,210,205,239,212,209,204,201,211,229,241,217,200,192,192,209,209,213,201,195,219,180,186,191,204,206,189,192,193,187,188,173,202,204,241,192,205,210,186,205,199,184,200,191,194,189 -IT,652,683,660,700,615,778,640,653,616,631,587,715,726,674,710,665,655,656,682,672,678,639,695,659,655,670,604,681,675,729,684,645,657,695,688,671,676,630,616,610,712,626,650,555,623,639,578,618,619,563,585,546,629,612,675,597,548,568,590,637,562,602,587,481,541,528,570,529,540,520,577 -LT,344,371,342,368,368,405,318,353,329,346,303,355,386,362,371,354,322,365,401,372,330,345,339,312,289,388,348,370,370,382,342,326,301,326,393,352,397,335,275,272,313,311,336,329,325,377,335,325,306,274,323,313,328,324,325,316,299,277,316,366,305,332,314,298,273,308,296,302,267,252,327 -LU,8,9,8,9,10,10,8,9,8,8,7,10,10,9,9,8,8,9,9,9,9,9,9,8,9,9,8,9,10,10,9,9,9,9,10,10,10,8,8,8,9,8,9,8,8,10,8,8,8,7,8,7,8,8,8,8,7,8,8,9,7,8,9,6,7,8,8,7,7,7,8 -LV,383,402,372,394,407,439,351,391,361,384,326,386,420,385,402,401,360,401,434,407,361,372,371,337,316,424,385,411,392,411,372,357,330,354,425,379,428,362,300,301,334,337,366,366,348,402,365,358,334,302,353,346,357,352,354,337,324,300,346,402,330,365,342,329,297,337,330,328,297,272,351 -MK,64,66,74,79,66,79,67,66,71,63,64,72,73,70,77,66,69,66,67,67,70,67,75,69,70,69,62,72,66,73,71,69,70,67,69,67,70,71,65,63,76,68,69,59,69,70,71,68,64,65,65,62,69,64,70,68,60,59,61,59,67,66,56,53,61,58,62,54,55,57,63 -ME,40,43,45,48,41,50,42,42,42,39,39,44,45,45,48,43,43,42,42,43,43,41,44,43,42,45,39,45,42,46,45,41,43,42,42,41,44,44,42,39,47,43,43,38,43,44,42,43,39,39,40,37,42,41,45,43,38,38,38,38,39,40,36,32,37,36,39,34,35,35,39 -NL,113,126,110,121,130,137,108,118,107,107,104,132,141,121,120,114,105,119,124,122,111,120,116,104,111,116,107,120,133,122,118,112,110,118,134,127,130,99,99,94,119,106,113,102,109,138,111,102,97,93,106,98,109,103,100,100,87,102,103,131,93,108,117,82,95,104,96,98,93,85,105 -NO,3339,3541,3020,3307,3639,3731,3315,3547,3122,3353,3400,3807,3641,3496,3783,3951,3259,3533,3425,3431,3339,3128,3312,3028,3094,3405,3369,3436,3457,3380,3502,3181,3141,3117,3541,3359,3449,3200,2890,2848,3057,3032,3196,3207,3170,3301,3060,3203,3062,2878,3166,3028,2983,2943,2874,2828,2918,2954,3018,3442,2796,3157,2946,2754,2793,2859,2963,2930,2981,2711,3035 -PL,1249,1438,1294,1478,1446,1586,1279,1349,1269,1305,1215,1435,1527,1450,1467,1280,1193,1321,1504,1431,1293,1347,1314,1200,1184,1419,1270,1393,1431,1491,1336,1265,1178,1304,1494,1377,1507,1255,1077,1078,1312,1252,1310,1197,1302,1512,1326,1249,1171,1052,1268,1192,1296,1230,1262,1244,1117,1092,1220,1426,1177,1274,1251,1068,1063,1163,1158,1120,1019,1017,1252 -PT,107,103,99,109,80,131,106,93,93,97,78,106,104,115,103,98,103,97,112,110,115,116,106,109,109,109,99,139,107,102,89,90,95,104,99,108,85,89,75,87,104,96,104,85,66,88,62,82,96,90,90,78,87,97,103,90,91,91,84,96,80,101,100,81,79,87,79,96,81,72,78 -RO,736,811,883,948,822,968,810,789,826,731,744,826,890,867,873,706,791,776,858,792,787,798,838,782,752,855,772,836,780,880,801,811,763,807,917,815,882,827,710,700,846,801,848,701,806,869,842,805,740,710,760,721,844,754,799,780,684,695,695,757,793,781,709,662,681,721,720,690,624,648,740 -RS,222,252,273,300,249,313,248,249,254,231,226,270,283,271,274,233,253,243,267,254,251,246,272,242,243,265,233,265,241,273,251,255,244,253,275,256,268,254,231,220,275,243,263,223,250,270,259,250,237,217,234,217,262,238,265,247,217,214,222,229,251,244,214,192,222,223,232,211,197,212,234 -SK,178,210,195,217,208,228,192,193,185,185,178,208,213,211,219,179,187,192,203,200,192,188,195,178,178,197,188,204,193,215,191,190,180,193,212,198,207,187,172,173,202,186,191,171,186,204,193,184,177,162,184,174,190,184,191,182,163,161,170,188,175,179,174,142,161,170,176,157,152,158,180 -SI,61,72,67,73,69,78,65,66,62,61,59,75,75,73,73,62,65,67,71,69,67,68,71,62,61,69,62,71,68,74,67,65,65,69,73,71,71,63,59,58,70,61,63,55,61,71,63,62,62,54,59,55,65,63,67,61,54,57,57,65,58,59,59,46,55,57,58,53,52,53,61 -SE,3891,4219,3560,3919,4426,4488,3950,4223,3662,3988,3814,4451,4260,4021,4358,4613,3929,4280,4255,4254,4043,3806,3975,3634,3625,4238,4132,4314,4246,4287,4301,3913,3840,3819,4588,4139,4376,3931,3476,3446,3785,3695,3893,3991,3916,4073,3757,3950,3781,3446,3898,3778,3755,3769,3632,3561,3606,3590,3806,4397,3474,3935,3675,3452,3421,3635,3693,3705,3689,3247,3807 +name,1941,1942,1943,1944,1945,1946,1947,1948,1949,1950,1951,1952,1953,1954,1955,1956,1957,1958,1959,1960,1961,1962,1963,1964,1965,1966,1967,1968,1969,1970,1971,1972,1973,1974,1975,1976,1977,1978,1979,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023 +AL,73,70,62,70,74,64,59,65,62,57,56,58,65,67,55,70,58,59,61,54,54,62,62,62,68,59,60,57,57,58,61,57,64,60,61,61,54,63,57,65,63,58,61,57,57,58,61,62,56,53,67,62,62,53,61,61,60,60,56,58,57,53,61,58,65,64,55,55,54,53,58,58,50,46,54,50,54,47,49,50,54,74,47 +AT,448,429,391,433,413,406,403,399,376,383,374,416,379,422,419,449,387,393,378,389,367,437,425,406,424,376,385,400,409,413,394,407,408,379,384,400,380,414,405,428,397,384,377,411,420,406,409,378,369,372,415,373,383,334,380,416,371,373,368,343,371,337,370,370,384,362,333,340,351,388,337,353,358,302,324,336,347,315,320,322,358,456,307 +BE,107,109,89,104,96,100,101,90,92,101,95,105,94,102,109,113,91,97,88,89,84,111,114,100,100,91,94,101,102,101,96,102,101,90,97,97,92,103,108,105,99,94,96,99,112,107,108,85,84,83,101,92,94,82,89,110,90,89,84,79,89,78,89,90,87,86,76,87,87,107,75,90,99,69,81,88,82,81,80,71,89,107,74 +BG,358,373,318,329,339,331,304,319,305,299,283,294,334,367,295,363,306,283,320,268,277,303,324,313,323,259,297,292,320,286,300,306,323,303,296,314,287,310,293,326,301,312,295,306,326,313,333,320,276,272,332,303,318,266,307,335,325,303,273,275,284,272,322,277,298,296,259,266,266,273,313,295,258,250,261,265,281,261,232,250,277,372,228 +BA,192,189,158,181,181,174,168,168,167,155,144,164,167,183,158,194,157,157,152,143,141,173,180,176,176,155,162,157,165,165,163,154,170,156,153,168,145,170,153,176,163,154,157,159,167,160,166,158,146,139,174,152,162,143,157,171,156,158,147,132,145,130,156,146,164,148,134,132,139,147,144,147,133,113,137,134,144,128,125,127,141,179,116 +CH,245,231,213,239,230,222,215,217,210,212,214,225,210,226,221,240,211,215,203,214,200,234,231,214,229,213,217,219,225,227,217,221,224,213,216,220,208,219,220,229,216,208,208,227,224,217,217,200,202,200,217,204,208,183,206,217,195,205,203,192,200,189,199,204,209,193,187,194,193,213,179,196,204,178,181,189,191,174,184,176,197,245,171 +CZ,373,366,311,341,326,330,353,313,299,316,297,344,307,351,350,377,313,322,301,314,294,358,361,345,351,297,296,320,342,336,319,326,322,282,296,315,299,330,328,347,316,308,298,323,350,326,346,297,280,283,333,299,316,279,312,364,319,296,288,265,309,285,308,302,311,300,269,275,289,338,281,298,306,242,261,283,287,262,255,259,302,378,248 +DE,1563,1537,1250,1388,1316,1354,1464,1255,1205,1315,1254,1428,1237,1416,1484,1567,1282,1342,1229,1266,1183,1486,1528,1390,1399,1269,1211,1341,1447,1429,1300,1388,1356,1184,1258,1339,1221,1377,1433,1433,1348,1288,1273,1384,1492,1425,1491,1201,1156,1135,1367,1241,1312,1155,1295,1548,1288,1220,1172,1085,1250,1168,1269,1236,1239,1198,1072,1159,1202,1459,1112,1232,1305,992,1095,1188,1148,1092,1070,1019,1230,1509,1021 +DK,200,198,151,167,158,171,185,158,142,164,160,176,152,175,186,190,163,178,154,169,154,184,194,176,182,181,153,170,182,186,159,166,159,147,147,172,161,171,190,177,175,164,155,162,191,181,190,151,137,132,160,147,166,154,162,191,162,154,147,134,161,148,158,151,150,144,134,137,150,193,144,159,160,125,138,145,139,142,134,125,150,193,138 +ES,1045,986,927,1012,830,994,860,807,904,853,962,911,927,948,785,1092,934,848,821,890,742,952,981,967,949,878,890,839,933,899,971,961,943,931,928,928,797,906,887,911,793,787,807,920,860,891,781,783,684,767,911,852,889,724,627,759,605,735,796,750,757,677,746,813,856,714,776,785,736,856,673,810,830,649,684,722,688,768,708,672,728,933,655 +EE,373,352,266,284,307,298,320,274,250,285,297,310,283,297,321,336,278,310,281,305,253,298,323,292,306,324,283,310,333,317,285,285,289,256,245,327,302,324,299,313,289,278,260,271,328,294,332,281,232,240,257,257,285,288,265,306,281,278,262,235,273,271,278,270,270,260,250,230,268,315,253,286,259,254,225,261,255,254,235,209,270,365,246 +FI,3989,3729,3012,3160,3611,3348,3577,3151,3012,3213,3285,3442,3085,3179,3880,3819,3296,3626,3180,3386,3065,3513,3467,3329,3526,3873,3284,3693,3640,3494,3572,3178,3458,2963,3074,3691,3490,3743,3419,3607,3572,3331,3296,3192,3894,3516,3793,3402,2888,3075,3209,3163,3319,3341,3213,3343,3302,3468,3259,2915,3303,3307,3231,3159,2998,3101,3018,3009,3219,3604,2961,3351,2985,2983,2795,3030,3127,3061,3116,2726,3260,4325,3109 +FR,1671,1578,1350,1586,1458,1490,1421,1336,1381,1447,1430,1480,1422,1489,1396,1633,1338,1378,1240,1332,1196,1584,1630,1499,1471,1331,1408,1412,1473,1469,1475,1453,1545,1356,1462,1445,1312,1454,1502,1545,1396,1315,1424,1484,1599,1518,1520,1269,1237,1227,1501,1375,1389,1168,1280,1454,1222,1337,1263,1190,1299,1132,1294,1346,1376,1263,1199,1301,1277,1522,1060,1300,1408,1034,1143,1266,1236,1150,1165,1043,1279,1535,1075 +GB,946,965,835,869,773,860,910,819,781,890,906,934,809,901,925,935,813,890,787,849,825,1000,1046,912,970,917,839,874,894,869,805,880,862,842,838,851,863,873,972,880,896,823,824,838,923,944,903,801,764,746,859,824,857,793,794,886,738,752,745,771,816,726,751,743,748,734,706,790,777,930,707,832,841,683,759,769,720,764,750,725,764,985,690 +GR,218,222,205,217,223,201,166,200,192,177,166,162,210,211,160,215,184,169,196,164,172,182,184,192,199,162,185,176,171,164,184,185,195,186,188,190,160,186,172,191,184,192,190,179,174,180,194,193,176,165,206,197,195,161,179,191,192,180,167,177,174,164,191,172,184,189,161,158,155,143,188,179,148,136,165,150,164,139,146,148,156,230,138 +HR,160,166,131,147,151,147,145,138,127,131,116,140,140,155,137,166,131,132,122,119,114,149,158,152,144,124,133,129,143,138,137,131,144,124,124,139,121,140,130,149,133,132,130,133,148,141,144,128,118,114,143,123,138,115,126,145,132,129,124,106,120,109,135,127,140,123,109,110,116,130,123,123,118,92,114,116,119,111,102,105,119,153,96 +HU,304,332,268,281,302,285,297,277,251,262,236,291,282,321,289,333,272,273,246,248,235,288,321,309,298,244,272,268,293,283,276,264,283,244,255,281,267,287,271,311,272,282,265,274,313,294,305,274,243,238,299,262,288,240,271,310,291,272,267,232,269,243,300,267,289,265,229,230,243,275,267,259,250,198,235,251,255,234,211,228,259,331,211 +IE,219,217,194,196,175,204,220,191,176,210,216,215,191,208,213,211,190,204,184,205,195,234,250,208,232,210,216,213,222,214,190,226,204,213,198,211,210,205,239,212,209,204,201,211,229,241,217,200,192,192,209,209,213,201,195,219,180,186,191,204,206,189,192,193,187,188,173,202,204,241,192,205,210,186,205,199,184,200,191,194,189,258,166 +IT,806,757,663,758,734,704,693,675,666,652,652,683,660,700,615,778,640,653,616,631,587,715,726,674,710,665,655,656,682,672,678,639,695,659,655,670,604,681,675,729,684,645,657,695,688,671,676,630,616,610,712,626,650,555,623,639,578,618,619,563,585,546,629,612,675,597,548,568,590,637,562,602,587,481,541,528,570,529,540,520,577,746,495 +LT,439,424,323,336,358,357,382,323,303,336,344,371,342,368,368,405,318,353,329,346,303,355,386,362,371,354,322,365,401,372,330,345,339,312,289,388,348,370,370,382,342,326,301,326,393,352,397,335,275,272,313,311,336,329,325,377,335,325,306,274,323,313,328,324,325,316,299,277,316,366,305,332,314,298,273,308,296,302,267,252,327,431,273 +LU,10,10,8,9,9,9,9,8,8,9,8,9,8,9,10,10,8,9,8,8,7,10,10,9,9,8,8,9,9,9,9,9,9,8,9,9,8,9,10,10,9,9,9,9,10,10,10,8,8,8,9,8,9,8,8,10,8,8,8,7,8,7,8,8,8,8,7,8,8,9,7,8,9,6,7,8,8,7,7,7,8,10,7 +LV,489,461,348,366,392,389,413,352,325,369,383,402,372,394,407,439,351,391,361,384,326,386,420,385,402,401,360,401,434,407,361,372,371,337,316,424,385,411,392,411,372,357,330,354,425,379,428,362,300,301,334,337,366,366,348,402,365,358,334,302,353,346,357,352,354,337,324,300,346,402,330,365,342,329,297,337,330,328,297,272,351,473,308 +MK,83,80,72,80,83,75,69,73,72,67,64,66,74,79,66,79,67,66,71,63,64,72,73,70,77,66,69,66,67,67,70,67,75,69,70,69,62,72,66,73,71,69,70,67,69,67,70,71,65,63,76,68,69,59,69,70,71,68,64,65,65,62,69,64,70,68,60,59,61,59,67,66,56,53,61,58,62,54,55,57,63,85,54 +ME,52,48,43,48,49,45,43,46,46,41,40,43,45,48,41,50,42,42,42,39,39,44,45,45,48,43,43,42,42,43,43,41,44,43,42,45,39,45,42,46,45,41,43,42,42,41,44,44,42,39,47,43,43,38,43,44,42,43,39,39,40,37,42,41,45,43,38,38,38,38,39,40,36,32,37,36,39,34,35,35,39,53,34 +NL,132,136,106,120,109,119,129,107,106,119,113,126,110,121,130,137,108,118,107,107,104,132,141,121,120,114,105,119,124,122,111,120,116,104,111,116,107,120,133,122,118,112,110,118,134,127,130,99,99,94,119,106,113,102,109,138,111,102,97,93,106,98,109,103,100,100,87,102,103,131,93,108,117,82,95,104,96,98,93,85,105,128,87 +NO,3658,3823,3379,3441,3537,3479,3726,3442,3268,3480,3339,3541,3020,3307,3639,3731,3315,3547,3122,3353,3400,3807,3641,3496,3783,3951,3259,3533,3425,3431,3339,3128,3312,3028,3094,3405,3369,3436,3457,3380,3502,3181,3141,3117,3541,3359,3449,3200,2890,2848,3057,3032,3196,3207,3170,3301,3060,3203,3062,2878,3166,3028,2983,2943,2874,2828,2918,2954,3018,3442,2796,3157,2946,2754,2793,2859,2963,2930,2981,2711,3035,4108,3040 +PL,1615,1584,1275,1356,1364,1378,1517,1290,1230,1296,1249,1438,1294,1478,1446,1586,1279,1349,1269,1305,1215,1435,1527,1450,1467,1280,1193,1321,1504,1431,1293,1347,1314,1200,1184,1419,1270,1393,1431,1491,1336,1265,1178,1304,1494,1377,1507,1255,1077,1078,1312,1252,1310,1197,1302,1512,1326,1249,1171,1052,1268,1192,1296,1230,1262,1244,1117,1092,1220,1426,1177,1274,1251,1068,1063,1163,1158,1120,1019,1017,1252,1599,1033 +PT,114,114,102,113,86,110,95,79,94,96,107,103,99,109,80,131,106,93,93,97,78,106,104,115,103,98,103,97,112,110,115,116,106,109,109,109,99,139,107,102,89,90,95,104,99,108,85,89,75,87,104,96,104,85,66,88,62,82,96,90,90,78,87,97,103,90,91,91,84,96,80,101,100,81,79,87,79,96,81,72,78,98,75 +RO,931,939,801,813,874,851,839,815,799,780,736,811,883,948,822,968,810,789,826,731,744,826,890,867,873,706,791,776,858,792,787,798,838,782,752,855,772,836,780,880,801,811,763,807,917,815,882,827,710,700,846,801,848,701,806,869,842,805,740,710,760,721,844,754,799,780,684,695,695,757,793,781,709,662,681,721,720,690,624,648,740,973,611 +RS,292,305,246,274,285,276,265,263,261,239,222,252,273,300,249,313,248,249,254,231,226,270,283,271,274,233,253,243,267,254,251,246,272,242,243,265,233,265,241,273,251,255,244,253,275,256,268,254,231,220,275,243,263,223,250,270,259,250,237,217,234,217,262,238,265,247,217,214,222,229,251,244,214,192,222,223,232,211,197,212,234,306,192 +SK,221,219,199,207,209,201,212,202,189,189,178,210,195,217,208,228,192,193,185,185,178,208,213,211,219,179,187,192,203,200,192,188,195,178,178,197,188,204,193,215,191,190,180,193,212,198,207,187,172,173,202,186,191,171,186,204,193,184,177,162,184,174,190,184,191,182,163,161,170,188,175,179,174,142,161,170,176,157,152,158,180,231,150 +SI,81,77,66,74,72,71,72,69,61,65,61,72,67,73,69,78,65,66,62,61,59,75,75,73,73,62,65,67,71,69,67,68,71,62,61,69,62,71,68,74,67,65,65,69,73,71,71,63,59,58,70,61,63,55,61,71,63,62,62,54,59,55,65,63,67,61,54,57,57,65,58,59,59,46,55,57,58,53,52,53,61,77,51 +SE,4509,4537,3713,3939,4134,4059,4374,3918,3633,4015,3891,4219,3560,3919,4426,4488,3950,4223,3662,3988,3814,4451,4260,4021,4358,4613,3929,4280,4255,4254,4043,3806,3975,3634,3625,4238,4132,4314,4246,4287,4301,3913,3840,3819,4588,4139,4376,3931,3476,3446,3785,3695,3893,3991,3916,4073,3757,3950,3781,3446,3898,3778,3755,3769,3632,3561,3606,3590,3806,4397,3474,3935,3675,3452,3421,3635,3693,3705,3689,3247,3807,5084,3769 diff --git a/data/era5-annual-runoff-per-country.csv b/data/era5-annual-runoff-per-country.csv index 44055ee7..862ff743 100644 --- a/data/era5-annual-runoff-per-country.csv +++ b/data/era5-annual-runoff-per-country.csv @@ -1,34 +1,34 @@ -name,1951,1952,1953,1954,1955,1956,1957,1958,1959,1960,1961,1962,1963,1964,1965,1966,1967,1968,1969,1970,1971,1972,1973,1974,1975,1976,1977,1978,1979,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021 -AL,9921.845743062182,13900.722955927391,9710.931748556879,10574.663320350839,19341.246363906943,17818.135264440367,11442.713887022777,18086.801216289034,15090.128262650946,21363.26299103702,9091.698545327283,15645.252953546536,26744.08978638467,15193.032691775385,15984.127347228396,21350.165383112253,12855.252167834218,13420.929958145927,18491.92173822311,23127.97840031047,15647.322475185589,18621.138651647332,15899.845988546795,17394.46706381624,9941.517004312835,15056.517227865626,13546.145796746823,19078.487245825123,21734.889469128113,18681.018783118114,17330.7404590047,11854.58505075189,8621.019687739607,10958.633158319248,11928.114867207425,14053.437518158384,11478.547049310142,9794.162917024925,9309.704198107493,7531.383319836307,13417.032891488734,9396.693750356952,9409.269914411027,9602.245346872442,13721.093186096236,18546.848247572612,10075.274288997707,13160.130634381218,14979.3178626011,11173.442130939595,10656.167120597063,11333.102860367679,12218.649211273396,16968.134523176956,13531.444805168734,12351.485474236735,7777.709342414358,9395.578415692386,13162.74078695902,21279.522933687273,7131.262505063414,9979.8596278608,14852.708775826433,10550.911120633964,9976.081734060937,14930.01135394502,8322.107101243482,14871.511785547276,8735.240828081134,9201.521447196284,15965.927842545672 -AT,82083.36248472283,77811.80736180999,72623.28291775129,92135.9214601233,96990.37023646115,85097.60090834703,90540.2512560922,87926.80279582532,87840.33332785149,95771.51050456028,80100.0597367668,89961.83970292915,94250.95465501286,83836.27750897587,127502.64206362939,114305.16098508416,109841.9915507258,94950.56912780282,78607.44609072198,109558.29937241938,70075.51296406337,91798.91825728992,87743.39136454638,99172.3185471974,112079.92282492586,77403.35720896415,141634.05110901658,127491.44958971704,103513.31397365611,94072.4808516721,90719.06004331182,74178.70850428617,71954.43763473642,75430.93523721115,83080.51606488624,68680.54577957527,88997.37641681913,78365.83036558417,88108.35920540475,70055.7892534604,83590.790849829,80943.05621285089,88054.73669568839,64323.72991620271,76516.31784700113,88537.52869871816,74980.7275243927,76703.23963260197,90699.65433828124,82618.46892777775,70279.06284438398,76864.90473420777,38883.771847486016,64333.37459685728,71147.24783545699,61897.64030624322,56722.92181769331,64364.47029441406,80300.64217575881,73731.45518976511,60840.309523971024,77575.9681355992,74925.98195655535,87076.27014563217,55659.14157544586,71826.27744178762,65790.17399828041,62758.61938425757,63451.07652587882,63443.71962749899,60531.78695214822 -BE,2265.3933295931465,2299.7187477495568,1601.6287456097295,903.9162052526016,1681.6717933948885,1553.0826978024484,2040.8513201770277,2596.453742857211,1572.1256345899858,1777.5813929080084,2626.96550179367,2109.9419394561014,1095.552430877252,1153.3090754765356,2926.2047527605378,3273.3444811813465,2455.1462895729287,2709.4254348359373,2044.223080485547,3206.6821776003253,1403.4368531879622,1739.01009962226,1509.0861879549796,2489.4865224203572,2440.1073430603583,1152.6261514941527,2218.937185944037,2759.156597708309,2657.6330859359005,2904.764428136616,3657.83503348807,2493.2486286738635,2917.270148862828,2660.3113890881846,1897.0141168954808,2027.0116832611525,2822.5902787959144,2964.8457470488684,1943.761175974426,1192.1713008070133,1419.0119225208746,1475.2557281855488,1769.9742186132778,2647.8931490311224,2842.9882619071104,880.5015070232031,1197.5019163599245,2093.4616667083587,2614.387699318041,2455.605567526133,2964.8707630509653,2512.265466519676,1585.2842393930487,929.2255532772314,1281.2863306004592,1121.936801130382,2183.018089275124,2209.175694512598,1603.8823576630593,1544.7801250586733,1495.8450265096008,1698.4561860883377,1826.3476500921734,1803.4120415623293,1828.4613155494794,2424.4009518987536,1091.1596112517386,1814.2183897620703,1171.4609814371709,2056.8674333110275,2031.8962335362714 -BG,7806.165358407907,7617.879584469116,9313.915986883927,12665.103843530671,18165.703005314856,20925.111917073817,12315.95137614649,14307.215196342033,11652.743586791588,16042.778215050708,15000.011504505326,15411.367697482856,23959.284980030086,13749.537818599603,14477.782616320155,19301.913180173906,18779.64649297375,14471.754083245778,21905.976678210158,20337.934359162857,20009.368056844236,22642.8256575709,23291.544303799303,14596.266505770593,16672.574320967968,18465.93977853061,15123.0740003407,16401.953223276385,17509.31598177905,19228.19892146255,14599.539868168713,16192.551398587811,14129.882713144674,15395.87601647586,11853.583066311567,12273.885630367266,13296.879487675578,12962.17913948413,10178.141531948864,10101.748861881337,17833.95419276747,11290.360773903032,7050.797194092144,7109.831367133397,12879.727662239184,15849.231621819281,14036.330428384765,14600.701492135015,14377.532715491632,7705.7385359122845,8414.14109970378,12768.14503989922,11390.56673556146,10492.153643864705,15665.526881990423,13518.231074072748,9896.670696858979,8514.724851169687,10495.74229946015,14331.982881842356,7228.669024198985,6779.132502990354,8418.829006301234,16447.848686603495,15740.35987338571,11179.883037182273,8789.581276892988,11591.504961624405,6993.71096793164,8046.946588923506,10797.966046345515 -BA,23467.52324346034,27732.028396263388,21355.39127202247,21518.57483169428,38198.02048713121,26864.382517947488,20815.045478598513,30820.77172304343,26041.454718573805,32434.848569531026,20613.68138756547,29427.6067241259,33216.04695823398,35126.00882914745,35434.02869793576,33198.240125653894,27292.13451964922,28870.288475787063,34855.527471322755,40158.11721181478,26000.45986387781,30818.815452611776,25386.059210704487,29741.547122355205,28211.934365516674,30030.474291377286,28262.88980319508,36551.088477619196,29006.131913946814,27659.528119372964,23600.138376555304,18313.888493525537,16009.416242480263,24722.92263318226,19189.890721764787,24683.705818314953,20989.732351804592,20569.866983400992,18470.90664070533,14851.755901997987,25891.954823722892,19819.186826010166,18921.820235243023,20522.576037546467,27525.961934821684,30713.034955646464,21412.918614981576,21344.870107122344,26543.94285666588,19942.69654945001,23180.52553450498,25225.421682786062,17217.841128593398,25342.758536033023,22647.27040540609,20311.586220306715,13360.715771642963,18725.9052067595,21814.111731615874,31110.03238839381,11170.209355889527,11624.972537489217,20618.808518471225,26321.87815471064,18023.46871599901,20293.60245322545,15351.515268991054,23732.8659965382,19641.494693119246,14018.69083089969,18476.061736021245 -CH,96134.70662377033,78280.50065735803,75567.82823680741,75624.52378636891,84437.3353706146,82858.22758683441,70824.92066589095,89311.09471805945,65693.73271567532,95736.9646979805,81823.38979168108,71941.93730231738,93291.72439376163,67130.87226350856,103459.16116112942,100678.84780651607,97724.23622167954,103707.47043615916,84031.35407618036,103162.60292359442,67550.72288370335,80436.85045155323,87045.62560332651,91431.67032188186,103676.61542793321,68303.04781353423,112322.8911282786,90003.61666202937,91028.65800738563,79435.56403729375,84362.27602442491,71967.17653868764,71356.17066607098,70327.74412176231,72852.24726422041,60501.17532436925,83372.98013161327,74234.0349385613,61890.14596593614,54336.52856303606,58271.018596959824,62644.46734396249,75790.31562737533,69637.43067630196,63290.99710001813,60360.96649160057,57476.7610131337,62343.013781750116,78693.52307555292,73667.98513148604,75122.7031133837,70327.33051388402,33043.59766466136,48968.55545142022,47632.62274536815,48588.59714178414,58398.342285916435,57742.38347363473,42559.51792407362,59408.23894223828,57129.80243604273,71163.63652517287,70858.4268509181,119263.66699527185,55000.17388464577,60563.73895679769,51249.64855691976,45607.547672961526,60542.786248766475,50488.945849740405,62587.04259357905 -CZ,5741.811363153907,6430.8701452824125,6608.002211449021,5547.98412657507,11139.03676570244,9975.869136026628,10340.80877548634,13532.930674695574,7705.735677184478,7812.801872929193,10964.355351867012,10646.504003233118,9719.560436989883,9722.51782291833,19015.37334991238,15549.667939948402,15927.255735311868,12882.876953226223,10353.403919517015,11853.576269226234,10677.61771918624,10413.95103805089,6895.283030051313,9424.903567027312,12922.392855988994,9137.874616036986,17532.348544152504,18533.830228515635,15250.542039808313,10190.043701741004,11537.335951365127,9452.574932230946,5833.228549596612,5329.206957903766,7580.6976645678615,7069.4638232187235,11641.01844987317,9790.145937530902,7320.767578365769,5406.873869997462,5625.637231646826,7239.835546851702,6440.471526964078,10089.617439184036,12468.55796361433,12603.058166229213,11520.381274473184,9364.18293733304,10411.740148030329,8903.525346614548,9434.353552398077,14463.707947162726,8297.335864348554,6305.0428212307515,9383.102792968806,9420.905095490103,7053.530866655102,7934.504156979587,9131.766698685136,11027.709419784474,7973.011990884055,6406.065926850977,10728.184944007047,6645.433895571247,6345.82145823084,6499.675988954033,7016.463805946017,5619.135413529,4807.786670051716,5785.96568493957,7986.497102061061 -DE,37514.09293381361,41854.44001450301,38958.042582957896,35908.26729908446,54302.825387264325,53900.0627595872,49537.22949846474,58655.764340031456,37627.8257805545,42967.11131060519,58392.899587658314,47246.42956098116,36543.949785170116,36184.44504543948,76454.75505815547,77563.28089874369,65706.46393994696,62713.75591115003,50646.052126013274,74501.93068438585,33965.80961685558,36936.195615288314,42537.597915180464,56616.68921207296,56744.68927682596,34807.98202654087,56642.02192030585,72249.78534694383,61866.53633607989,55155.47864420242,60392.13262306467,54016.67652252674,53772.61604029787,44957.681101902934,45340.54136937088,48213.458592237475,66703.44582319073,60862.17019199549,41014.24992391146,32890.656129618095,33561.7114818598,35496.398391557115,43529.12222860253,55750.09930602895,59293.24549493131,37708.06634825865,37073.71374151417,42227.76427477898,55642.269571505705,50261.32889407123,50099.1653624443,63663.80803911536,33932.84222448403,24148.0930626651,39327.588161226195,34424.75266875371,37774.7468462978,39904.36386097259,38761.37664135111,43371.39725264248,35879.97041420226,37163.99533546333,50391.241894760955,37073.97568394232,36139.75677566728,42435.96482981362,34097.15586833773,36767.50341472353,28228.762854078665,31841.550922412007,39703.58319343355 -DK,540.6494487174206,225.87222027671936,240.57818634617777,370.37587945377,297.48979685029366,176.69208006156848,208.79213770008658,418.4554011794129,292.8102234525168,354.1220188699609,405.97396851309696,497.6042583966885,394.062076725445,291.3295691524654,304.5438623304028,591.4756757289098,629.1912050409122,479.19182164984767,353.75286712188796,454.0799499391422,338.48620130918835,357.676218196066,271.5200039604648,288.04880490856954,389.1757124784286,200.03398963977796,391.555913715114,397.29527898061724,379.8043799580585,514.7861201159348,505.7304478209306,381.57816134966373,489.3522161731293,301.717309458533,340.1024114463131,377.00536197863613,349.05617800662674,489.4227950883326,223.23589869898433,269.17110101837187,288.12441629799764,229.5796362727797,293.635637901186,464.89056739838395,386.583686695923,100.62743648672041,155.50324274468716,357.89273417380645,451.04208007822723,403.3432275318107,376.8995908622044,395.3896113505339,169.30459202493293,211.16687292699712,238.87912856891634,225.9981737610454,469.5229598902774,334.5021223766927,213.88993390188736,267.6283833775911,319.04746406617295,376.09256801086406,317.617725512616,353.26516627601364,404.2285931109035,408.5235836651241,351.86517079981496,305.15203377590285,361.8170659996248,441.45504335438466,312.14518786073575 -ES,49381.7181466273,42506.42487287232,29823.22966835847,33520.89863652133,42189.87541544434,45018.59980204108,27387.4917761516,45602.58407624375,62626.425244916936,94141.56249288519,66884.65469167147,67115.06386526441,82553.38984370838,62483.82642486853,54633.78478265801,97009.71651609142,47073.863176177474,55800.125131342014,85746.55555964066,72028.42027117872,75395.28981871007,97165.39237066667,52196.13272216457,55183.75674414718,44434.367388636274,45688.29479796517,107175.1885102772,376352.4882722023,122222.92956502433,46652.32256081546,42283.296882354574,45745.627884904505,49745.69531403085,62249.891346371536,58920.93920695715,48705.91089393512,49506.55303300167,60104.870297070986,41417.79510072781,40561.22111509541,48584.70074175896,41980.59062088254,40901.640882458836,43899.80095828514,42846.07384845135,68844.74333042726,58684.98840324564,44556.941393617584,38670.78308215774,46815.51322723987,68212.19494117319,35011.91030341564,50475.76713743041,37227.78564631605,21874.579431867896,35771.95151620915,29462.81813629621,28957.890173503936,32351.242924601538,48787.03990347552,31409.59113067243,24367.630996980257,54440.48961905899,50212.19464108009,33563.81874118821,47043.00929278407,21662.04001863739,49011.344970588914,40719.843457258816,40971.905822060144,45241.676643808205 -EE,1013.5168183220601,680.8524541483096,896.074617741102,657.0814978199126,1138.8891299468455,1033.2360728791764,1360.240006093535,1004.6950457068739,934.8774758153143,610.9977141696028,1048.0576885162222,1477.5046721309996,819.8347754595931,595.9919701101757,664.1717649435486,1178.672290642306,1112.7122616720162,1003.3658496852601,739.8499843054013,970.7396668771637,1156.5986195943308,988.0643633892423,963.0153554345143,973.0118510683724,980.7669857762866,637.7376743732589,1121.0664931638019,1485.3090062847518,1014.4287893275732,701.3843644468006,1196.8127434491068,1136.1215677919786,956.0911286121622,761.6620263988196,1109.6559596693944,945.8137139982223,1246.4435124663337,1081.0063689141978,1180.9951778617858,1433.4526339800395,1044.4962572791317,875.9289554452553,699.3911738148247,770.5276589427534,923.6083617038047,353.16555295606884,572.4961222214871,770.5595680360855,811.3520528619258,480.6204379373417,526.2783934002991,624.0252970776874,317.7449195254128,502.6073570248072,523.2149143909173,201.10844695565802,516.5064948459781,787.1638506300568,929.6905631193786,943.6792182991617,682.831069893961,933.1072250631543,692.097322772721,449.72254117947216,392.4173074585239,635.9676582877272,753.2897252561945,692.2577760060705,496.359994444626,685.8431112730673,512.1207984775516 -FI,37605.30609105331,44212.92045299994,43354.55443946037,40909.90249435943,45502.4817028443,31224.476747778986,41198.39417754888,37433.6293362709,28205.152814527217,22733.614552593714,36790.55908786393,45364.155643266684,28818.806516416316,35920.468538637644,43898.27049057533,41064.81684006838,48740.383821508905,37939.33952866352,30918.72594548001,32676.177851908644,33754.05667642144,33605.2810320407,40213.26757169686,46999.65997165162,40988.91471061698,26205.15827640759,41167.74001571865,25866.031354509407,34538.39918373158,28474.82496101674,52752.27216474974,31970.598763267328,35752.18054352296,34628.515223969094,27958.317625583102,31764.196982878522,30015.16030263079,26346.04561795855,37548.80632649389,27745.021972655202,27337.35846234453,42346.93155636462,32896.698992597754,22520.879762748657,23208.41547315833,19884.31515309925,22475.195958848508,33947.011695157635,26939.65172877202,32198.274008146243,28319.0878193112,25068.601753935876,16735.56947336933,29510.104024439464,31518.153863523567,22455.334677451585,28461.18834411878,38706.997170480194,29619.48059832548,29700.763598820195,30127.95453034573,45678.82538666528,31580.034602099753,29256.819253905658,43076.06538538077,49489.04536337738,37281.852766328535,33818.560580958125,35465.53242057822,41410.545080415,41567.72109759498 -FR,135094.49763612094,106772.95460342853,80280.31347296045,74490.3783289433,106687.24245156738,78827.6552216503,82482.11027901788,110101.24506868717,102231.57173723538,134019.59611286796,107848.78158188045,101612.59295931137,120079.53146371644,89568.66448178052,139585.30725052828,153772.55334552698,108799.90779570332,132530.25240249874,145056.24077647954,157344.40874499045,121182.68180087653,135378.15307040678,100408.75538533325,132164.7891142495,122382.6612084835,103706.50937621434,174240.52624537304,149853.10658224527,139992.58441391244,124892.14391178945,138893.07653615897,136102.17909683278,136298.85434619425,112484.75227834118,103842.75366698352,105453.40277549413,121935.76619549586,138379.45234918175,71924.60006455882,68830.14474115185,82603.77634528138,113556.04015089742,104293.46882473862,133822.28230897718,106263.7222156586,102481.34621013676,87080.48957235571,78895.08319488679,105735.9820525235,105626.7734987112,124313.3006359069,92144.11932119,73537.29646152578,78137.60525599915,58286.814348802,68928.05802270533,75743.7796112372,83489.37862547796,61518.93775410048,71311.23184506799,58810.65089438103,62581.66694545332,108566.1756738168,106810.3035898108,64981.51870529863,80829.27869961815,55203.39988738071,99034.47679069567,69318.43286028007,80169.78942387344,91015.86505778233 -GB,30949.55733300973,26701.9428167181,24834.423901258084,36292.29933386541,21223.79048684261,23695.358277919488,25987.57619398849,29907.138025215598,20532.262685247148,29689.355990879158,29755.7349738537,26456.609914217806,26827.642866860373,23873.145060634775,28779.074240844395,32368.78110647752,35977.394307138864,31071.326823875082,26446.212227329706,32888.07880400224,26015.735528737063,30161.596277105604,23017.874991984525,33287.290870143,28286.810808036254,25581.658649545065,33234.01349421005,31886.616851943854,36147.13303532815,34133.94123938026,36553.158784009356,34512.41082508442,31536.090754720717,28572.314934249887,33275.60304522956,33665.001595891976,29575.879486696827,34513.91395990196,28917.71257427291,33609.176553253485,26240.381135352633,32589.192149993025,28862.78621634025,32807.4074928576,28488.125666145854,20034.93309228305,23116.389736102017,36396.90134863421,35532.192401610075,39288.48780281492,29384.81519197965,36157.931892943554,21622.405196644995,32040.743170655453,30471.91526631006,28610.855281883465,33870.42864117695,37281.10394972126,33227.13302814586,22275.90424618414,30588.672790028984,34121.358436958675,27706.739428180073,37230.134388487466,33699.78058075289,31403.25713954576,26791.643973463473,25928.83232662774,30294.933954035565,36884.703709615125,29006.326905891798 -GR,10056.853784361187,12820.948661755581,12880.09713386794,18009.367077161973,21378.182658177288,29473.438101212807,11770.791493072751,17561.986308683798,14068.336209442465,21036.22984593777,10600.651816726047,19253.94027167335,33903.71393281602,18899.558983910043,17444.093386496537,23383.013428670623,15695.74724949063,19981.88173635776,22837.72155515553,20477.278624856677,21882.21250746782,25630.65513791745,23207.036995448034,22044.998521881807,12010.599831751206,19432.667576547075,10483.216379681078,17709.119022396855,23856.615736538963,24868.152661457774,21212.948012018598,19349.21918097175,12988.230031417628,16765.76278511076,15711.921661553248,14768.388537620414,15953.652684601051,12434.694106124754,8265.22849298033,8609.897588471382,14667.4570613014,8511.641075667938,10022.119539267658,13526.966234241496,13788.166042309897,20514.65575034389,13890.655920759986,13637.35506363307,17713.539092926785,10603.937587040999,9085.188935046803,13431.534902143163,18237.093734480102,15007.784707319466,15322.74478671021,15189.508916238075,9627.363919501071,7964.610286111404,17730.814440852053,21625.83796282357,10692.331430067703,11630.957871879938,16037.311332863352,13770.795305947142,17211.823293162783,14042.921372422215,9179.230226001768,15781.789228215457,11036.807907632012,8148.5700551254695,17837.056638784255 -HR,11579.119396857217,11616.785819262946,7841.411279590863,7071.274416413621,12992.083400898755,8164.444623202054,7347.788717816701,10332.483465428491,10537.238741811467,14479.677080073134,8479.661662510043,11787.072656273116,12862.758623825672,13371.003032039336,15990.668161905827,14470.61427957732,10644.752476691014,11929.91211438134,13611.162472968235,14159.968181810045,8748.9998436583,15752.17883640428,8576.387915317433,13121.48137750286,9754.915357083264,10574.934113880938,12339.192872828058,13526.82683401755,12407.125261525374,13314.467203516846,10589.407510915647,8240.264291214513,7438.667886688274,10422.72691638502,8167.94266416981,9179.047407118169,8517.079324039581,8574.526688740114,7000.526114186264,6044.501432950472,8424.638032244015,7640.079353180473,8493.38568955205,7673.466816227631,10113.96973112034,10475.583854629525,8161.447547783003,7730.598307694147,9688.392563588315,7721.732973546704,8547.91442119075,9110.630144082634,5654.409914846469,10449.908940883637,8544.776426500835,7703.363550389989,5356.181480736791,7888.76450496884,7981.496673656482,12159.302077747214,4232.3935344941665,4890.509308451722,11184.320641158047,12936.9950624162,6984.475921997467,9245.055929521772,6446.204647314787,9728.03403894923,7468.897556955247,6117.536838412799,7342.634549696792 -HU,455.56009955160323,446.6314758325084,756.8984035642842,450.9800559529234,596.2398492658431,618.0772176598714,433.5306920564666,626.6699838105968,736.6623236995524,901.6628598550868,955.0588428000881,1010.9577650474141,1739.1969813800204,989.7527773708749,1794.409065735425,2141.779921355298,1587.224990709883,671.8419282109866,1725.8355118081338,2716.2211930211206,1024.7764676942138,1478.4297579371664,1174.856199808959,1380.70450266654,1412.5023358393119,740.4722294240396,2077.892875467242,1382.2122922975911,1644.3319455706533,1323.8387691522125,1268.4319727220777,1050.1561979992803,938.7640185853782,701.18673897032,1176.2218406698482,1296.5341770126697,1360.4975641937087,1113.61620590335,686.0888933448485,340.9577501900905,585.7771480398817,710.4671894364953,683.5190908209308,901.0768559915952,882.6855336512624,1696.8667744778322,1072.391017559335,1091.9628754375356,1986.255314199332,1224.1891103332646,723.3105004079888,525.4091579835733,484.84230756145246,965.526984284363,1182.1048527111184,1188.8833398485922,413.588170351592,480.2721461089437,695.3047160171476,1875.1966820116616,1424.9690036241302,335.4105482008909,1504.9089491398734,1485.372787096725,1238.0161031995012,1379.3214547239281,769.0933902263423,1164.4795232335282,401.1056623458921,586.967649321741,686.5594518266915 -IE,4646.742323576806,3498.3840852321937,2544.6578209214726,5419.077105100732,3195.826410242899,2135.0862825425293,3896.6031468108818,5166.0575828547335,3059.5430902455864,4420.523688067632,3920.565970045664,2677.658691610785,3238.0778129090736,2814.899041865502,4169.640537659651,4975.659864222032,4131.237385306254,4051.6063579500055,3501.9810380506974,4162.222738923252,3277.97066925379,4149.907266073347,3535.126191101705,5127.397347805918,3672.8392481603883,3233.4968306810615,4670.955668853978,4602.662496185422,5486.794576286672,5974.860239294209,4958.279351735843,5539.220771177308,4814.939874910363,4704.080605250908,5214.372862088697,5579.052373684467,4424.669817520398,5645.170219782959,4099.107666436778,4291.534915722901,4394.7348113611615,4474.842169491055,4574.431496810677,5809.6920444413845,4586.044007664658,4500.049573188483,4158.002042556449,5864.151573833777,5389.317421868625,5602.682945402753,3910.7950042057882,5664.313605042361,3809.963053559503,4119.424688220842,4183.3641097691625,4756.243128847813,4885.19842669336,5839.576896771334,6529.030637616166,3851.468550158534,4739.247453475783,5116.873230654843,4346.868803644192,5780.6460718128965,5283.18966576195,5253.677788447396,4182.318532051922,4579.528423701424,5332.55258145869,6160.501534866154,5064.7000821427855 -IT,172598.9057049173,100312.33974168803,109683.21050744873,113141.95504995789,104153.60992530867,105687.75931391625,108887.88674499927,120408.08444781008,120463.4718424679,176044.98820314519,110983.47718210719,108661.3888675,154479.99952596257,116866.21274767391,140970.07450678368,148795.1431459142,123467.91624840492,141856.2012294592,135145.74559391223,121025.70232844367,123064.2692081519,158675.8479818625,126408.22188494769,122698.61897984576,134048.90222163082,141351.40587318933,183551.69201916258,152672.0987165877,152463.24070715246,129214.69623803807,126406.52003570941,111266.52387106014,105493.22493839792,124930.0152276459,118811.00097800085,107475.13484215035,108705.61617459086,105553.12911266208,89696.47047566871,79022.57929325203,102862.21537887226,110384.19134436303,114736.91039530469,100806.28542965792,84803.88252804938,120635.19844583895,89284.70639754581,91327.36842742827,112272.49254052607,121214.98394688763,100240.69732658492,110327.24980482439,67667.3134977533,93496.05161894935,74054.55044130566,77775.69469931335,75858.44303177306,92083.61659880447,94760.12813659209,130298.81027721112,113068.08204604911,102862.27823880903,124544.42911673438,173919.9577739064,87247.90399947706,91232.2769509342,68511.40136893332,110593.38607511377,103785.63569256017,87869.12433673623,92697.61009264145 -LT,3126.90226977831,1475.6787373817235,2475.307120672793,1105.461669987713,2895.355169108127,2328.291848466701,2773.100951693534,3565.1300060634676,1726.340017569786,2441.2869150017605,2739.255286641479,3393.627534013301,1948.972938917477,1503.9304931932238,2081.2181025571476,2387.4356286402945,2953.7202011167815,2442.643404228762,1575.9037853337413,3106.077653607497,2438.6998492607095,2321.3335398002305,2455.4955733417105,2703.979150849733,2983.428649152806,1271.81865757996,2291.2165997279003,3624.3054978962264,3066.52482874976,2692.4730181815085,3077.131323187848,2583.193918114293,2346.227785803459,1679.5438811966314,3224.9829735377853,3118.546035682447,2841.2519919875017,2793.689370450914,2688.2457239306805,3241.443663108373,2559.3234295762136,1702.002348908642,2110.46892607847,2728.2093372090403,2507.383271401867,987.8894968242066,1210.6320667482842,2619.0625969612056,2041.7494156189375,1077.1024719580128,1501.0630179433206,2093.0805484121715,761.1714789679556,1438.6161577229564,2052.75406588365,780.7629955514127,1806.053023318598,1637.6952437203324,1294.75789506843,2045.7605537026564,2014.9263059368886,1513.2399919084748,1700.3251444304037,1310.6134461272318,707.068706016363,991.8607300754669,2210.5243412620116,1945.761381971759,656.9514438734058,938.7049363615082,1091.9224609932446 -LU,418.42920407960173,402.65326898169104,245.66041042705334,140.5213247873591,281.65083355306945,239.8578158452654,369.22101543741394,483.78479198242695,296.9130979040409,276.1232951910135,459.8791399297912,360.88050800564,165.48000564979304,183.08661927563494,549.4327262552889,615.540914852196,430.5724249074097,464.9649192455573,366.43220700786026,562.8555247965114,228.93628708423907,211.2546971696525,190.68337965968735,337.11947715404034,376.86900759463407,153.06162250371963,361.9412034686643,457.17182659872975,474.4467062976349,501.35146460886807,586.4359860050876,446.87720467222556,554.4763793156474,495.34599197145104,293.22338952418016,311.9404169698979,470.26963100793296,539.1966800252237,318.1508533879876,216.80440341571304,213.96493869659722,161.79868975175182,244.05142665329072,449.452274496719,514.0163752203341,140.3179752581265,173.96942733863747,312.73695003201067,389.83220575942556,401.09422881834865,533.6829497064967,363.9324325652479,243.9448809708681,141.83825708815394,174.8785415794147,180.93947242990794,337.92060394534246,342.3576188750888,243.21751195509597,234.05807708972188,220.32797632404777,264.38928306703747,336.6463620917912,349.32621250528973,289.85643977541105,421.5786735040934,195.16848080033785,348.6723038250213,181.01976524286118,391.4073725411393,335.53788814460387 -LV,2742.6556866506367,1837.8325027812643,2554.170761349389,1408.345733539928,2983.189997195677,2825.061612760662,3252.3974243255384,2886.2924715658505,1956.4363533791625,2047.5738516633846,2396.823663944175,4066.8157610189646,1865.1937720377932,1213.3814003625498,1799.0008428733133,2700.1570058485836,2685.678969351212,2489.884608424664,1643.1417843905015,2409.13500668555,2592.9272177648704,2235.207139152776,2147.227486416904,2582.289866216919,2592.1022950383003,1400.2389662856958,2360.0229699498527,4017.8529783072768,2384.614549536931,2229.7220033855633,3208.439621353169,2678.7332051326716,2668.5315414635693,1705.3124319563372,2816.1467857724074,2779.7212753206704,2548.8356765576805,2583.5815153824487,2861.3233286998366,3801.9993997175525,2885.329135114707,2265.350140707051,1837.2956559244064,2087.5337557102102,2249.8356476827344,853.5386928200795,1689.684828383291,2653.3751344813045,2109.0096432617606,1182.3316650686795,1487.9132295244858,1974.1178903388873,1027.3242775723352,1736.361793878124,1741.9204323497202,701.3946960765046,1526.0106468283361,1844.9956663852768,1722.690767934754,2194.735679264339,1757.0892689497684,1974.9007736328324,1883.6056453393867,1405.6090440523008,1355.4439688992024,1295.7921755717964,2182.9434502570357,1620.95881111662,955.2966396067089,1490.1655775988,1270.5915407733335 -MK,3934.258904454459,3692.0813330320566,3668.815892166788,5481.091007849558,8488.277349663611,8514.814821434962,4861.071723840239,6878.625964986481,6399.572820343104,7510.160197503697,3410.068805763449,5394.1653784487335,11852.098687247973,6238.562291909563,5957.282159254423,7239.168545674971,6704.994620967638,5343.254672711665,7690.122171456297,7543.1707758996035,6080.923842798452,9518.22452251825,7931.36999814296,7898.373553315725,5296.06496225987,7158.993856907015,5568.700579027411,6653.618095140364,7658.60790313609,7384.3372015299765,6359.355811805509,6311.4116310067075,4303.248848133382,4384.047563174607,4091.082807237865,4939.915122052565,4226.085259691547,2768.0039025432975,3428.9467644601264,2294.222990992098,4834.495252793617,3325.6396521134557,2522.690466358069,2976.927612194815,4721.490759581051,6218.919122814915,3840.4729075548557,4211.680380445357,5623.802967061423,3683.311545054556,3010.5387111938044,4030.600677147787,3746.016920616422,4369.907262961726,4375.170573617275,4171.022704527618,2702.082556773276,2500.5709190188645,4370.4076446296785,5204.859735574909,2501.3689966568136,2522.4142074268602,3898.568577966785,4120.496629723428,4145.529030725163,3980.5145191045517,2967.8383253548755,4325.530795978238,2351.320803369623,2869.298937488066,4644.825806471018 -ME,11787.036249138713,18552.496612764233,11097.228684668844,11567.930061084586,19293.29710814638,13959.20571478613,13046.18440612437,18177.491235848003,15681.709335959564,19349.950981118494,11658.5954682407,15979.557715971854,21966.784229466284,16319.44299708603,17409.40677448169,19267.118980997075,14169.834731486397,15882.152400499632,19649.60512617265,23317.326436161944,15993.83236649745,16724.197023812518,13680.494674103275,16947.38155713745,13396.216553276576,17144.464963358845,16187.204710073258,20417.876685607593,22530.120155126962,16197.908505488407,14818.545420474018,12719.274460270883,9622.464989039141,13369.875886578782,13078.849292203078,14951.85122042927,13681.691957367619,12435.207850184568,11885.89235643587,9516.042262408026,16926.063160570764,13462.909968102742,10042.999218445517,11916.714437943356,15158.632522247093,19735.53511234594,11491.695345503264,14075.890065382568,13952.132515246369,11677.32833931845,13022.018989752623,13358.031703205903,11534.177624564669,17635.48411958614,14124.491791464301,12392.738752899031,9187.905148236658,11106.432128782126,13780.482580236776,19889.70854314474,7004.765863876614,8204.168680103983,13925.208337027952,12712.182323875111,8528.030231315688,13924.890575378206,9037.809310147335,13972.095665379566,11360.358971424077,10280.43536084397,13775.545442858487 -NL,132.53876435280281,105.25304624029256,97.45390390948565,59.629996943856206,126.17294913858377,100.92785922135535,110.87608266899032,166.47617616014315,115.03831975395074,102.21679880589629,222.71184206736731,164.32395798051633,88.95057195947412,70.61286255591416,116.64203846425596,229.2598570422647,176.71083656110923,163.34614915846544,135.23541074052787,153.03725726999411,84.2996330000842,75.50229060079856,87.79575328623356,149.0911400069578,185.89892961221338,65.86854604814957,122.15247506918442,163.86971747802477,177.38160465294413,173.97671792234624,210.94537197363266,138.15528852664815,157.7989701791954,164.8431327190436,156.27033317337316,144.7881412408737,172.55396177597567,195.57239229094031,113.32208112175368,57.91841778157319,69.80575559844598,81.95247611107445,154.25504072397393,213.16593963287744,185.6861094126344,50.88580789479815,88.6770750590963,180.59511129061775,175.09641999581325,134.73866646771805,175.89672487625361,170.10222446753016,112.93557838603432,69.46018341018433,89.82211025011819,57.645723008783136,122.86182387892966,128.20825436811367,82.72980960226074,90.07937846739043,113.22245968360778,109.86527486394354,113.13774281842522,111.87536788688416,113.53271254663122,140.01537175074324,68.64291161119651,101.90820271032081,52.313311323047174,116.8375292031251,120.98114114085548 -NO,268338.23623559956,311516.6976002237,354741.82081862347,287750.3093395867,277336.7794878445,277384.3177020413,343804.7636848019,309909.39898200217,279870.8316011512,240316.6988310159,337135.04130569333,320967.163231472,302346.276731951,373063.16157364036,335043.0981929385,271478.1367425455,378240.8226290755,318135.4534804216,235625.6466770364,244547.82624561962,342177.033729037,330660.24479289196,373161.4848287328,318697.1789732917,345399.49658985186,314419.8189439592,230963.73709983748,236070.00307120252,312676.53469423676,195102.01076090848,231756.2941107393,196457.66930311252,255890.52889680286,207980.18388759278,250218.19793663023,221472.83820330602,212315.28230042476,222774.69138098884,210372.25250462766,210846.3448041017,177438.35447152023,214108.95671057343,190417.02324135366,175558.31404488848,199041.30532793878,154342.13931409398,175902.47007757265,215120.2640019617,216733.8944940455,261370.58822336013,212333.63219332727,174324.40313248086,166539.1770850326,202012.88317391058,238213.58919108333,170401.06338583003,217109.81457913358,185684.48673203168,210764.408869451,180452.94255978477,359959.68381729786,220133.80814962948,201815.91045998363,193256.2376798832,225141.35101977925,194502.14845672756,241700.45583039062,203049.47691663646,189440.21996991744,241270.48318453485,196628.83730256284 -PL,10323.210329129464,10102.880830361863,15243.556644277647,6316.576710839603,13844.440254769646,15125.266765937784,15697.445163802722,20547.692524530372,11483.751822471613,12736.801698643669,14242.313560202334,19975.533453959804,14197.61841088385,13385.258603233342,17555.946503015533,19526.389852009346,24990.22970093532,18554.95543612019,10610.557656768206,19356.459381427685,19657.930603206125,18732.50311338716,16842.606570419066,21220.5523273715,24157.44207211756,15607.0096942104,21677.41127186343,21298.26291533998,22732.35472759213,23096.919661671975,20539.587419825057,15690.593097829358,10514.527177352287,9757.003902443616,18214.127020951153,14743.470132504332,14160.372458977477,15574.81487531224,12340.373515759937,8851.92371521615,10791.411372967183,10330.247763133175,10795.54750377256,14738.227630388099,14374.438386354948,12177.648415014886,14450.3889852054,16473.394758962142,14491.781691174727,11650.64866730713,13724.64695911459,14468.081620316902,7425.533290488975,8039.550610683221,9468.54351850099,6656.34004885665,7842.310345436549,9593.638690519654,8828.256800239144,15874.058827778785,13537.338641985261,6174.396712827303,9509.500411581042,9436.102173346171,5360.954908688417,6309.399306468634,12402.024613286878,10602.512257049677,4725.103685514244,6963.3130148445025,9124.964758601689 -PT,8372.129135066036,5969.31083523099,2397.6586063764958,3916.9962424030286,9654.699566273128,9603.363148222781,4163.162870337118,9179.465624121007,12131.854156488425,23406.646477036124,10635.077955993413,11029.404714084018,16475.917880350513,12087.55769685171,9804.492049893286,26178.184180984652,5669.820254013018,7559.528320921835,15405.930172416956,11163.387795923936,7088.025310785357,10191.330496756716,7128.069257230898,9335.631563569985,5236.13128329125,4572.251664471752,32605.577182903042,149323.86523586328,34294.81803755196,6061.11721308257,5836.654497517657,4843.7020037722705,7814.782411245904,9395.759000250582,14057.157961109715,8192.288977642056,7655.760207241933,9203.645337709624,7637.7450966202805,7971.812432879156,7421.37678917759,2935.2871895485287,6329.939163930326,8820.796560748415,8944.109217212255,14719.27363438586,11050.779819799696,8379.75073967695,5590.525909815882,10485.131539563748,20476.028147427794,7715.162648805314,11939.81827695821,5116.2596877306005,3047.0903835389504,9273.489011971804,5135.313604526994,3331.418010762715,6338.048693933388,12105.616460835923,7097.468638389797,3348.213951094401,11773.920694867731,13465.586888738457,4379.602415677836,11394.463621230614,3104.8789319036796,7216.150544702967,6653.765864361187,5720.580737490315,6685.493238874142 -RO,19346.344374357534,19295.50358149612,27702.617554877004,24644.67023787785,52797.882559506776,42262.7162454671,34689.01143849651,42583.74114324521,28973.147767939397,36882.64868179707,29063.471368254675,39575.626278602576,31898.858414072816,36283.98261870408,42207.50410750205,41714.957670180156,43270.313090823794,40474.54716426176,53565.91349208634,74865.02611452111,48190.11154819992,57257.26135564704,46523.80416861758,44300.83791323246,54901.1701096736,47361.27151969814,49156.09266449866,56461.30009557872,56062.45724028393,58506.511677323855,48499.732173446595,45901.64074770707,33171.12855766741,38310.49272614435,39377.41586450999,28638.803870927226,29424.553978129992,39429.14347226504,39300.73151290347,27525.04968129815,47095.77682827757,30948.90232192865,32939.14262154816,30155.99148400138,35067.04977301419,35592.25066656211,45790.67404045973,42596.77718301701,39223.35618808953,25767.93476452563,30963.38609617001,30034.908201595936,23323.05819720473,28517.276888551194,41560.85308154171,43999.15897330077,21427.19650317167,25084.131072037453,18450.764512377853,38182.39266803458,20422.517191285966,14090.243196787245,21998.55223596247,30809.187151501046,22071.90474198888,31906.454754357772,20763.073093214778,25620.747974742615,19248.368008063957,23289.011422937554,26669.846131314298 -RS,5968.673897869134,7450.873145908507,8457.218084424714,10466.134979346216,18700.932768467817,18783.264457747777,9647.157790116991,14650.396388121844,12360.14810912279,11640.673528541236,8882.446980807292,14762.619600006037,18399.930168952076,13646.806400326906,14489.119148714197,14104.804002759012,15462.34970923992,10894.228431882033,15095.088711819322,17853.281396922546,12916.46431873845,14207.379133135786,14418.834370720482,12752.5957180177,15544.588217112372,15820.904121022999,13534.055257847373,15215.146639151637,12996.626380040323,13029.919161637963,10196.668003968522,10284.440766030819,7022.693160251435,8401.249663323453,8435.803252725058,10265.915943082338,8623.546341166757,7782.429841191159,9309.259412625106,4849.32865233776,9480.40891035497,8237.330612702019,6688.9341497267815,7113.796951355988,8945.392918392941,11391.406447517316,8770.78174795334,7797.074404297926,11205.567652241964,7490.638744688466,6890.626375097134,8721.152535625733,6986.899718851523,10014.515393386338,11863.685949406314,12576.12226693591,6930.903954576986,6933.68818306879,8436.351253425808,11680.836931112251,5673.257881932308,4512.930178739657,6094.8735580904795,11616.485592765608,8423.704814584986,9582.260510301834,6280.960312130538,10196.683253538626,5869.428152205924,6637.126459584064,8223.324863109141 -SK,6885.47211985414,6956.591212284293,7116.928783228905,4573.269245439668,8169.130903254093,6799.092645889461,7190.199390340817,8819.311158887818,6679.080394471699,8681.76846617681,6714.688996801623,8780.135433744465,9674.052829336924,7510.89080672348,12315.672504913706,11551.288938768861,10658.095118642217,7760.467099195778,7334.701885838525,11743.053621084162,9101.455526104724,11252.254293590808,7089.2297957961755,11692.575497480175,11870.620362608648,8320.411050776946,12664.612697391287,10840.053026355938,9544.145158945727,8285.851845920657,6554.955003499832,6014.385824757616,5119.229272436076,5566.197178944401,8283.751441606055,4576.739002618681,5529.727054286627,4884.285979430572,7030.767074231632,6380.185010419688,6092.06334940043,4789.6919845964085,3988.728835990378,6100.445127127096,6174.336930810738,5839.492530664778,6739.897695678873,6073.547495791424,6239.663814901599,5985.626769812626,5946.910319392802,5350.4706690989715,2897.742723724252,3812.447551889599,5110.734824948982,5466.7571969447445,3822.1224333717755,4189.745228198809,3608.9397299109723,8925.718578619328,5520.297117401385,2338.023634704523,3634.014208205775,5079.021357670912,3789.3939110245783,3599.858750509022,4222.707040620564,3096.1701009538024,2158.8092391216674,4542.203594136759,4220.210061470592 -SI,11752.798116380785,9378.602450871665,7169.30972105447,8221.267552376507,10568.142173580174,7705.119699907013,8559.440940655051,9496.2682906945,10641.418485889739,13644.09405840664,8756.865265119286,10877.994032024846,13095.473005855994,10408.222250010325,17110.767148034232,13257.44359305617,10163.420442559087,10743.809603872312,11213.570708968457,12419.028940460184,8518.188189494229,14975.496301661744,10217.449212514946,10521.007791805137,10759.209828867228,9699.803414833696,14530.647692616923,14530.715084460895,12439.143395173185,11990.046930204517,9001.647305603035,10111.423546501868,6824.172499114327,10267.968760960974,10233.709492652128,9562.175707725224,11231.941186997657,8178.692540046128,8568.526412713521,8103.883097000008,9639.895318751553,9157.43571368136,6814.50158997294,8089.913064615838,8170.645053083449,10765.044719853075,6713.170108103339,8936.8362631332,9542.147586182518,8964.110093884974,8653.114721452737,8353.75428643612,4558.589088593759,9432.405903987477,6927.92152111369,7408.760392367348,5834.871486466167,9336.912851251047,8839.494837985663,10766.693939731143,5073.937522450624,6475.598951785893,9482.419717110955,13414.771212079502,5906.5584945059145,9942.151265372298,7696.092255607837,9521.835825354157,8476.124990692872,6361.010018809483,7504.908031281684 -SE,134822.12256461562,136958.6213484757,175679.6273876132,133962.36617697845,126787.36233921513,130861.49290945902,151133.2982760823,140247.52694114187,130508.52271890236,137421.63579450184,164132.0529341604,142467.57286064525,135886.87305352974,163762.3962091317,161128.38161511483,148196.28633931337,167892.87677861055,132017.696130452,105953.99289036485,119407.22923207263,147139.6039546916,151309.55114963037,164893.61323375857,137608.18237925606,149284.78771947857,108749.15747050248,131859.531763871,117290.48017785426,131751.97960784068,111117.27409251402,110551.615114603,81550.1135393761,114677.940027603,102356.46697333359,128977.80283219383,94483.98267503879,114937.96457142044,93256.91327642264,96751.29026217168,91074.34139788701,87931.96567208324,96560.83822983531,116746.94389793588,64626.25905820224,75156.91767736348,51489.37512699482,74729.26978432719,134673.01513808029,110440.65771030058,149033.9827088587,143576.83069821307,93040.02354987273,64075.160200006496,102493.08906248632,106664.82884715112,90351.43771324471,94801.81222609026,92445.93458012305,88919.20770998915,108111.72872590217,127440.80298482065,124444.06618883039,98961.91573013023,87913.90602563386,115956.48217115598,95526.84265190935,103459.08985036932,86058.90377748368,99040.9103276256,110493.22773914765,94182.7249029169 +name,1941,1942,1943,1944,1945,1946,1947,1948,1949,1950,1951,1952,1953,1954,1955,1956,1957,1958,1959,1960,1961,1962,1963,1964,1965,1966,1967,1968,1969,1970,1971,1972,1973,1974,1975,1976,1977,1978,1979,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023 +AL,18102.536984526414,9362.260840506311,6061.862820153789,14283.904408113205,9819.184491956023,10228.254519456263,13788.284450369407,9581.270108803017,7938.384836547995,9040.045836743586,9921.845743062182,13900.722955927391,9710.931748556879,10574.663320350839,19341.246363906943,17818.135264440367,11442.713887022777,18086.801216289034,15090.128262650946,21363.26299103702,9091.698545327283,15645.252953546536,26744.08978638467,15193.032691775385,15984.127347228396,21350.165383112253,12855.252167834218,13420.929958145927,18491.92173822311,23127.97840031047,15647.322475185589,18621.138651647332,15899.845988546795,17394.46706381624,9941.517004312835,15056.517227865626,13546.145796746823,19078.487245825123,21734.889469128113,18681.018783118114,17330.7404590047,11854.58505075189,8621.019687739607,10958.633158319248,11928.114867207425,14053.437518158384,11478.547049310142,9794.162917024925,9309.704198107493,7531.383319836307,13417.032891488734,9396.693750356952,9409.269914411027,9602.245346872442,13721.093186096236,18546.848247572612,10075.274288997707,13160.130634381218,14979.3178626011,11173.442130939595,10656.167120597063,11333.102860367679,12218.649211273396,16968.134523176956,13531.444805168734,12351.485474236735,7777.709342414358,9395.578415692386,13162.74078695902,21279.522933687273,7131.262505063414,9979.8596278608,14852.708775826433,10550.911120633964,9976.081734060937,14930.01135394502,8322.107101243482,14871.511785547276,8735.240828081134,9201.521447196284,15965.927842545672,17077.541703886527,14104.446634983404 +AT,86531.51734236983,70042.82795415918,55673.922583448926,73201.56141483008,77917.51024365649,61393.73261184873,55048.20583071285,91666.41522182143,63572.87951011166,70463.32537160354,82083.36248472283,77811.80736180999,72623.28291775129,92135.9214601233,96990.37023646115,85097.60090834703,90540.2512560922,87926.80279582532,87840.33332785149,95771.51050456028,80100.0597367668,89961.83970292915,94250.95465501286,83836.27750897587,127502.64206362939,114305.16098508416,109841.9915507258,94950.56912780282,78607.44609072198,109558.29937241938,70075.51296406337,91798.91825728992,87743.39136454638,99172.3185471974,112079.92282492586,77403.35720896415,141634.05110901658,127491.44958971704,103513.31397365611,94072.4808516721,90719.06004331182,74178.70850428617,71954.43763473642,75430.93523721115,83080.51606488624,68680.54577957527,88997.37641681913,78365.83036558417,88108.35920540475,70055.7892534604,83590.790849829,80943.05621285089,88054.73669568839,64323.72991620271,76516.31784700113,88537.52869871816,74980.7275243927,76703.23963260197,90699.65433828124,82618.46892777775,70279.06284438398,76864.90473420777,38883.771847486016,64333.37459685728,71147.24783545699,61897.64030624322,56722.92181769331,64364.47029441406,80300.64217575881,73731.45518976511,60840.309523971024,77575.9681355992,74925.98195655535,87076.27014563217,55659.14157544586,71826.27744178762,65790.17399828041,62758.61938425757,63451.07652587882,63443.71962749899,60531.78695214822,73567.34222821795,63977.67627260971 +BE,1961.4766789766115,1152.2739476527088,818.2925521347668,792.0469723263222,1593.7272156934505,1164.481755817633,873.079191781097,1304.591358175381,1034.616549532147,1222.8331458954544,2265.3933295931465,2299.7187477495568,1601.6287456097295,903.9162052526016,1681.6717933948885,1553.0826978024484,2040.8513201770277,2596.453742857211,1572.1256345899858,1777.5813929080084,2626.96550179367,2109.9419394561014,1095.552430877252,1153.3090754765356,2926.2047527605378,3273.3444811813465,2455.1462895729287,2709.4254348359373,2044.223080485547,3206.6821776003253,1403.4368531879622,1739.01009962226,1509.0861879549796,2489.4865224203572,2440.1073430603583,1152.6261514941527,2218.937185944037,2759.156597708309,2657.6330859359005,2904.764428136616,3657.83503348807,2493.2486286738635,2917.270148862828,2660.3113890881846,1897.0141168954808,2027.0116832611525,2822.5902787959144,2964.8457470488684,1943.761175974426,1192.1713008070133,1419.0119225208746,1475.2557281855488,1769.9742186132778,2647.8931490311224,2842.9882619071104,880.5015070232031,1197.5019163599245,2093.4616667083587,2614.387699318041,2455.605567526133,2964.8707630509653,2512.265466519676,1585.2842393930487,929.2255532772314,1281.2863306004592,1121.936801130382,2183.018089275124,2209.175694512598,1603.8823576630593,1544.7801250586733,1495.8450265096008,1698.4561860883377,1826.3476500921734,1803.4120415623293,1828.4613155494794,2424.4009518987536,1091.1596112517386,1814.2183897620703,1171.4609814371709,2056.8674333110275,2031.8962335362714,2907.836627473,2067.170862372187 +BG,17071.604805088016,14499.026058918833,7232.927769745818,11963.806716811183,6134.985167502051,6976.20752617876,9144.48113078865,10476.930594688338,7275.174931794049,5983.600586215025,7806.165358407907,7617.879584469116,9313.915986883927,12665.103843530671,18165.703005314856,20925.111917073817,12315.95137614649,14307.215196342033,11652.743586791588,16042.778215050708,15000.011504505326,15411.367697482856,23959.284980030086,13749.537818599603,14477.782616320155,19301.913180173906,18779.64649297375,14471.754083245778,21905.976678210158,20337.934359162857,20009.368056844236,22642.8256575709,23291.544303799303,14596.266505770593,16672.574320967968,18465.93977853061,15123.0740003407,16401.953223276385,17509.31598177905,19228.19892146255,14599.539868168713,16192.551398587811,14129.882713144674,15395.87601647586,11853.583066311567,12273.885630367266,13296.879487675578,12962.17913948413,10178.141531948864,10101.748861881337,17833.95419276747,11290.360773903032,7050.797194092144,7109.831367133397,12879.727662239184,15849.231621819281,14036.330428384765,14600.701492135015,14377.532715491632,7705.7385359122845,8414.14109970378,12768.14503989922,11390.56673556146,10492.153643864705,15665.526881990423,13518.231074072748,9896.670696858979,8514.724851169687,10495.74229946015,14331.982881842356,7228.669024198985,6779.132502990354,8418.829006301234,16447.848686603495,15740.35987338571,11179.883037182273,8789.581276892988,11591.504961624405,6993.71096793164,8046.946588923506,10797.966046345515,10462.980330774792,8581.638814176174 +BA,31614.29255116213,25762.18251896169,12898.683737355397,30643.34571996033,20289.25506629573,16692.532556519098,20416.06449106326,23823.10383354778,16264.762900161431,21522.105898137404,23467.52324346034,27732.028396263388,21355.39127202247,21518.57483169428,38198.02048713121,26864.382517947488,20815.045478598513,30820.77172304343,26041.454718573805,32434.848569531026,20613.68138756547,29427.6067241259,33216.04695823398,35126.00882914745,35434.02869793576,33198.240125653894,27292.13451964922,28870.288475787063,34855.527471322755,40158.11721181478,26000.45986387781,30818.815452611776,25386.059210704487,29741.547122355205,28211.934365516674,30030.474291377286,28262.88980319508,36551.088477619196,29006.131913946814,27659.528119372964,23600.138376555304,18313.888493525537,16009.416242480263,24722.92263318226,19189.890721764787,24683.705818314953,20989.732351804592,20569.866983400992,18470.90664070533,14851.755901997987,25891.954823722892,19819.186826010166,18921.820235243023,20522.576037546467,27525.961934821684,30713.034955646464,21412.918614981576,21344.870107122344,26543.94285666588,19942.69654945001,23180.52553450498,25225.421682786062,17217.841128593398,25342.758536033023,22647.27040540609,20311.586220306715,13360.715771642963,18725.9052067595,21814.111731615874,31110.03238839381,11170.209355889527,11624.972537489217,20618.808518471225,26321.87815471064,18023.46871599901,20293.60245322545,15351.515268991054,23732.8659965382,19641.494693119246,14018.69083089969,18476.061736021245,20106.92588679312,25396.55419340511 +CH,71147.26157637448,54904.32201155619,50017.90208838156,60290.759721214985,73027.45831149221,73481.642346746,55618.537744757035,90032.09246974687,51962.38939916655,67177.84116696635,96134.70662377033,78280.50065735803,75567.82823680741,75624.52378636891,84437.3353706146,82858.22758683441,70824.92066589095,89311.09471805945,65693.73271567532,95736.9646979805,81823.38979168108,71941.93730231738,93291.72439376163,67130.87226350856,103459.16116112942,100678.84780651607,97724.23622167954,103707.47043615916,84031.35407618036,103162.60292359442,67550.72288370335,80436.85045155323,87045.62560332651,91431.67032188186,103676.61542793321,68303.04781353423,112322.8911282786,90003.61666202937,91028.65800738563,79435.56403729375,84362.27602442491,71967.17653868764,71356.17066607098,70327.74412176231,72852.24726422041,60501.17532436925,83372.98013161327,74234.0349385613,61890.14596593614,54336.52856303606,58271.018596959824,62644.46734396249,75790.31562737533,69637.43067630196,63290.99710001813,60360.96649160057,57476.7610131337,62343.013781750116,78693.52307555292,73667.98513148604,75122.7031133837,70327.33051388402,33043.59766466136,48968.55545142022,47632.62274536815,48588.59714178414,58398.342285916435,57742.38347363473,42559.51792407362,59408.23894223828,57129.80243604273,71163.63652517287,70858.4268509181,119263.66699527185,55000.17388464577,60563.73895679769,51249.64855691976,45607.547672961526,60542.786248766475,50488.945849740405,62587.04259357905,67091.5970044437,65549.52567888667 +CZ,21212.78579882691,10928.058379661825,3384.888399390222,4480.397678992209,6399.719467990146,6768.6515946008185,5799.446185009577,9411.786777037812,5860.23463242501,5599.396992120427,5741.811363153907,6430.8701452824125,6608.002211449021,5547.98412657507,11139.03676570244,9975.869136026628,10340.80877548634,13532.930674695574,7705.735677184478,7812.801872929193,10964.355351867012,10646.504003233118,9719.560436989883,9722.51782291833,19015.37334991238,15549.667939948402,15927.255735311868,12882.876953226223,10353.403919517015,11853.576269226234,10677.61771918624,10413.95103805089,6895.283030051313,9424.903567027312,12922.392855988994,9137.874616036986,17532.348544152504,18533.830228515635,15250.542039808313,10190.043701741004,11537.335951365127,9452.574932230946,5833.228549596612,5329.206957903766,7580.6976645678615,7069.4638232187235,11641.01844987317,9790.145937530902,7320.767578365769,5406.873869997462,5625.637231646826,7239.835546851702,6440.471526964078,10089.617439184036,12468.55796361433,12603.058166229213,11520.381274473184,9364.18293733304,10411.740148030329,8903.525346614548,9434.353552398077,14463.707947162726,8297.335864348554,6305.0428212307515,9383.102792968806,9420.905095490103,7053.530866655102,7934.504156979587,9131.766698685136,11027.709419784474,7973.011990884055,6406.065926850977,10728.184944007047,6645.433895571247,6345.82145823084,6499.675988954033,7016.463805946017,5619.135413529,4807.786670051716,5785.96568493957,7986.497102061061,9207.147123091028,7072.340654546769 +DE,60501.03103951161,38175.41955606091,20879.426305645527,28284.75340336476,36103.32351052005,34891.089702429555,28788.726755549767,47593.248184570766,29278.51011479424,29170.60074491641,37514.09293381361,41854.44001450301,38958.042582957896,35908.26729908446,54302.825387264325,53900.0627595872,49537.22949846474,58655.764340031456,37627.8257805545,42967.11131060519,58392.899587658314,47246.42956098116,36543.949785170116,36184.44504543948,76454.75505815547,77563.28089874369,65706.46393994696,62713.75591115003,50646.052126013274,74501.93068438585,33965.80961685558,36936.195615288314,42537.597915180464,56616.68921207296,56744.68927682596,34807.98202654087,56642.02192030585,72249.78534694383,61866.53633607989,55155.47864420242,60392.13262306467,54016.67652252674,53772.61604029787,44957.681101902934,45340.54136937088,48213.458592237475,66703.44582319073,60862.17019199549,41014.24992391146,32890.656129618095,33561.7114818598,35496.398391557115,43529.12222860253,55750.09930602895,59293.24549493131,37708.06634825865,37073.71374151417,42227.76427477898,55642.269571505705,50261.32889407123,50099.1653624443,63663.80803911536,33932.84222448403,24148.0930626651,39327.588161226195,34424.75266875371,37774.7468462978,39904.36386097259,38761.37664135111,43371.39725264248,35879.97041420226,37163.99533546333,50391.241894760955,37073.97568394232,36139.75677566728,42435.96482981362,34097.15586833773,36767.50341472353,28228.762854078665,31841.550922412007,39703.58319343355,47521.52513933251,37833.900158720804 +DK,200.778317198711,183.58783222570864,184.92589840547043,103.64850861603549,315.2220887719417,314.1000544312628,214.28103955721934,161.43191114338694,185.2111475627714,358.69279426367336,540.6494487174206,225.87222027671936,240.57818634617777,370.37587945377,297.48979685029366,176.69208006156848,208.79213770008658,418.4554011794129,292.8102234525168,354.1220188699609,405.97396851309696,497.6042583966885,394.062076725445,291.3295691524654,304.5438623304028,591.4756757289098,629.1912050409122,479.19182164984767,353.75286712188796,454.0799499391422,338.48620130918835,357.676218196066,271.5200039604648,288.04880490856954,389.1757124784286,200.03398963977796,391.555913715114,397.29527898061724,379.8043799580585,514.7861201159348,505.7304478209306,381.57816134966373,489.3522161731293,301.717309458533,340.1024114463131,377.00536197863613,349.05617800662674,489.4227950883326,223.23589869898433,269.17110101837187,288.12441629799764,229.5796362727797,293.635637901186,464.89056739838395,386.583686695923,100.62743648672041,155.50324274468716,357.89273417380645,451.04208007822723,403.3432275318107,376.8995908622044,395.3896113505339,169.30459202493293,211.16687292699712,238.87912856891634,225.9981737610454,469.5229598902774,334.5021223766927,213.88993390188736,267.6283833775911,319.04746406617295,376.09256801086406,317.617725512616,353.26516627601364,404.2285931109035,408.5235836651241,351.86517079981496,305.15203377590285,361.8170659996248,441.45504335438466,312.14518786073575,466.99320427604425,510.02659659134173 +ES,78876.635611407,38950.09623429436,33841.47292664868,23156.438140457663,24925.319205598968,36553.92702862067,59420.41145835762,44448.20001757085,23003.718514410986,25989.029770929596,49381.7181466273,42506.42487287232,29823.22966835847,33520.89863652133,42189.87541544434,45018.59980204108,27387.4917761516,45602.58407624375,62626.425244916936,94141.56249288519,66884.65469167147,67115.06386526441,82553.38984370838,62483.82642486853,54633.78478265801,97009.71651609142,47073.863176177474,55800.125131342014,85746.55555964066,72028.42027117872,75395.28981871007,97165.39237066667,52196.13272216457,55183.75674414718,44434.367388636274,45688.29479796517,107175.1885102772,376352.4882722023,122222.92956502433,46652.32256081546,42283.296882354574,45745.627884904505,49745.69531403085,62249.891346371536,58920.93920695715,48705.91089393512,49506.55303300167,60104.870297070986,41417.79510072781,40561.22111509541,48584.70074175896,41980.59062088254,40901.640882458836,43899.80095828514,42846.07384845135,68844.74333042726,58684.98840324564,44556.941393617584,38670.78308215774,46815.51322723987,68212.19494117319,35011.91030341564,50475.76713743041,37227.78564631605,21874.579431867896,35771.95151620915,29462.81813629621,28957.890173503936,32351.242924601538,48787.03990347552,31409.59113067243,24367.630996980257,54440.48961905899,50212.19464108009,33563.81874118821,47043.00929278407,21662.04001863739,49011.344970588914,40719.843457258816,40971.905822060144,45241.676643808205,43434.78481736922,37030.44475893774 +EE,539.9157561831089,506.40067622283726,641.2579701177458,634.8633672657788,801.3715228148297,966.9758840524717,584.81736437269,583.7023862826877,945.7850249371241,794.7615397417561,1013.5168183220601,680.8524541483096,896.074617741102,657.0814978199126,1138.8891299468455,1033.2360728791764,1360.240006093535,1004.6950457068739,934.8774758153143,610.9977141696028,1048.0576885162222,1477.5046721309996,819.8347754595931,595.9919701101757,664.1717649435486,1178.672290642306,1112.7122616720162,1003.3658496852601,739.8499843054013,970.7396668771637,1156.5986195943308,988.0643633892423,963.0153554345143,973.0118510683724,980.7669857762866,637.7376743732589,1121.0664931638019,1485.3090062847518,1014.4287893275732,701.3843644468006,1196.8127434491068,1136.1215677919786,956.0911286121622,761.6620263988196,1109.6559596693944,945.8137139982223,1246.4435124663337,1081.0063689141978,1180.9951778617858,1433.4526339800395,1044.4962572791317,875.9289554452553,699.3911738148247,770.5276589427534,923.6083617038047,353.16555295606884,572.4961222214871,770.5595680360855,811.3520528619258,480.6204379373417,526.2783934002991,624.0252970776874,317.7449195254128,502.6073570248072,523.2149143909173,201.10844695565802,516.5064948459781,787.1638506300568,929.6905631193786,943.6792182991617,682.831069893961,933.1072250631543,692.097322772721,449.72254117947216,392.4173074585239,635.9676582877272,753.2897252561945,692.2577760060705,496.359994444626,685.8431112730673,512.1207984775516,963.0464221974397,483.833486345915 +FI,19863.633124385666,19889.46363566243,43612.48513693398,40476.15110774358,37963.60387525601,30946.988617446677,26642.94999177761,35266.32524904611,44524.796955378566,33134.502072929165,37605.30609105331,44212.92045299994,43354.55443946037,40909.90249435943,45502.4817028443,31224.476747778986,41198.39417754888,37433.6293362709,28205.152814527217,22733.614552593714,36790.55908786393,45364.155643266684,28818.806516416316,35920.468538637644,43898.27049057533,41064.81684006838,48740.383821508905,37939.33952866352,30918.72594548001,32676.177851908644,33754.05667642144,33605.2810320407,40213.26757169686,46999.65997165162,40988.91471061698,26205.15827640759,41167.74001571865,25866.031354509407,34538.39918373158,28474.82496101674,52752.27216474974,31970.598763267328,35752.18054352296,34628.515223969094,27958.317625583102,31764.196982878522,30015.16030263079,26346.04561795855,37548.80632649389,27745.021972655202,27337.35846234453,42346.93155636462,32896.698992597754,22520.879762748657,23208.41547315833,19884.31515309925,22475.195958848508,33947.011695157635,26939.65172877202,32198.274008146243,28319.0878193112,25068.601753935876,16735.56947336933,29510.104024439464,31518.153863523567,22455.334677451585,28461.18834411878,38706.997170480194,29619.48059832548,29700.763598820195,30127.95453034573,45678.82538666528,31580.034602099753,29256.819253905658,43076.06538538077,49489.04536337738,37281.852766328535,33818.560580958125,35465.53242057822,41410.545080415,41567.72109759498,59630.270126901065,45592.22770628446 +FR,104970.07553291276,54987.520121202884,58127.609672438644,63952.96688103039,76761.15501948349,76267.11605975665,80986.06867218221,93733.93578796726,58479.255039973585,73543.6737690567,135094.49763612094,106772.95460342853,80280.31347296045,74490.3783289433,106687.24245156738,78827.6552216503,82482.11027901788,110101.24506868717,102231.57173723538,134019.59611286796,107848.78158188045,101612.59295931137,120079.53146371644,89568.66448178052,139585.30725052828,153772.55334552698,108799.90779570332,132530.25240249874,145056.24077647954,157344.40874499045,121182.68180087653,135378.15307040678,100408.75538533325,132164.7891142495,122382.6612084835,103706.50937621434,174240.52624537304,149853.10658224527,139992.58441391244,124892.14391178945,138893.07653615897,136102.17909683278,136298.85434619425,112484.75227834118,103842.75366698352,105453.40277549413,121935.76619549586,138379.45234918175,71924.60006455882,68830.14474115185,82603.77634528138,113556.04015089742,104293.46882473862,133822.28230897718,106263.7222156586,102481.34621013676,87080.48957235571,78895.08319488679,105735.9820525235,105626.7734987112,124313.3006359069,92144.11932119,73537.29646152578,78137.60525599915,58286.814348802,68928.05802270533,75743.7796112372,83489.37862547796,61518.93775410048,71311.23184506799,58810.65089438103,62581.66694545332,108566.1756738168,106810.3035898108,64981.51870529863,80829.27869961815,55203.39988738071,99034.47679069567,69318.43286028007,80169.78942387344,91015.86505778233,86376.11895764538,79293.98493428944 +GB,22702.364220315787,23680.52284249295,26272.032009932474,26333.09456007779,26845.290009259897,23656.475751452526,24568.79566354481,30136.97835561064,27079.44100895935,30643.318549445074,30949.55733300973,26701.9428167181,24834.423901258084,36292.29933386541,21223.79048684261,23695.358277919488,25987.57619398849,29907.138025215598,20532.262685247148,29689.355990879158,29755.7349738537,26456.609914217806,26827.642866860373,23873.145060634775,28779.074240844395,32368.78110647752,35977.394307138864,31071.326823875082,26446.212227329706,32888.07880400224,26015.735528737063,30161.596277105604,23017.874991984525,33287.290870143,28286.810808036254,25581.658649545065,33234.01349421005,31886.616851943854,36147.13303532815,34133.94123938026,36553.158784009356,34512.41082508442,31536.090754720717,28572.314934249887,33275.60304522956,33665.001595891976,29575.879486696827,34513.91395990196,28917.71257427291,33609.176553253485,26240.381135352633,32589.192149993025,28862.78621634025,32807.4074928576,28488.125666145854,20034.93309228305,23116.389736102017,36396.90134863421,35532.192401610075,39288.48780281492,29384.81519197965,36157.931892943554,21622.405196644995,32040.743170655453,30471.91526631006,28610.855281883465,33870.42864117695,37281.10394972126,33227.13302814586,22275.90424618414,30588.672790028984,34121.358436958675,27706.739428180073,37230.134388487466,33699.78058075289,31403.25713954576,26791.643973463473,25928.83232662774,30294.933954035565,36884.703709615125,29006.326905891798,39928.21741502616,32593.01729082114 +GR,21241.640655847503,17837.993739159116,8331.553303186349,18980.73391736116,15389.589824927982,17680.338927422024,17876.06808235439,10232.387409542045,7698.396804685536,9934.422230608423,10056.853784361187,12820.948661755581,12880.09713386794,18009.367077161973,21378.182658177288,29473.438101212807,11770.791493072751,17561.986308683798,14068.336209442465,21036.22984593777,10600.651816726047,19253.94027167335,33903.71393281602,18899.558983910043,17444.093386496537,23383.013428670623,15695.74724949063,19981.88173635776,22837.72155515553,20477.278624856677,21882.21250746782,25630.65513791745,23207.036995448034,22044.998521881807,12010.599831751206,19432.667576547075,10483.216379681078,17709.119022396855,23856.615736538963,24868.152661457774,21212.948012018598,19349.21918097175,12988.230031417628,16765.76278511076,15711.921661553248,14768.388537620414,15953.652684601051,12434.694106124754,8265.22849298033,8609.897588471382,14667.4570613014,8511.641075667938,10022.119539267658,13526.966234241496,13788.166042309897,20514.65575034389,13890.655920759986,13637.35506363307,17713.539092926785,10603.937587040999,9085.188935046803,13431.534902143163,18237.093734480102,15007.784707319466,15322.74478671021,15189.508916238075,9627.363919501071,7964.610286111404,17730.814440852053,21625.83796282357,10692.331430067703,11630.957871879938,16037.311332863352,13770.795305947142,17211.823293162783,14042.921372422215,9179.230226001768,15781.789228215457,11036.807907632012,8148.5700551254695,17837.056638784255,15741.120020191216,14783.113316616393 +HR,10616.868981781858,8773.16604903597,5244.73498702366,9536.576803956808,6559.078522391053,5970.043528122038,9472.615822894124,8274.401876717038,4545.145600490753,7872.286618923516,11579.119396857217,11616.785819262946,7841.411279590863,7071.274416413621,12992.083400898755,8164.444623202054,7347.788717816701,10332.483465428491,10537.238741811467,14479.677080073134,8479.661662510043,11787.072656273116,12862.758623825672,13371.003032039336,15990.668161905827,14470.61427957732,10644.752476691014,11929.91211438134,13611.162472968235,14159.968181810045,8748.9998436583,15752.17883640428,8576.387915317433,13121.48137750286,9754.915357083264,10574.934113880938,12339.192872828058,13526.82683401755,12407.125261525374,13314.467203516846,10589.407510915647,8240.264291214513,7438.667886688274,10422.72691638502,8167.94266416981,9179.047407118169,8517.079324039581,8574.526688740114,7000.526114186264,6044.501432950472,8424.638032244015,7640.079353180473,8493.38568955205,7673.466816227631,10113.96973112034,10475.583854629525,8161.447547783003,7730.598307694147,9688.392563588315,7721.732973546704,8547.91442119075,9110.630144082634,5654.409914846469,10449.908940883637,8544.776426500835,7703.363550389989,5356.181480736791,7888.76450496884,7981.496673656482,12159.302077747214,4232.3935344941665,4890.509308451722,11184.320641158047,12936.9950624162,6984.475921997467,9245.055929521772,6446.204647314787,9728.03403894923,7468.897556955247,6117.536838412799,7342.634549696792,8157.817278196762,11474.85293662869 +HU,1086.1958142470096,978.3249526234807,212.97414604675294,340.3983517473703,430.1881201465951,194.65656630284064,473.19850543225607,261.9192709197555,196.10367582160737,215.18183697898402,455.56009955160323,446.6314758325084,756.8984035642842,450.9800559529234,596.2398492658431,618.0772176598714,433.5306920564666,626.6699838105968,736.6623236995524,901.6628598550868,955.0588428000881,1010.9577650474141,1739.1969813800204,989.7527773708749,1794.409065735425,2141.779921355298,1587.224990709883,671.8419282109866,1725.8355118081338,2716.2211930211206,1024.7764676942138,1478.4297579371664,1174.856199808959,1380.70450266654,1412.5023358393119,740.4722294240396,2077.892875467242,1382.2122922975911,1644.3319455706533,1323.8387691522125,1268.4319727220777,1050.1561979992803,938.7640185853782,701.18673897032,1176.2218406698482,1296.5341770126697,1360.4975641937087,1113.61620590335,686.0888933448485,340.9577501900905,585.7771480398817,710.4671894364953,683.5190908209308,901.0768559915952,882.6855336512624,1696.8667744778322,1072.391017559335,1091.9628754375356,1986.255314199332,1224.1891103332646,723.3105004079888,525.4091579835733,484.84230756145246,965.526984284363,1182.1048527111184,1188.8833398485922,413.588170351592,480.2721461089437,695.3047160171476,1875.1966820116616,1424.9690036241302,335.4105482008909,1504.9089491398734,1485.372787096725,1238.0161031995012,1379.3214547239281,769.0933902263423,1164.4795232335282,401.1056623458921,586.967649321741,686.5594518266915,503.19975913371877,1084.6060135452235 +IE,2938.9562959168748,3198.9169680516616,3187.5912945813584,2803.8552573374072,3777.1449651405105,4362.104175198847,3412.9484028266015,4033.7677356343165,3724.1620931207913,4522.92029724601,4646.742323576806,3498.3840852321937,2544.6578209214726,5419.077105100732,3195.826410242899,2135.0862825425293,3896.6031468108818,5166.0575828547335,3059.5430902455864,4420.523688067632,3920.565970045664,2677.658691610785,3238.0778129090736,2814.899041865502,4169.640537659651,4975.659864222032,4131.237385306254,4051.6063579500055,3501.9810380506974,4162.222738923252,3277.97066925379,4149.907266073347,3535.126191101705,5127.397347805918,3672.8392481603883,3233.4968306810615,4670.955668853978,4602.662496185422,5486.794576286672,5974.860239294209,4958.279351735843,5539.220771177308,4814.939874910363,4704.080605250908,5214.372862088697,5579.052373684467,4424.669817520398,5645.170219782959,4099.107666436778,4291.534915722901,4394.7348113611615,4474.842169491055,4574.431496810677,5809.6920444413845,4586.044007664658,4500.049573188483,4158.002042556449,5864.151573833777,5389.317421868625,5602.682945402753,3910.7950042057882,5664.313605042361,3809.963053559503,4119.424688220842,4183.3641097691625,4756.243128847813,4885.19842669336,5839.576896771334,6529.030637616166,3851.468550158534,4739.247453475783,5116.873230654843,4346.868803644192,5780.6460718128965,5283.18966576195,5253.677788447396,4182.318532051922,4579.528423701424,5332.55258145869,6160.501534866154,5064.7000821427855,7011.136828867076,6289.684272273017 +IT,111872.14392769105,83456.12455924545,65177.42783168641,92174.45776985734,93705.54545517343,102552.6760192628,102231.02170173851,123449.91092719955,75535.87312511557,87009.71778444933,172598.9057049173,100312.33974168803,109683.21050744873,113141.95504995789,104153.60992530867,105687.75931391625,108887.88674499927,120408.08444781008,120463.4718424679,176044.98820314519,110983.47718210719,108661.3888675,154479.99952596257,116866.21274767391,140970.07450678368,148795.1431459142,123467.91624840492,141856.2012294592,135145.74559391223,121025.70232844367,123064.2692081519,158675.8479818625,126408.22188494769,122698.61897984576,134048.90222163082,141351.40587318933,183551.69201916258,152672.0987165877,152463.24070715246,129214.69623803807,126406.52003570941,111266.52387106014,105493.22493839792,124930.0152276459,118811.00097800085,107475.13484215035,108705.61617459086,105553.12911266208,89696.47047566871,79022.57929325203,102862.21537887226,110384.19134436303,114736.91039530469,100806.28542965792,84803.88252804938,120635.19844583895,89284.70639754581,91327.36842742827,112272.49254052607,121214.98394688763,100240.69732658492,110327.24980482439,67667.3134977533,93496.05161894935,74054.55044130566,77775.69469931335,75858.44303177306,92083.61659880447,94760.12813659209,130298.81027721112,113068.08204604911,102862.27823880903,124544.42911673438,173919.9577739064,87247.90399947706,91232.2769509342,68511.40136893332,110593.38607511377,103785.63569256017,87869.12433673623,92697.61009264145,89877.21983741673,96543.33467044581 +LT,2330.311115607803,1278.321859711663,1329.382406669865,1835.9539018834093,1806.4585215317375,2463.9928727279507,1216.0039899784208,1896.293414052239,1974.9185999663864,2624.3584436168967,3126.90226977831,1475.6787373817235,2475.307120672793,1105.461669987713,2895.355169108127,2328.291848466701,2773.100951693534,3565.1300060634676,1726.340017569786,2441.2869150017605,2739.255286641479,3393.627534013301,1948.972938917477,1503.9304931932238,2081.2181025571476,2387.4356286402945,2953.7202011167815,2442.643404228762,1575.9037853337413,3106.077653607497,2438.6998492607095,2321.3335398002305,2455.4955733417105,2703.979150849733,2983.428649152806,1271.81865757996,2291.2165997279003,3624.3054978962264,3066.52482874976,2692.4730181815085,3077.131323187848,2583.193918114293,2346.227785803459,1679.5438811966314,3224.9829735377853,3118.546035682447,2841.2519919875017,2793.689370450914,2688.2457239306805,3241.443663108373,2559.3234295762136,1702.002348908642,2110.46892607847,2728.2093372090403,2507.383271401867,987.8894968242066,1210.6320667482842,2619.0625969612056,2041.7494156189375,1077.1024719580128,1501.0630179433206,2093.0805484121715,761.1714789679556,1438.6161577229564,2052.75406588365,780.7629955514127,1806.053023318598,1637.6952437203324,1294.75789506843,2045.7605537026564,2014.9263059368886,1513.2399919084748,1700.3251444304037,1310.6134461272318,707.068706016363,991.8607300754669,2210.5243412620116,1945.761381971759,656.9514438734058,938.7049363615082,1091.9224609932446,2359.271808740987,1316.3428075862207 +LU,410.92374814105005,160.31307322354917,118.4486719933025,98.1780404815946,219.99495836090017,162.18791666671726,138.32051761008012,209.31021249802274,188.8584966582342,199.41682878475427,418.42920407960173,402.65326898169104,245.66041042705334,140.5213247873591,281.65083355306945,239.8578158452654,369.22101543741394,483.78479198242695,296.9130979040409,276.1232951910135,459.8791399297912,360.88050800564,165.48000564979304,183.08661927563494,549.4327262552889,615.540914852196,430.5724249074097,464.9649192455573,366.43220700786026,562.8555247965114,228.93628708423907,211.2546971696525,190.68337965968735,337.11947715404034,376.86900759463407,153.06162250371963,361.9412034686643,457.17182659872975,474.4467062976349,501.35146460886807,586.4359860050876,446.87720467222556,554.4763793156474,495.34599197145104,293.22338952418016,311.9404169698979,470.26963100793296,539.1966800252237,318.1508533879876,216.80440341571304,213.96493869659722,161.79868975175182,244.05142665329072,449.452274496719,514.0163752203341,140.3179752581265,173.96942733863747,312.73695003201067,389.83220575942556,401.09422881834865,533.6829497064967,363.9324325652479,243.9448809708681,141.83825708815394,174.8785415794147,180.93947242990794,337.92060394534246,342.3576188750888,243.21751195509597,234.05807708972188,220.32797632404777,264.38928306703747,336.6463620917912,349.32621250528973,289.85643977541105,421.5786735040934,195.16848080033785,348.6723038250213,181.01976524286118,391.4073725411393,335.53788814460387,454.30136063817054,354.9932023222274 +LV,2075.0062187312337,1183.1651412244635,1388.2608755110323,1633.9975845005233,2100.728221746779,2307.307714455485,1183.142585833944,1669.1622848941527,2049.0129879285278,2215.5671267410553,2742.6556866506367,1837.8325027812643,2554.170761349389,1408.345733539928,2983.189997195677,2825.061612760662,3252.3974243255384,2886.2924715658505,1956.4363533791625,2047.5738516633846,2396.823663944175,4066.8157610189646,1865.1937720377932,1213.3814003625498,1799.0008428733133,2700.1570058485836,2685.678969351212,2489.884608424664,1643.1417843905015,2409.13500668555,2592.9272177648704,2235.207139152776,2147.227486416904,2582.289866216919,2592.1022950383003,1400.2389662856958,2360.0229699498527,4017.8529783072768,2384.614549536931,2229.7220033855633,3208.439621353169,2678.7332051326716,2668.5315414635693,1705.3124319563372,2816.1467857724074,2779.7212753206704,2548.8356765576805,2583.5815153824487,2861.3233286998366,3801.9993997175525,2885.329135114707,2265.350140707051,1837.2956559244064,2087.5337557102102,2249.8356476827344,853.5386928200795,1689.684828383291,2653.3751344813045,2109.0096432617606,1182.3316650686795,1487.9132295244858,1974.1178903388873,1027.3242775723352,1736.361793878124,1741.9204323497202,701.3946960765046,1526.0106468283361,1844.9956663852768,1722.690767934754,2194.735679264339,1757.0892689497684,1974.9007736328324,1883.6056453393867,1405.6090440523008,1355.4439688992024,1295.7921755717964,2182.9434502570357,1620.95881111662,955.2966396067089,1490.1655775988,1270.5915407733335,2073.003327028273,1565.9994997763404 +MK,6885.807602475421,5074.413420314535,2741.8766807152024,6785.078522769966,3818.8527078184798,3928.4886922987157,5350.383608273902,4779.5936944428995,3468.062322874052,3554.9350756385556,3934.258904454459,3692.0813330320566,3668.815892166788,5481.091007849558,8488.277349663611,8514.814821434962,4861.071723840239,6878.625964986481,6399.572820343104,7510.160197503697,3410.068805763449,5394.1653784487335,11852.098687247973,6238.562291909563,5957.282159254423,7239.168545674971,6704.994620967638,5343.254672711665,7690.122171456297,7543.1707758996035,6080.923842798452,9518.22452251825,7931.36999814296,7898.373553315725,5296.06496225987,7158.993856907015,5568.700579027411,6653.618095140364,7658.60790313609,7384.3372015299765,6359.355811805509,6311.4116310067075,4303.248848133382,4384.047563174607,4091.082807237865,4939.915122052565,4226.085259691547,2768.0039025432975,3428.9467644601264,2294.222990992098,4834.495252793617,3325.6396521134557,2522.690466358069,2976.927612194815,4721.490759581051,6218.919122814915,3840.4729075548557,4211.680380445357,5623.802967061423,3683.311545054556,3010.5387111938044,4030.600677147787,3746.016920616422,4369.907262961726,4375.170573617275,4171.022704527618,2702.082556773276,2500.5709190188645,4370.4076446296785,5204.859735574909,2501.3689966568136,2522.4142074268602,3898.568577966785,4120.496629723428,4145.529030725163,3980.5145191045517,2967.8383253548755,4325.530795978238,2351.320803369623,2869.298937488066,4644.825806471018,4443.426947798344,4862.542134317402 +ME,17887.124634490927,10608.377394539566,7624.148328862119,18300.25523347599,11276.651201832965,10903.50785537606,13234.193237094172,12631.716364532,11223.474429487562,12340.195715253074,11787.036249138713,18552.496612764233,11097.228684668844,11567.930061084586,19293.29710814638,13959.20571478613,13046.18440612437,18177.491235848003,15681.709335959564,19349.950981118494,11658.5954682407,15979.557715971854,21966.784229466284,16319.44299708603,17409.40677448169,19267.118980997075,14169.834731486397,15882.152400499632,19649.60512617265,23317.326436161944,15993.83236649745,16724.197023812518,13680.494674103275,16947.38155713745,13396.216553276576,17144.464963358845,16187.204710073258,20417.876685607593,22530.120155126962,16197.908505488407,14818.545420474018,12719.274460270883,9622.464989039141,13369.875886578782,13078.849292203078,14951.85122042927,13681.691957367619,12435.207850184568,11885.89235643587,9516.042262408026,16926.063160570764,13462.909968102742,10042.999218445517,11916.714437943356,15158.632522247093,19735.53511234594,11491.695345503264,14075.890065382568,13952.132515246369,11677.32833931845,13022.018989752623,13358.031703205903,11534.177624564669,17635.48411958614,14124.491791464301,12392.738752899031,9187.905148236658,11106.432128782126,13780.482580236776,19889.70854314474,7004.765863876614,8204.168680103983,13925.208337027952,12712.182323875111,8528.030231315688,13924.890575378206,9037.809310147335,13972.095665379566,11360.358971424077,10280.43536084397,13775.545442858487,18675.41168703515,16242.925847180693 +NL,99.18845393334344,83.88743307052296,56.20087850777987,37.585642699707755,99.89955066870836,85.24103376999277,67.24342284849837,62.39508342046794,36.26442939184667,68.54685542245758,132.53876435280281,105.25304624029256,97.45390390948565,59.629996943856206,126.17294913858377,100.92785922135535,110.87608266899032,166.47617616014315,115.03831975395074,102.21679880589629,222.71184206736731,164.32395798051633,88.95057195947412,70.61286255591416,116.64203846425596,229.2598570422647,176.71083656110923,163.34614915846544,135.23541074052787,153.03725726999411,84.2996330000842,75.50229060079856,87.79575328623356,149.0911400069578,185.89892961221338,65.86854604814957,122.15247506918442,163.86971747802477,177.38160465294413,173.97671792234624,210.94537197363266,138.15528852664815,157.7989701791954,164.8431327190436,156.27033317337316,144.7881412408737,172.55396177597567,195.57239229094031,113.32208112175368,57.91841778157319,69.80575559844598,81.95247611107445,154.25504072397393,213.16593963287744,185.6861094126344,50.88580789479815,88.6770750590963,180.59511129061775,175.09641999581325,134.73866646771805,175.89672487625361,170.10222446753016,112.93557838603432,69.46018341018433,89.82211025011819,57.645723008783136,122.86182387892966,128.20825436811367,82.72980960226074,90.07937846739043,113.22245968360778,109.86527486394354,113.13774281842522,111.87536788688416,113.53271254663122,140.01537175074324,68.64291161119651,101.90820271032081,52.313311323047174,116.8375292031251,120.98114114085548,164.29761287763637,131.20083352074832 +NO,210079.934684001,272449.26769557595,393598.97632244934,344434.1421584134,309228.23588460323,302048.4910019859,239994.31574717446,322040.9441520717,407801.21594778565,321823.44753004133,268338.23623559956,311516.6976002237,354741.82081862347,287750.3093395867,277336.7794878445,277384.3177020413,343804.7636848019,309909.39898200217,279870.8316011512,240316.6988310159,337135.04130569333,320967.163231472,302346.276731951,373063.16157364036,335043.0981929385,271478.1367425455,378240.8226290755,318135.4534804216,235625.6466770364,244547.82624561962,342177.033729037,330660.24479289196,373161.4848287328,318697.1789732917,345399.49658985186,314419.8189439592,230963.73709983748,236070.00307120252,312676.53469423676,195102.01076090848,231756.2941107393,196457.66930311252,255890.52889680286,207980.18388759278,250218.19793663023,221472.83820330602,212315.28230042476,222774.69138098884,210372.25250462766,210846.3448041017,177438.35447152023,214108.95671057343,190417.02324135366,175558.31404488848,199041.30532793878,154342.13931409398,175902.47007757265,215120.2640019617,216733.8944940455,261370.58822336013,212333.63219332727,174324.40313248086,166539.1770850326,202012.88317391058,238213.58919108333,170401.06338583003,217109.81457913358,185684.48673203168,210764.408869451,180452.94255978477,359959.68381729786,220133.80814962948,201815.91045998363,193256.2376798832,225141.35101977925,194502.14845672756,241700.45583039062,203049.47691663646,189440.21996991744,241270.48318453485,196628.83730256284,369452.758534526,226746.71170530186 +PL,19040.141287526196,13600.003649707409,4232.940272903036,6263.550300160282,11725.980915175824,9423.536450722735,7520.9931031649085,15128.087437533712,12302.274212909391,11903.809643162125,10323.210329129464,10102.880830361863,15243.556644277647,6316.576710839603,13844.440254769646,15125.266765937784,15697.445163802722,20547.692524530372,11483.751822471613,12736.801698643669,14242.313560202334,19975.533453959804,14197.61841088385,13385.258603233342,17555.946503015533,19526.389852009346,24990.22970093532,18554.95543612019,10610.557656768206,19356.459381427685,19657.930603206125,18732.50311338716,16842.606570419066,21220.5523273715,24157.44207211756,15607.0096942104,21677.41127186343,21298.26291533998,22732.35472759213,23096.919661671975,20539.587419825057,15690.593097829358,10514.527177352287,9757.003902443616,18214.127020951153,14743.470132504332,14160.372458977477,15574.81487531224,12340.373515759937,8851.92371521615,10791.411372967183,10330.247763133175,10795.54750377256,14738.227630388099,14374.438386354948,12177.648415014886,14450.3889852054,16473.394758962142,14491.781691174727,11650.64866730713,13724.64695911459,14468.081620316902,7425.533290488975,8039.550610683221,9468.54351850099,6656.34004885665,7842.310345436549,9593.638690519654,8828.256800239144,15874.058827778785,13537.338641985261,6174.396712827303,9509.500411581042,9436.102173346171,5360.954908688417,6309.399306468634,12402.024613286878,10602.512257049677,4725.103685514244,6963.3130148445025,9124.964758601689,11007.49968811077,9338.109838427506 +PT,17389.910347609384,7956.13238827081,8339.728858321132,2075.556471824434,3081.9400195534536,5428.965709165979,13519.093138173732,8266.183051247071,2305.611210847845,4751.414112923152,8372.129135066036,5969.31083523099,2397.6586063764958,3916.9962424030286,9654.699566273128,9603.363148222781,4163.162870337118,9179.465624121007,12131.854156488425,23406.646477036124,10635.077955993413,11029.404714084018,16475.917880350513,12087.55769685171,9804.492049893286,26178.184180984652,5669.820254013018,7559.528320921835,15405.930172416956,11163.387795923936,7088.025310785357,10191.330496756716,7128.069257230898,9335.631563569985,5236.13128329125,4572.251664471752,32605.577182903042,149323.86523586328,34294.81803755196,6061.11721308257,5836.654497517657,4843.7020037722705,7814.782411245904,9395.759000250582,14057.157961109715,8192.288977642056,7655.760207241933,9203.645337709624,7637.7450966202805,7971.812432879156,7421.37678917759,2935.2871895485287,6329.939163930326,8820.796560748415,8944.109217212255,14719.27363438586,11050.779819799696,8379.75073967695,5590.525909815882,10485.131539563748,20476.028147427794,7715.162648805314,11939.81827695821,5116.2596877306005,3047.0903835389504,9273.489011971804,5135.313604526994,3331.418010762715,6338.048693933388,12105.616460835923,7097.468638389797,3348.213951094401,11773.920694867731,13465.586888738457,4379.602415677836,11394.463621230614,3104.8789319036796,7216.150544702967,6653.765864361187,5720.580737490315,6685.493238874142,8486.079863634486,8329.192253601708 +RO,46545.50823036229,35517.718755954746,12007.97895282227,19448.903832688356,20440.973051620913,15132.394486734484,21523.767710605425,32075.68337603852,20312.088233477134,14653.654680967496,19346.344374357534,19295.50358149612,27702.617554877004,24644.67023787785,52797.882559506776,42262.7162454671,34689.01143849651,42583.74114324521,28973.147767939397,36882.64868179707,29063.471368254675,39575.626278602576,31898.858414072816,36283.98261870408,42207.50410750205,41714.957670180156,43270.313090823794,40474.54716426176,53565.91349208634,74865.02611452111,48190.11154819992,57257.26135564704,46523.80416861758,44300.83791323246,54901.1701096736,47361.27151969814,49156.09266449866,56461.30009557872,56062.45724028393,58506.511677323855,48499.732173446595,45901.64074770707,33171.12855766741,38310.49272614435,39377.41586450999,28638.803870927226,29424.553978129992,39429.14347226504,39300.73151290347,27525.04968129815,47095.77682827757,30948.90232192865,32939.14262154816,30155.99148400138,35067.04977301419,35592.25066656211,45790.67404045973,42596.77718301701,39223.35618808953,25767.93476452563,30963.38609617001,30034.908201595936,23323.05819720473,28517.276888551194,41560.85308154171,43999.15897330077,21427.19650317167,25084.131072037453,18450.764512377853,38182.39266803458,20422.517191285966,14090.243196787245,21998.55223596247,30809.187151501046,22071.90474198888,31906.454754357772,20763.073093214778,25620.747974742615,19248.368008063957,23289.011422937554,26669.846131314298,27207.74766143675,29386.08741874402 +RS,12898.821957973714,11760.375115749226,5762.823681912337,10868.645457667955,6851.647476795101,6489.091511565737,8016.427124526303,11096.059611708006,8784.092785314235,6844.044739629848,5968.673897869134,7450.873145908507,8457.218084424714,10466.134979346216,18700.932768467817,18783.264457747777,9647.157790116991,14650.396388121844,12360.14810912279,11640.673528541236,8882.446980807292,14762.619600006037,18399.930168952076,13646.806400326906,14489.119148714197,14104.804002759012,15462.34970923992,10894.228431882033,15095.088711819322,17853.281396922546,12916.46431873845,14207.379133135786,14418.834370720482,12752.5957180177,15544.588217112372,15820.904121022999,13534.055257847373,15215.146639151637,12996.626380040323,13029.919161637963,10196.668003968522,10284.440766030819,7022.693160251435,8401.249663323453,8435.803252725058,10265.915943082338,8623.546341166757,7782.429841191159,9309.259412625106,4849.32865233776,9480.40891035497,8237.330612702019,6688.9341497267815,7113.796951355988,8945.392918392941,11391.406447517316,8770.78174795334,7797.074404297926,11205.567652241964,7490.638744688466,6890.626375097134,8721.152535625733,6986.899718851523,10014.515393386338,11863.685949406314,12576.12226693591,6930.903954576986,6933.68818306879,8436.351253425808,11680.836931112251,5673.257881932308,4512.930178739657,6094.8735580904795,11616.485592765608,8423.704814584986,9582.260510301834,6280.960312130538,10196.683253538626,5869.428152205924,6637.126459584064,8223.324863109141,9163.49600405833,11744.973584261521 +SK,10526.703178382486,6581.841291454129,2779.6437301689157,6043.293021386589,8068.711762338472,4716.572547513843,4263.788112757618,7436.446538154413,5733.7942336519345,5091.94458281141,6885.47211985414,6956.591212284293,7116.928783228905,4573.269245439668,8169.130903254093,6799.092645889461,7190.199390340817,8819.311158887818,6679.080394471699,8681.76846617681,6714.688996801623,8780.135433744465,9674.052829336924,7510.89080672348,12315.672504913706,11551.288938768861,10658.095118642217,7760.467099195778,7334.701885838525,11743.053621084162,9101.455526104724,11252.254293590808,7089.2297957961755,11692.575497480175,11870.620362608648,8320.411050776946,12664.612697391287,10840.053026355938,9544.145158945727,8285.851845920657,6554.955003499832,6014.385824757616,5119.229272436076,5566.197178944401,8283.751441606055,4576.739002618681,5529.727054286627,4884.285979430572,7030.767074231632,6380.185010419688,6092.06334940043,4789.6919845964085,3988.728835990378,6100.445127127096,6174.336930810738,5839.492530664778,6739.897695678873,6073.547495791424,6239.663814901599,5985.626769812626,5946.910319392802,5350.4706690989715,2897.742723724252,3812.447551889599,5110.734824948982,5466.7571969447445,3822.1224333717755,4189.745228198809,3608.9397299109723,8925.718578619328,5520.297117401385,2338.023634704523,3634.014208205775,5079.021357670912,3789.3939110245783,3599.858750509022,4222.707040620564,3096.1701009538024,2158.8092391216674,4542.203594136759,4220.210061470592,2961.7796555712353,3705.0055429796116 +SI,8932.252857660058,6769.672897729603,3971.443731895907,6727.0196008878875,5996.060296331706,4552.96378417224,7787.375795346372,9548.071392837826,4890.204133819321,7094.028640574649,11752.798116380785,9378.602450871665,7169.30972105447,8221.267552376507,10568.142173580174,7705.119699907013,8559.440940655051,9496.2682906945,10641.418485889739,13644.09405840664,8756.865265119286,10877.994032024846,13095.473005855994,10408.222250010325,17110.767148034232,13257.44359305617,10163.420442559087,10743.809603872312,11213.570708968457,12419.028940460184,8518.188189494229,14975.496301661744,10217.449212514946,10521.007791805137,10759.209828867228,9699.803414833696,14530.647692616923,14530.715084460895,12439.143395173185,11990.046930204517,9001.647305603035,10111.423546501868,6824.172499114327,10267.968760960974,10233.709492652128,9562.175707725224,11231.941186997657,8178.692540046128,8568.526412713521,8103.883097000008,9639.895318751553,9157.43571368136,6814.50158997294,8089.913064615838,8170.645053083449,10765.044719853075,6713.170108103339,8936.8362631332,9542.147586182518,8964.110093884974,8653.114721452737,8353.75428643612,4558.589088593759,9432.405903987477,6927.92152111369,7408.760392367348,5834.871486466167,9336.912851251047,8839.494837985663,10766.693939731143,5073.937522450624,6475.598951785893,9482.419717110955,13414.771212079502,5906.5584945059145,9942.151265372298,7696.092255607837,9521.835825354157,8476.124990692872,6361.010018809483,7504.908031281684,6349.444363482254,12494.86342976942 +SE,97729.33612275522,104893.13349064675,152164.30605691054,147567.81837152236,172633.569885874,142454.06079467706,109264.12507808255,145966.80578917114,158733.59492028368,161670.6171803483,134822.12256461562,136958.6213484757,175679.6273876132,133962.36617697845,126787.36233921513,130861.49290945902,151133.2982760823,140247.52694114187,130508.52271890236,137421.63579450184,164132.0529341604,142467.57286064525,135886.87305352974,163762.3962091317,161128.38161511483,148196.28633931337,167892.87677861055,132017.696130452,105953.99289036485,119407.22923207263,147139.6039546916,151309.55114963037,164893.61323375857,137608.18237925606,149284.78771947857,108749.15747050248,131859.531763871,117290.48017785426,131751.97960784068,111117.27409251402,110551.615114603,81550.1135393761,114677.940027603,102356.46697333359,128977.80283219383,94483.98267503879,114937.96457142044,93256.91327642264,96751.29026217168,91074.34139788701,87931.96567208324,96560.83822983531,116746.94389793588,64626.25905820224,75156.91767736348,51489.37512699482,74729.26978432719,134673.01513808029,110440.65771030058,149033.9827088587,143576.83069821307,93040.02354987273,64075.160200006496,102493.08906248632,106664.82884715112,90351.43771324471,94801.81222609026,92445.93458012305,88919.20770998915,108111.72872590217,127440.80298482065,124444.06618883039,98961.91573013023,87913.90602563386,115956.48217115598,95526.84265190935,103459.08985036932,86058.90377748368,99040.9103276256,110493.22773914765,94182.7249029169,169752.66741900297,99896.09236620825 From cbeb9834ac441ae8d54ba6e62f3f12804af7b031 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 14:08:58 +0100 Subject: [PATCH 117/173] update hydro capacity and generation data --- data/eia_hydro_annual_capacity.csv | 103 +++++++++++++------------- data/eia_hydro_annual_generation.csv | 106 +++++++++++++-------------- rules/build_electricity.smk | 1 + scripts/build_hydro_profile.py | 11 ++- 4 files changed, 114 insertions(+), 107 deletions(-) diff --git a/data/eia_hydro_annual_capacity.csv b/data/eia_hydro_annual_capacity.csv index e14697bc..8092096d 100644 --- a/data/eia_hydro_annual_capacity.csv +++ b/data/eia_hydro_annual_capacity.csv @@ -1,50 +1,53 @@ -# https://www.eia.gov/international/data/world/electricity/electricity-generation?pd=2&p=00000000000000000000008&u=1&f=A&v=mapbubble&a=-&i=none&vo=value&t=R&g=000000000000002&l=73-1028i008017kg6368g80a4k000e0ag00gg0004g8g0ho00g000400008&s=315532800000&e=1577836800000&ev=false& -Report generated on: 07-21-2022 16:15:18 -"API","","1980","1981","1982","1983","1984","1985","1986","1987","1988","1989","1990","1991","1992","1993","1994","1995","1996","1997","1998","1999","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2011","2012","2013","2014","2015","2016","2017","2018","2019","2020" -"","hydroelectricity installed capacity (million kW)","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","" -"INTL.33-7-EURO-MK.A"," Europe","136.143","137.425","141.734","143.763","142.894","144.738","147.631","150.428","153.428","153.345","139.346","134.524","137.463","138.338","139.688","141.47","142.121","143.595","143.957","146.4204","147.3512","147.736","152.173","152.938","150.4894","151.424","152.276","154.198","155.405","156.988","159.495","162.112","165.452","170.146","171.588","174.906","176.998","178.221","180.846","181.68","184.909" -"INTL.33-7-ALB-MK.A"," Albania","0.5","0.525","0.55","0.6","0.625","0.65","0.675","0.68","0.69","0.69","1.668","1.668","1.668","1.668","1.668","1.445","1.445","1.445","1.445","1.445","1.445","1.445","1.445","1.445","1.445","1.432","1.432","1.432","1.45","1.45","1.461","1.508","1.628","1.781","1.725","1.798","1.913","2.047","2.105","2.193","2.39" -"INTL.33-7-AUT-MK.A"," Austria","8.206","9.157","9.51","9.582","10.034","10.171","10.423","10.691","10.762","10.858","7.028","7.129","7.204","7.202","7.245","7.323","7.385","7.54","7.685","7.669","7.676","7.703","7.567","7.607","7.613","7.667","7.684","7.845","7.848","7.827","7.913","7.947","7.97","8.272","8.321","8.457","8.493","8.506","8.591","8.63","9.001" -"INTL.33-7-BEL-MK.A"," Belgium","0.073","0.08","0.086","0.086","0.086","0.087","0.089","0.09","0.093","0.095","0.094","0.094","0.094","0.095","0.095","0.096","0.096","0.096","0.097","0.103","0.103","0.111","0.111","0.11","0.115","0.105","0.107","0.11","0.111","0.11","0.118","0.119","0.12","0.119","0.121","0.112","0.115","0.107","0.108","0.108","0.12" -"INTL.33-7-BIH-MK.A"," Bosnia and Herzegovina","--","--","--","--","--","--","--","--","--","--","--","--","1.2","1.2","1.139","1.219","1.219","1.219","1.624","1.983","1.983","1.993","2.38","2.38","2.38","2.38","2.411","2.411","2.117","2.117","2.117","2.117","2.12","2.12","2.049","2.055","2.084","2.084","2.09","2.09","2.093" -"INTL.33-7-BGR-MK.A"," Bulgaria","1.895","1.895","1.895","1.975","1.975","1.975","1.975","1.975","1.975","1.973","1.973","1.401","1.401","1.401","1.401","1.401","1.401","1.803","1.803","1.803","1.881","1.706","1.948","1.984","1.984","1.984","1.984","2.012","2.12","2.137","2.184","2.035","2.095","2.165","2.19","2.206","2.206","2.21","2.359","2.366","1.725" -"INTL.33-7-HRV-MK.A"," Croatia","--","--","--","--","--","--","--","--","--","--","--","--","1.769","1.77","1.77","1.781","1.785","1.785","1.785","1.785","1.785","1.785","1.775","1.783","1.79","1.804","1.804","1.782","1.782","1.799","1.848","1.848","1.848","1.897","1.9","1.915","1.912","1.912","1.913","1.913","1.848" -"INTL.33-7-CYP-MK.A"," Cyprus","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" -"INTL.33-7-CZE-MK.A"," Czech Republic","--","--","--","--","--","--","--","--","--","--","--","--","--","0.911","0.906","0.908","0.868","0.905","0.888","1.008","0.952","1","1","1.004","1.015","1.02","1.016","1.024","1.029","1.037","1.049","1.05","1.065","1.08","1.08","1.088","1.09","1.093","1.094","1.096","1.097" -"INTL.33-7-DNK-MK.A"," Denmark","0.008","0.008","0.01","0.009","0.009","0.009","0.009","0.009","0.009","0.011","0.01","0.01","0.01","0.01","0.008","0.01","0.01","0.01","0.011","0.011","0.01","0.011","0.011","0.011","0.011","0.011","0.009","0.009","0.009","0.009","0.009","0.009","0.009","0.009","0.009","0.007","0.009","0.009","0.009","0.009","0.009" -"INTL.33-7-EST-MK.A"," Estonia","--","--","--","--","--","--","--","--","--","--","--","--","0.001","0.001","0.001","0.001","0.001","0.001","0.001","0.0012","0.0012","0.003","0.003","0.004","0.004","0.005","0.005","0.005","0.005","0.007","0.006","0.005","0.008","0.008","0.005","0.006","0.006","0.007","0.007","0.007","0.008" -"INTL.33-7-FRO-MK.A"," Faroe Islands","0.018","0.018","0.018","0.018","0.018","0.018","0.018","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.0314","0.032","0.032","0.032","0.032","0.0314","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.039","0.039","0.04","0.04","0.04","0.04","0.04","0.04","0.039" -"INTL.33-7-FIN-MK.A"," Finland","2.42","2.467","2.474","2.503","2.497","2.505","2.555","2.586","2.597","2.586","2.621","2.648","2.679","2.731","2.736","2.777","2.785","2.861","2.881","2.881","2.882","2.926","2.964","2.966","2.999","3.035","3.062","3.102","3.122","3.145","3.155","3.196","3.196","3.224","3.248","3.249","3.249","3.272","3.287","3.287","3.263" -"INTL.33-7-CSK-MK.A"," Former Czechoslovakia","2.578","2.832","2.84","2.84","2.875","2.897","2.89","2.975","2.988","3.042","3.036","3.061","3.061","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" -"INTL.33-7-SCG-MK.A"," Former Serbia and Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","2.25","2.25","2.25","2.25","2.25","2.25","2.25","2.296","2.296","2.296","2.296","2.296","2.206","2.206","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" -"INTL.33-7-YUG-MK.A"," Former Yugoslavia","6.2","6.25","5.886","5.886","6.386","6.736","7.086","7.386","7.625","7.686","7.386","7.386","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" -"INTL.33-7-FRA-MK.A"," France","17.431","17.63","18.247","18.4","18.661","19.034","19.786","19.991","20.174","20.338","17.717","17.655","17.767","17.837","17.902","17.898","18","18.018","18.024","17.947","17.646","17.674","17.775","17.927","17.812","17.808","17.82","17.832","17.922","18.009","18.156","18.373","18.388","18.379","18.392","18.415","18.486","18.561","18.857","18.88","19.671" -"INTL.33-7-DEU-MK.A"," Germany","--","--","--","--","--","--","--","--","--","--","--","3.31","3.317","3.385","3.471","3.624","3.563","3.569","3.642","3.802","4.086","4.101","4.193","4.088","4.209","4.134","4.117","4.083","4.104","4.283","4.252","4.469","4.451","4.433","4.424","4.433","4.442","4.449","4.456","4.456","4.658" -"INTL.33-7-DDR-MK.A"," Germany, East","1.852","1.845","1.852","1.851","1.845","1.844","1.844","1.844","1.844","1.844","1.844","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" -"INTL.33-7-DEUW-MK.A"," Germany, West","6.45","6.509","6.531","6.631","6.668","6.71","6.71","6.71","6.85","6.86","6.86","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" -"INTL.33-7-GIB-MK.A"," Gibraltar","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" -"INTL.33-7-GRC-MK.A"," Greece","1.415","1.714","1.714","1.714","1.714","1.822","1.822","1.822","1.836","1.986","2.093","2.197","2.208","2.208","2.208","2.208","2.207","2.412","2.241","2.344","2.373","2.377","2.379","2.38","2.4","2.407","2.435","2.451","2.477","2.502","2.516","2.525","2.537","2.539","2.69","2.693","2.693","2.693","2.71","2.71","2.697" -"INTL.33-7-HUN-MK.A"," Hungary","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.054","0.049","0.049","0.049","0.049","0.051","0.053","0.053","0.055","0.056","0.057","0.057","0.057","0.057","0.057","0.057","0.057","0.056" -"INTL.33-7-ISL-MK.A"," Iceland","0.545","0.615","0.755","0.755","0.755","0.755","0.756","0.756","0.756","0.756","0.756","0.779","0.879","0.879","0.884","0.884","0.884","0.923","0.956","1.016","1.064","1.109","1.155","1.155","1.163","1.163","1.163","1.758","1.879","1.875","1.883","1.884","1.877","1.984","1.984","1.987","1.987","1.995","2.099","2.099","2.086" -"INTL.33-7-IRL-MK.A"," Ireland","0.224","0.224","0.225","0.225","0.226","0.226","0.221","0.222","0.222","0.222","0.223","0.226","0.226","0.226","0.227","0.227","0.232","0.233","0.233","0.236","0.236","0.238","0.24","0.24","0.24","0.234","0.234","0.234","0.234","0.234","0.237","0.237","0.237","0.237","0.237","0.237","0.237","0.237","0.237","0.237","0.237" -"INTL.33-7-ITA-MK.A"," Italy","15.826","15.766","16.877","17.125","12.166","12.16","12.419","12.435","12.495","12.547","12.582","12.692","12.718","12.788","12.864","12.964","12.999","13.06","13.058","13.417","13.389","13.456","13.557","13.703","13.789","13.89","13.528","13.573","13.732","13.827","13.976","14.193","14.325","14.454","14.506","14.628","14.991","15.109","15.182","15.583","14.908" -"INTL.33-7-XKS-MK.A"," Kosovo","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","0.043","0.043","0.045","0.046","0.046","0.046","0.046","0.046","0.078","0.08","0.096","0.096","0.092" -"INTL.33-7-LVA-MK.A"," Latvia","--","--","--","--","--","--","--","--","--","--","--","--","1.499","1.504","1.506","1.521","1.521","1.487","1.517","1.523","1.523","1.565","1.565","1.537","1.536","1.536","1.536","1.536","1.536","1.536","1.576","1.576","1.576","1.587","1.588","1.588","1.564","1.564","1.565","1.565","1.576" -"INTL.33-7-LTU-MK.A"," Lithuania","--","--","--","--","--","--","--","--","--","--","--","--","0.106","0.106","0.108","0.108","0.108","0.108","0.108","0.112","0.112","0.113","0.103","0.109","0.11","0.117","0.117","0.115","0.115","0.116","0.116","0.116","0.116","0.116","0.117","0.117","0.117","0.117","0.117","0.117","0.116" -"INTL.33-7-LUX-MK.A"," Luxembourg","0.029","0.029","0.029","0.032","0.032","0.032","0.032","0.032","0.032","0.032","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.039","0.039","0.039","0.039","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.035","0.034","0.034","0.034" -"INTL.33-7-MLT-MK.A"," Malta","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" -"INTL.33-7-MNE-MK.A"," Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","0.658","0.658","0.658","0.658","0.658","0.658","0.658","0.651","0.651","0.651","0.651","0.652","0.652","0.652","0.658" -"INTL.33-7-NLD-MK.A"," Netherlands","0","0","0","0","0","0.002","0.002","0.002","0.002","0.025","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.038" -"INTL.33-7-MKD-MK.A"," North Macedonia","--","--","--","--","--","--","--","--","--","--","--","--","0.426","0.426","0.413","0.423","0.423","0.434","0.434","0.4338","0.434","0.436","0.436","0.448","0.516","0.54","0.545","0.546","0.552","0.553","0.555","0.556","0.595","0.603","0.617","0.632","0.66","0.674","0.674","0.674","0.674" -"INTL.33-7-NOR-MK.A"," Norway","19.443","20.963","21.789","22.37","22.628","23.076","23.555","24.358","24.453","25.728","25.817","25.823","25.963","25.983","26.215","27.379","27.494","27.327","26.982","27.54","26.766","26.319","26.604","26.947","26.721","27.222","27.398","27.647","28.062","28.188","28.367","28.618","29.158","29.682","29.889","29.939","30.281","30.382","31.12","31.182","31.556" -"INTL.33-7-POL-MK.A"," Poland","0.647","0.647","0.647","0.647","0.645","0.646","0.646","0.646","0.772","0.647","0.467","0.467","0.468","0.475","0.489","0.482","0.492","0.495","0.501","0.505","0.509","0.517","0.517","0.524","0.535","0.542","0.549","0.546","0.553","0.556","0.56","0.564","0.569","0.573","0.582","0.588","0.597","0.591","0.592","0.592","0.605" -"INTL.33-7-PRT-MK.A"," Portugal","2.516","2.615","2.854","2.944","3.016","2.721","2.818","2.82","2.722","2.799","2.783","2.772","3.146","3.613","3.697","3.848","3.867","3.877","3.94","3.93","3.918","3.943","3.966","3.966","3.974","3.968","4.004","4.012","4.009","4.042","4.057","4.49","4.414","4.363","4.368","4.446","4.458","4.462","4.484","4.484","4.373" -"INTL.33-7-ROU-MK.A"," Romania","3.455","3.533","3.734","3.885","4.062","4.42","4.706","5.057","5.421","5.583","5.666","5.723","5.687","5.872","5.938","6.011","6.038","6.074","6.081","6.082","6.12","6.122","6.242","6.248","6.279","6.289","6.282","6.331","6.362","6.358","6.382","6.391","6.456","6.249","6.256","6.359","6.377","6.328","6.328","6.328","6.221" -"INTL.33-7-SRB-MK.A"," Serbia","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","2.206","2.206","2.206","2.221","2.293","2.322","2.352","2.317","2.403","2.408","2.416","2.424","2.424","2.424","2.484" -"INTL.33-7-SVK-MK.A"," Slovakia","--","--","--","--","--","--","--","--","--","--","--","--","--","0.925","1.208","1.525","1.655","1.664","1.682","1.684","1.685","1.586","1.586","1.588","1.602","1.596","1.597","1.599","1.632","1.571","1.6","1.607","1.606","1.607","1.607","1.606","1.608","1.607","1.612","1.612","1.505" -"INTL.33-7-SVN-MK.A"," Slovenia","--","--","--","--","--","--","--","--","--","--","--","--","0.755","0.755","0.756","0.757","0.734","0.734","0.861","0.846","0.846","0.839","0.983","0.974","0.974","0.979","1.009","1.018","1.027","1.07","1.074","1.073","1.074","1.119","1.116","1.115","1.113","1.167","1.163","1.163","1.344" -"INTL.33-7-ESP-MK.A"," Spain","13.473","10.869","10.945","10.917","10.935","10.959","11.153","10.556","10.984","11.597","11.32","11.429","11.484","11.484","11.545","11.689","11.793","11.596","11.537","11.802","12.672","12.744","15.55","15.525","12.82","12.808","12.907","12.961","13.04","13.069","13.275","13.283","13.293","14.076","14.081","14.086","14.053","14.052","14.053","14.053","14.292" -"INTL.33-7-SWE-MK.A"," Sweden","14.859","14.919","15.215","15.29","15.445","15.69","15.813","15.996","16.112","15.759","15.904","15.891","16.021","15.867","16.072","15.725","15.776","16.371","16.169","16.432","16.506","16.523","16.187","16.098","16.302","16.302","16.234","16.592","16.352","16.544","16.624","16.478","16.315","16.395","15.897","16.23","16.367","16.403","16.332","16.332","16.379" -"INTL.33-7-CHE-MK.A"," Switzerland","11.45","11.46","11.47","11.47","11.48","11.48","11.51","11.51","11.52","11.58","3.474","3.484","3.504","3.509","3.526","3.541","3.55","3.553","3.584","3.614","3.636","3.642","3.653","3.669","3.65","3.682","3.694","3.7","3.709","3.749","3.81","3.852","3.882","3.896","3.948","3.996","4.06","4.112","4.193","4.193","4.193" -"INTL.33-7-TUR-MK.A"," Turkey","2.131","2.356","3.082","3.239","3.875","3.875","3.878","5.003","6.219","6.598","6.764","7.114","8.379","9.682","9.865","9.863","9.935","10.102","10.307","10.537","11.175","11.673","12.241","12.579","12.645","12.906","13.063","13.395","13.829","14.553","15.831","17.137","19.609","22.289","23.643","25.868","26.681","27.273","28.291","28.503","30.984" -"INTL.33-7-GBR-MK.A"," United Kingdom","2.451","2.451","2.451","2.721","4.188","4.19","4.192","4.197","4.196","1.424","1.11","1.415","1.423","1.425","1.425","1.432","1.455","1.488","1.475","1.477","1.485","1.629","1.59","1.486","1.499","1.501","1.515","1.522","1.626","1.638","1.637","1.673","1.693","1.709","1.73","1.777","1.836","1.873","1.878","1.878","1.879" +# https://www.eia.gov/international/data/world/electricity/electricity-generation?pd=2&p=00000000000000000000008&u=1&f=A&v=mapbubble&a=-&i=none&vo=value&t=R&g=000000000000002&l=73-1028i008017kg6368g80a4k000e0ag00gg0004g8g0ho00g000400008&l=72-00000000000000000000000000080000000000000000000g&s=315532800000&e=1609459200000&ev=false& +Report generated on: 03-14-2024 13:39:49 +"API","","1980","1981","1982","1983","1984","1985","1986","1987","1988","1989","1990","1991","1992","1993","1994","1995","1996","1997","1998","1999","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2011","2012","2013","2014","2015","2016","2017","2018","2019","2020","2021" +"","hydroelectricity installed capacity (million kW)","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","" +"INTL.33-7-EURO-MK.A"," Europe","136.143","137.425","141.734","143.763","142.894","144.738","147.631","150.428","153.428","153.345","139.346","134.524","137.463","138.338","139.688","141.47","142.121","143.595","143.957","146.4204","147.3512","147.736","152.173","152.938","150.4894","151.424","152.276","154.198","155.405","156.988","159.495","162.112","165.452","170.146","171.588","174.906","176.998","178.221","180.212","181.039","184.728","185.46" +"INTL.33-7-ALB-MK.A"," Albania","0.5","0.525","0.55","0.6","0.625","0.65","0.675","0.68","0.69","0.69","1.668","1.668","1.668","1.668","1.668","1.445","1.445","1.445","1.445","1.445","1.445","1.445","1.445","1.445","1.445","1.432","1.432","1.432","1.45","1.45","1.461","1.508","1.628","1.781","1.725","1.798","1.913","2.047","2.105","2.193","2.39","2.39" +"INTL.33-7-AUT-MK.A"," Austria","8.206","9.157","9.51","9.582","10.034","10.171","10.423","10.691","10.762","10.858","7.028","7.129","7.204","7.202","7.245","7.323","7.385","7.54","7.685","7.669","7.676","7.703","7.567","7.607","7.613","7.667","7.684","7.845","7.848","7.827","7.913","7.947","7.97","8.272","8.321","8.457","8.493","8.506","8.591","8.63","9.001","9.151" +"INTL.33-7-BEL-MK.A"," Belgium","0.073","0.08","0.086","0.086","0.086","0.087","0.089","0.09","0.093","0.095","0.094","0.094","0.094","0.095","0.095","0.096","0.096","0.096","0.097","0.103","0.103","0.111","0.111","0.11","0.115","0.105","0.107","0.11","0.111","0.11","0.118","0.119","0.12","0.119","0.121","0.112","0.115","0.107","0.108","0.108","0.12","0.12" +"INTL.33-7-BIH-MK.A"," Bosnia and Herzegovina","--","--","--","--","--","--","--","--","--","--","--","--","1.2","1.2","1.139","1.219","1.219","1.219","1.624","1.983","1.983","1.993","2.38","2.38","2.38","2.38","2.411","2.411","2.117","2.117","2.117","2.117","2.12","2.12","2.049","2.055","2.084","2.084","2.09","2.09","2.093","1.747" +"INTL.33-7-BGR-MK.A"," Bulgaria","1.895","1.895","1.895","1.975","1.975","1.975","1.975","1.975","1.975","1.973","1.973","1.401","1.401","1.401","1.401","1.401","1.401","1.803","1.803","1.803","1.881","1.706","1.948","1.984","1.984","1.984","1.984","2.012","2.12","2.137","2.184","2.035","2.095","2.165","2.19","2.206","2.206","2.21","1.725","1.725","1.725","1.725" +"INTL.33-7-HRV-MK.A"," Croatia","--","--","--","--","--","--","--","--","--","--","--","--","1.769","1.77","1.77","1.781","1.785","1.785","1.785","1.785","1.785","1.785","1.775","1.783","1.79","1.804","1.804","1.782","1.782","1.799","1.848","1.848","1.848","1.897","1.9","1.915","1.912","1.912","1.913","1.913","1.848","1.874" +"INTL.33-7-CYP-MK.A"," Cyprus","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-7-CZE-MK.A"," Czechia","--","--","--","--","--","--","--","--","--","--","--","--","--","0.911","0.906","0.908","0.868","0.905","0.888","1.008","0.952","1","1","1.004","1.015","1.02","1.016","1.024","1.029","1.037","1.049","1.05","1.065","1.08","1.08","1.088","1.09","1.093","1.094","1.096","1.097","1.109" +"INTL.33-7-DNK-MK.A"," Denmark","0.008","0.008","0.01","0.009","0.009","0.009","0.009","0.009","0.009","0.011","0.01","0.01","0.01","0.01","0.008","0.01","0.01","0.01","0.011","0.011","0.01","0.011","0.011","0.011","0.011","0.011","0.009","0.009","0.009","0.009","0.009","0.009","0.009","0.009","0.009","0.007","0.009","0.009","0.009","0.009","0.009","0.007" +"INTL.33-7-EST-MK.A"," Estonia","--","--","--","--","--","--","--","--","--","--","--","--","0.001","0.001","0.001","0.001","0.001","0.001","0.001","0.0012","0.0012","0.003","0.003","0.004","0.004","0.005","0.005","0.005","0.005","0.007","0.006","0.005","0.008","0.008","0.005","0.006","0.006","0.007","0.007","0.007","0.008","0.004" +"INTL.33-7-FRO-MK.A"," Faroe Islands","0.018","0.018","0.018","0.018","0.018","0.018","0.018","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.0314","0.032","0.032","0.032","0.032","0.0314","0.031","0.031","0.031","0.031","0.031","0.031","0.031","0.039","0.039","0.04","0.04","0.04","0.04","0.04","0.04","0.039","0.039" +"INTL.33-7-FIN-MK.A"," Finland","2.42","2.467","2.474","2.503","2.497","2.505","2.555","2.586","2.597","2.586","2.621","2.648","2.679","2.731","2.736","2.777","2.785","2.861","2.881","2.881","2.882","2.926","2.964","2.966","2.999","3.035","3.062","3.102","3.122","3.145","3.155","3.196","3.196","3.224","3.248","3.249","3.249","3.272","3.287","3.287","3.263","3.263" +"INTL.33-7-CSK-MK.A"," Former Czechoslovakia","2.578","2.832","2.84","2.84","2.875","2.897","2.89","2.975","2.988","3.042","3.036","3.061","3.061","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-7-SCG-MK.A"," Former Serbia and Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","2.25","2.25","2.25","2.25","2.25","2.25","2.25","2.296","2.296","2.296","2.296","2.296","2.206","2.206","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-7-YUG-MK.A"," Former Yugoslavia","6.2","6.25","5.886","5.886","6.386","6.736","7.086","7.386","7.625","7.686","7.386","7.386","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-7-FRA-MK.A"," France","17.431","17.63","18.247","18.4","18.661","19.034","19.786","19.991","20.174","20.338","17.717","17.655","17.767","17.837","17.902","17.898","18","18.018","18.024","17.947","17.646","17.674","17.775","17.927","17.812","17.808","17.82","17.832","17.922","18.009","18.156","18.373","18.388","18.379","18.392","18.415","18.486","18.561","18.857","18.88","19.671","19.657" +"INTL.33-7-DEU-MK.A"," Germany","--","--","--","--","--","--","--","--","--","--","--","3.31","3.317","3.385","3.471","3.624","3.563","3.569","3.642","3.802","4.086","4.101","4.193","4.088","4.209","4.134","4.117","4.083","4.104","4.283","4.252","4.469","4.451","4.433","4.424","4.433","4.442","4.449","4.456","4.456","4.658","4.684" +"INTL.33-7-DDR-MK.A"," Germany, East","1.852","1.845","1.852","1.851","1.845","1.844","1.844","1.844","1.844","1.844","1.844","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-7-DEUW-MK.A"," Germany, West","6.45","6.509","6.531","6.631","6.668","6.71","6.71","6.71","6.85","6.86","6.86","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-7-GIB-MK.A"," Gibraltar","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-7-GRC-MK.A"," Greece","1.415","1.714","1.714","1.714","1.714","1.822","1.822","1.822","1.836","1.986","2.093","2.197","2.208","2.208","2.208","2.208","2.207","2.412","2.241","2.344","2.373","2.377","2.379","2.38","2.4","2.407","2.435","2.451","2.477","2.502","2.516","2.525","2.537","2.539","2.69","2.693","2.693","2.693","2.71","2.71","2.697","2.722" +"INTL.33-7-HUN-MK.A"," Hungary","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.048","0.054","0.049","0.049","0.049","0.049","0.051","0.053","0.053","0.055","0.056","0.057","0.057","0.057","0.057","0.057","0.057","0.057","0.056","0.058" +"INTL.33-7-ISL-MK.A"," Iceland","0.545","0.615","0.755","0.755","0.755","0.755","0.756","0.756","0.756","0.756","0.756","0.779","0.879","0.879","0.884","0.884","0.884","0.923","0.956","1.016","1.064","1.109","1.155","1.155","1.163","1.163","1.163","1.758","1.879","1.875","1.883","1.884","1.877","1.984","1.984","1.987","1.987","1.995","2.099","2.099","2.086","2.086" +"INTL.33-7-IRL-MK.A"," Ireland","0.224","0.224","0.225","0.225","0.226","0.226","0.221","0.222","0.222","0.222","0.223","0.226","0.226","0.226","0.227","0.227","0.232","0.233","0.233","0.236","0.236","0.238","0.24","0.24","0.24","0.234","0.234","0.234","0.234","0.234","0.237","0.237","0.237","0.237","0.237","0.237","0.237","0.237","0.237","0.237","0.237","0.216" +"INTL.33-7-ITA-MK.A"," Italy","15.826","15.766","16.877","17.125","12.166","12.16","12.419","12.435","12.495","12.547","12.582","12.692","12.718","12.788","12.864","12.964","12.999","13.06","13.058","13.417","13.389","13.456","13.557","13.703","13.789","13.89","13.528","13.573","13.732","13.827","13.976","14.193","14.325","14.454","14.506","14.628","14.991","15.109","15.182","15.583","14.908","14.908" +"INTL.33-7-XKS-MK.A"," Kosovo","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","0.043","0.043","0.045","0.046","0.046","0.046","0.046","0.046","0.078","0.08","0.096","0.096","0.092","0.092" +"INTL.33-7-LVA-MK.A"," Latvia","--","--","--","--","--","--","--","--","--","--","--","--","1.499","1.504","1.506","1.521","1.521","1.487","1.517","1.523","1.523","1.565","1.565","1.537","1.536","1.536","1.536","1.536","1.536","1.536","1.576","1.576","1.576","1.587","1.588","1.588","1.564","1.564","1.565","1.565","1.576","1.588" +"INTL.33-7-LTU-MK.A"," Lithuania","--","--","--","--","--","--","--","--","--","--","--","--","0.106","0.106","0.108","0.108","0.108","0.108","0.108","0.112","0.112","0.113","0.103","0.109","0.11","0.117","0.117","0.115","0.115","0.116","0.116","0.116","0.116","0.116","0.117","0.117","0.117","0.117","0.117","0.117","0.116","0.128" +"INTL.33-7-LUX-MK.A"," Luxembourg","0.029","0.029","0.029","0.032","0.032","0.032","0.032","0.032","0.032","0.032","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.039","0.039","0.039","0.039","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.034","0.035","0.034","0.034","0.034","0.037" +"INTL.33-7-MLT-MK.A"," Malta","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-7-MNE-MK.A"," Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","0.658","0.658","0.658","0.658","0.658","0.658","0.658","0.651","0.651","0.651","0.651","0.652","0.652","0.652","0.658","0.649" +"INTL.33-7-NLD-MK.A"," Netherlands","0","0","0","0","0","0.002","0.002","0.002","0.002","0.025","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.037","0.038","0.038" +"INTL.33-7-MKD-MK.A"," North Macedonia","--","--","--","--","--","--","--","--","--","--","--","--","0.426","0.426","0.413","0.423","0.423","0.434","0.434","0.4338","0.434","0.436","0.436","0.448","0.516","0.54","0.545","0.546","0.552","0.553","0.555","0.556","0.595","0.603","0.617","0.632","0.66","0.674","0.674","0.674","0.674","0.644" +"INTL.33-7-NOR-MK.A"," Norway","19.443","20.963","21.789","22.37","22.628","23.076","23.555","24.358","24.453","25.728","25.817","25.823","25.963","25.983","26.215","27.379","27.494","27.327","26.982","27.54","26.766","26.319","26.604","26.947","26.721","27.222","27.398","27.647","28.062","28.188","28.367","28.618","29.158","29.682","29.889","29.939","30.281","30.382","31.12","31.182","31.556","31.952" +"INTL.33-7-POL-MK.A"," Poland","0.647","0.647","0.647","0.647","0.645","0.646","0.646","0.646","0.772","0.647","0.467","0.467","0.468","0.475","0.489","0.482","0.492","0.495","0.501","0.505","0.509","0.517","0.517","0.524","0.535","0.542","0.549","0.546","0.553","0.556","0.56","0.564","0.569","0.573","0.582","0.588","0.597","0.591","0.592","0.592","0.605","0.605" +"INTL.33-7-PRT-MK.A"," Portugal","2.516","2.615","2.854","2.944","3.016","2.721","2.818","2.82","2.722","2.799","2.783","2.772","3.146","3.613","3.697","3.848","3.867","3.877","3.94","3.93","3.918","3.943","3.966","3.966","3.974","3.968","4.004","4.012","4.009","4.042","4.057","4.49","4.414","4.363","4.368","4.446","4.458","4.462","4.484","4.484","4.373","4.372" +"INTL.33-7-ROU-MK.A"," Romania","3.455","3.533","3.734","3.885","4.062","4.42","4.706","5.057","5.421","5.583","5.666","5.723","5.687","5.872","5.938","6.011","6.038","6.074","6.081","6.082","6.12","6.122","6.242","6.248","6.279","6.289","6.282","6.331","6.362","6.358","6.382","6.391","6.456","6.249","6.256","6.359","6.377","6.328","6.328","6.328","6.221","6.221" +"INTL.33-7-SRB-MK.A"," Serbia","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","2.206","2.206","2.206","2.221","2.293","2.322","2.352","2.317","2.403","2.408","2.416","2.424","2.424","2.424","2.484","2.491" +"INTL.33-7-SVK-MK.A"," Slovakia","--","--","--","--","--","--","--","--","--","--","--","--","--","0.925","1.208","1.525","1.655","1.664","1.682","1.684","1.685","1.586","1.586","1.588","1.602","1.596","1.597","1.599","1.632","1.571","1.6","1.607","1.606","1.607","1.607","1.606","1.608","1.607","1.612","1.612","1.505","1.505" +"INTL.33-7-SVN-MK.A"," Slovenia","--","--","--","--","--","--","--","--","--","--","--","--","0.755","0.755","0.756","0.757","0.734","0.734","0.861","0.846","0.846","0.839","0.983","0.974","0.974","0.979","1.009","1.018","1.027","1.07","1.074","1.073","1.074","1.119","1.116","1.115","1.113","1.167","1.163","1.163","1.163","1.121" +"INTL.33-7-ESP-MK.A"," Spain","13.473","10.869","10.945","10.917","10.935","10.959","11.153","10.556","10.984","11.597","11.32","11.429","11.484","11.484","11.545","11.689","11.793","11.596","11.537","11.802","12.672","12.744","15.55","15.525","12.82","12.808","12.907","12.961","13.04","13.069","13.275","13.283","13.293","14.076","14.081","14.086","14.053","14.052","14.053","14.053","14.292","14.308" +"INTL.33-7-SWE-MK.A"," Sweden","14.859","14.919","15.215","15.29","15.445","15.69","15.813","15.996","16.112","15.759","15.904","15.891","16.021","15.867","16.072","15.725","15.776","16.371","16.169","16.432","16.506","16.523","16.187","16.098","16.302","16.302","16.234","16.592","16.352","16.544","16.624","16.478","16.315","16.395","15.897","16.23","16.367","16.403","16.332","16.332","16.379","16.379" +"INTL.33-7-CHE-MK.A"," Switzerland","11.45","11.46","11.47","11.47","11.48","11.48","11.51","11.51","11.52","11.58","3.474","3.484","3.504","3.509","3.526","3.541","3.55","3.553","3.584","3.614","3.636","3.642","3.653","3.669","3.65","3.682","3.694","3.7","3.709","3.749","3.81","3.852","3.882","3.896","3.948","3.996","4.06","4.112","4.193","4.193","4.193","4.193" +"INTL.33-7-TUR-MK.A"," Turkiye","2.131","2.356","3.082","3.239","3.875","3.875","3.878","5.003","6.219","6.598","6.764","7.114","8.379","9.682","9.865","9.863","9.935","10.102","10.307","10.537","11.175","11.673","12.241","12.579","12.645","12.906","13.063","13.395","13.829","14.553","15.831","17.137","19.609","22.289","23.643","25.868","26.681","27.273","28.291","28.503","30.984","31.497" +"INTL.33-7-GBR-MK.A"," United Kingdom","2.451","2.451","2.451","2.721","4.188","4.19","4.192","4.197","4.196","1.424","1.11","1.415","1.423","1.425","1.425","1.432","1.455","1.488","1.475","1.477","1.485","1.629","1.59","1.486","1.499","1.501","1.515","1.522","1.626","1.638","1.637","1.673","1.693","1.709","1.73","1.777","1.836","1.873","1.878","1.878","1.879","1.88" +""," Eurasia","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","" +"INTL.33-7-MDA-MK.A"," Moldova","--","--","--","--","--","--","--","--","--","--","--","--","0.064","0.064","0.064","0.056","0.056","0.064","0.064","0.064","0.064","0.06","0.06","0.06","0.059","0.059","0.056","0.056","0.064","0.064","0.064","0.064","0.064","0.064","0.064","0.064","0.064","0.064","0.064","0.064","0.076","0.076" +"INTL.33-7-UKR-MK.A"," Ukraine","--","--","--","--","--","--","--","--","--","--","--","--","4.705","4.706","4.706","4.706","4.706","4.706","4.706","4.7","4.7","4.731","4.758","4.766","4.781","4.717","4.746","4.731","4.798","4.795","4.596","4.607","4.608","4.632","4.665","4.697","4.658","4.668","4.668","4.668","4.666","4.43" \ No newline at end of file diff --git a/data/eia_hydro_annual_generation.csv b/data/eia_hydro_annual_generation.csv index 859decf7..d4e64a31 100644 --- a/data/eia_hydro_annual_generation.csv +++ b/data/eia_hydro_annual_generation.csv @@ -1,53 +1,53 @@ -https://www.eia.gov/international/data/world/electricity/electricity-generation?pd=2&p=000000000000000000000000000000g&u=1&f=A&v=mapbubble&a=-&i=none&vo=value&t=R&g=000000000000002&l=73-1028i008017kg6368g80a4k000e0ag00gg0004g8g0ho00g000400008&l=72-00000000000000000000000000080000000000000000000g&s=315532800000&e=1609459200000&ev=false&,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -Report generated on: 01-06-2023 21:17:46,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -API,,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021 -,hydroelectricity net generation (billion kWh),,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -INTL.33-12-EURO-BKWH.A, Europe,"458,018","464,155","459,881","473,685","481,241","476,739","459,535","491,085","534,517","465,365","474,466","475,47","509,041","526,448","531,815","543,743","529,114164","543,845616","562,491501","566,861453","588,644662","584,806195","539,051405","503,7067","542,112443","542,974669","535,006084","538,449707","565,143111","561,761402","617,547148","540,926277","598,055253","629,44709","617,111295","613,079848","627,720566217","560,362524","616,5081462","606,5997419","644,1106599","628,1390143" -INTL.33-12-ALB-BKWH.A, Albania,"2,919","3,018","3,093","3,167","3,241","3,315","3,365","3,979","3,713","3,846","2,82","3,483","3,187","3,281","3,733","4,162","5,669","4,978","4,872","5,231","4,548","3,519","3,477","5,117","5,411","5,319","4,951","2,76","3,759","5,201","7,49133","4,09068","4,67775","6,88941","4,67676","5,83605","7,70418","4,47975","8,46648","5,15394","5,281","8,891943" -INTL.33-12-AUT-BKWH.A, Austria,"28,501","30,008","29,893","29,577","28,384","30,288","30,496","25,401","35,151","34,641","31,179","31,112","34,483","36,336","35,349","36,696","33,874","35,744","36,792","40,292","41,418","40,05","39,825","32,883","36,394","36,31","35,48","36,732","37,969","40,487","36,466","32,511","41,862","40,138","39,001","35,255","37,954","36,462","35,73","40,43655","41,9356096","38,75133" -INTL.33-12-BEL-BKWH.A, Belgium,"0,274","0,377","0,325","0,331","0,348","0,282","0,339","0,425","0,354","0,3","0,263","0,226","0,338","0,252","0,342","0,335","0,237","0,30195","0,38511","0,338","0,455","0,437","0,356","0,245","0,314","0,285","0,355","0,385","0,406","0,325","0,298","0,193","0,353","0,376","0,289","0,314","0,367","0,268","0,3135","0,302","0,2669","0,3933" -INTL.33-12-BIH-BKWH.A, Bosnia and Herzegovina,--,--,--,--,--,--,--,--,--,--,--,--,"3,374","2,343","3,424","3,607","5,104","4,608","4,511","5,477","5,043","5,129","5,215","4,456","5,919","5,938","5,798","3,961","4,818","6,177","7,946","4,343","4,173","7,164","5,876","5,495","5,585","3,7521","6,35382","6,02019","4,58","6,722" -INTL.33-12-BGR-BKWH.A, Bulgaria,"3,674","3,58","3,018","3,318","3,226","2,214","2,302","2,512","2,569","2,662","1,859","2,417","2,042","1,923","1,453","2,291","2,89","2,726","3,066","2,725","2,646","1,72","2,172","2,999","3,136","4,294","4,196","2,845","2,796","3,435","4,98168","2,84328","3,14622","3,99564","4,55598","5,59845","3,8412","2,79972","5,09553","2,929499","2,820398","4,819205" -INTL.33-12-HRV-BKWH.A, Croatia,--,--,--,--,--,--,--,--,--,--,--,--,"4,298","4,302","4,881","5,212","7,156","5,234","5,403","6,524","5,794","6,482","5,311","4,827","6,888","6,27","5,94","4,194","5,164","6,663","9,035","4,983","4,789","8,536","8,917","6,327","6,784","5,255","7,62399","5,87268","5,6624","7,1277" -INTL.33-12-CYP-BKWH.A, Cyprus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -INTL.33-12-CZE-BKWH.A, Czechia,--,--,--,--,--,--,--,--,--,--,--,--,--,"1,355","1,445","1,982","1,949","1,68201","1,382","1,664","1,7404","2,033","2,467","1,369","1,999","2,356","2,525","2,068","2,004","2,405","2,775","1,95","2,107","2,704","1,909","1,779","1,983","1,852","1,615","1,98792","2,143884","2,40852" -INTL.33-12-DNK-BKWH.A, Denmark,"0,03","0,031","0,028","0,036","0,028","0,027","0,029","0,029","0,032","0,027","0,027","0,026","0,028","0,027","0,033","0,03","0,019","0,019","0,02673","0,031","0,03","0,028","0,032","0,021","0,027","0,023","0,023","0,028","0,026","0,019","0,021","0,017","0,017","0,013","0,015","0,01803","0,01927","0,017871","0,0148621","0,0172171","0,017064","0,016295" -INTL.33-12-EST-BKWH.A, Estonia,--,--,--,--,--,--,--,--,--,--,--,--,"0,001","0,001","0,003","0,002","0,002","0,003","0,004","0,004","0,005","0,007","0,006","0,013","0,022","0,022","0,014","0,021","0,028","0,032","0,027","0,029999","0,042","0,026","0,027","0,027","0,035","0,025999","0,0150003","0,0189999","0,03","0,0248" -INTL.33-12-FRO-BKWH.A, Faroe Islands,"0,049","0,049","0,049","0,049","0,049","0,049","0,049","0,049","0,062","0,071","0,074","0,074","0,083","0,073","0,075","0,075","0,069564","0,075066","0,076501","0,069453","0,075262","0,075195","0,095535","0,08483","0,093443","0,097986","0,099934","0,103407","0,094921","0,091482","0,06676","0,092","0,099","0,091","0,121","0,132","0,105","0,11","0,107","0,102","0,11","0,11" -INTL.33-12-FIN-BKWH.A, Finland,"10,115","13,518","12,958","13,445","13,115","12,211","12,266","13,658","13,229","12,9","10,75","13,065","14,956","13,341","11,669","12,796","11,742","12,11958","14,9","12,652","14,513","13,073","10,668","9,495","14,919","13,646","11,379","14,035","16,941","12,559","12,743","12,278001","16,666998","12,672","13,240001","16,583999","15,634127","14,609473","13,1369998","12,2454823","15,883","15,766" -INTL.33-12-CSK-BKWH.A, Former Czechoslovakia,"4,8","4,2","3,7","3,9","3,2","4,3",4,"4,853","4,355","4,229","3,919","3,119","3,602",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,-- -INTL.33-12-SCG-BKWH.A, Former Serbia and Montenegro,--,--,--,--,--,--,--,--,--,--,--,--,"11,23","10,395","11,016","12,071","14,266","12,636","12,763","13,243","11,88","12,326","11,633","9,752","11,01","11,912",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,-- -INTL.33-12-YUG-BKWH.A, Former Yugoslavia,"27,868","25,044","23,295","21,623","25,645","24,363","27,474","25,98","25,612","23,256","19,601","18,929",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,-- -INTL.33-12-FRA-BKWH.A, France,"68,253","70,358","68,6","67,515","64,01","60,248","60,953","68,623","73,952","45,744","52,796","56,277","68,313","64,3","78,057","72,196","64,43","63,151","61,479","71,832","66,466","73,888","59,992","58,567","59,276","50,965","55,741","57,029","63,017","56,428","61,945","45,184","59,099","71,042","62,993","54,876","60,094","49,389","64,485","56,913891","62,06191","58,856657" -INTL.33-12-DEU-BKWH.A, Germany,--,--,--,--,--,--,--,--,--,--,--,"14,742","17,223","17,699","19,731","21,562","21,737","17,18343","17,044","19,451","21,515","22,506","22,893","19,071","20,866","19,442","19,808","20,957","20,239","18,841","20,678","17,323","21,331","22,66","19,31","18,664","20,214","19,985","17,694","19,731","18,322","19,252" -INTL.33-12-DDR-BKWH.A," Germany, East","1,658","1,718","1,748","1,683","1,748","1,758","1,767","1,726","1,719","1,551","1,389",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,-- -INTL.33-12-DEUW-BKWH.A," Germany, West","17,125","17,889","17,694","16,713","16,434","15,354","16,526","18,36","18,128","16,482","15,769",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,-- -INTL.33-12-GIB-BKWH.A, Gibraltar,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -INTL.33-12-GRC-BKWH.A, Greece,"3,396","3,398","3,551","2,331","2,852","2,792","3,222","2,768","2,354","1,888","1,751","3,068","2,181","2,26","2,573","3,494","4,305","3,84318","3,68","4,546","3,656","2,076","2,772","4,718","4,625","4,967","5,806","2,565","3,279","5,32","7,431","3,998","4,387","6,337","4,464","5,782","5,543","3,962","5,035","3,9798","3,343687","5,909225" -INTL.33-12-HUN-BKWH.A, Hungary,"0,111","0,166","0,158","0,153","0,179","0,153","0,152","0,167","0,167","0,156","0,176","0,192","0,156","0,164","0,159","0,161","0,205","0,21384","0,15345","0,179","0,176","0,184","0,192","0,169","0,203","0,2","0,184","0,208","0,211","0,226","0,184","0,215999","0,205999","0,207999","0,294001","0,226719","0,253308","0,213999","0,216","0,2129999","0,238","0,202379" -INTL.33-12-ISL-BKWH.A, Iceland,"3,053","3,085","3,407","3,588","3,738","3,667","3,846","3,918","4,169","4,217","4,162","4,162","4,267","4,421","4,47","4,635","4,724","5,15493","5,565","5,987","6,292","6,512","6,907","7,017","7,063","6,949","7,22","8,31","12,303","12,156","12,509999","12,381999","12,213999","12,747001","12,554","13,541","13,091609","13,891929","13,679377","13,32911","12,9196201","13,5746171" -INTL.33-12-IRL-BKWH.A, Ireland,"0,833","0,855","0,792","0,776","0,68","0,824","0,91","0,673","0,862","0,684","0,69","0,738","0,809","0,757","0,911","0,706","0,715","0,67122","0,907","0,838","0,838","0,59","0,903","0,592","0,624","0,625","0,717","0,66","0,959","0,893","0,593","0,699","0,795","0,593","0,701","0,798","0,674","0,685","0,687","0,87813","0,932656","0,750122" -INTL.33-12-ITA-BKWH.A, Italy,"44,997","42,782","41,216","40,96","41,923","40,616","40,626","39,05","40,205","33,647","31,31","41,817","41,778","41,011","44,212","37,404","41,617","41,18697","40,808","44,911","43,763","46,343","39,125","33,303","41,915","35,706","36,624","32,488","41,207","48,647","50,506","45,36477","41,45625","52,24626","57,95955","45,08163","42,00768","35,83701","48,29913","45,31824","47,551784","44,739" -INTL.33-12-XKS-BKWH.A, Kosovo,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,"0,075","0,119","0,154","0,104","0,095","0,142","0,149","0,139","0,243","0,177","0,27027","0,2079","0,262826","0,300635" -INTL.33-12-LVA-BKWH.A, Latvia,--,--,--,--,--,--,--,--,--,--,--,--,"2,498","2,846","3,272","2,908","1,841","2,922","2,99","2,729","2,791","2,805","2,438","2,243","3,078","3,293","2,671","2,706","3,078","3,422","3,487998","2,8568","3,677","2,838","1,953","1,841","2,522819","4,355513","2,4170639","2,0958919","2,5840101","2,6889293" -INTL.33-12-LTU-BKWH.A, Lithuania,--,--,--,--,--,--,--,--,--,--,--,--,"0,308","0,389","0,447","0,369","0,323","0,291","0,413","0,409","0,336","0,322","0,35","0,323","0,417","0,446193","0,393","0,417","0,398","0,42","0,535","0,475","0,419","0,516","0,395","0,346","0,45","0,597","0,427","0,34254","0,3006","0,3837" -INTL.33-12-LUX-BKWH.A, Luxembourg,"0,086","0,095","0,084","0,083","0,088","0,071","0,084","0,101","0,097","0,072","0,07","0,083","0,069","0,066","0,117","0,087","0,059","0,082","0,114","0,084","0,119","0,117","0,098","0,078","0,103","0,093","0,11","0,116","0,131","0,105","0,104","0,061","0,095","0,114","0,104","0,095","0,111","0,082","0,089","0,10593","0,091602","0,1068" -INTL.33-12-MLT-BKWH.A, Malta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -INTL.33-12-MNE-BKWH.A, Montenegro,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,"1,733","1,271","1,524","2,05","2,723","1,192","1,462","2,479","1,734","1,476","1,825","1,014","1,693443","1,262781","0,867637","1,212652" -INTL.33-12-NLD-BKWH.A, Netherlands,0,0,0,0,0,"0,003","0,003","0,001","0,002","0,037","0,119","0,079","0,119","0,091","0,1","0,087","0,079","0,09108","0,111","0,089","0,141","0,116","0,109","0,071","0,094","0,087","0,105","0,106","0,101","0,097","0,105","0,057","0,104389","0,11431","0,112202","0,0927","0,100078","0,060759","0,0723481","0,074182","0,0462851","0,0838927" -INTL.33-12-MKD-BKWH.A, North Macedonia,--,--,--,--,--,--,--,--,--,--,--,--,"0,817","0,517","0,696","0,793","0,842","0,891","1,072","1,375","1,158","0,62","0,749","1,36","1,467","1,477","1,634",1,"0,832","1,257","2,407","1,419","1,031","1,568","1,195","1,846","1,878","1,099","1,773","1,15236","1,277144","1,451623" -INTL.33-12-NOR-BKWH.A, Norway,"82,717","91,876","91,507","104,704","104,895","101,464","95,321","102,341","107,919","117,369","119,933","109,032","115,505","118,024","110,398","120,315","102,823","108,677","114,546","120,237","140,4","119,258","128,078","104,425","107,693","134,331","118,175","132,319","137,654","124,03","116,257","119,78","141,189","127,551","134,844","136,662","142,244","141,651","138,202","123,66288","141,69",144 -INTL.33-12-POL-BKWH.A, Poland,"2,326","2,116","1,528","1,658","1,394","1,833","1,534","1,644","1,775","1,593","1,403","1,411","1,492","1,473","1,716","1,868","1,912","1,941","2,286","2,133","2,085","2,302","2,256","1,654","2,06","2,179","2,022","2,328","2,13","2,351","2,9","2,313","2,02","2,421","2,165","1,814","2,117","2,552","1,949","1,93842","2,118337","2,339192" -INTL.33-12-PRT-BKWH.A, Portugal,"7,873","4,934","6,82","7,897","9,609","10,512","8,364","9,005","12,037","5,72","9,065","8,952","4,599","8,453","10,551","8,26","14,613","12,97395","12,853","7,213","11,21","13,894","7,722","15,566","9,77","4,684","10,892","9,991","6,73","8,201","15,954","11,423","5,589","13,652","15,471","8,615","15,608","5,79","12,316","8,6526","12,082581","11,846464" -INTL.33-12-ROU-BKWH.A, Romania,"12,506","12,605","11,731","9,934","11,208","11,772","10,688","11,084","13,479","12,497","10,87","14,107","11,583","12,64","12,916","16,526","15,597","17,334","18,69","18,107","14,63","14,774","15,886","13,126","16,348","20,005","18,172","15,806","17,023","15,379","19,684","14,581","11,945","14,807","18,618","16,467","17,848","14,349","17,48736","15,580622","15,381243","17,376933" -INTL.33-12-SRB-BKWH.A, Serbia,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,"10,855","9,937","9,468","10,436","11,772","8,58","9,193","10,101","10,893","9,979","10,684","9,061","10,53261","9,457175","9,034496","11,284232" -INTL.33-12-SVK-BKWH.A, Slovakia,--,--,--,--,--,--,--,--,--,--,--,--,--,"3,432","4,311","4,831","4,185","4,023","4,224","4,429","4,569","4,878","5,215","3,4452","4,059","4,592","4,355","4,406",4,"4,324","5,184","3,211","3,687","4,329","3,762","3,701","4,302","4,321","3,506","4,27383","4,517","4,17" -INTL.33-12-SVN-BKWH.A, Slovenia,--,--,--,--,--,--,--,--,--,--,--,--,"3,379","2,974","3,348","3,187","3,616","3,046","3,4","3,684","3,771","3,741","3,265","2,916","4,033","3,426","3,555","3,233","3,978","4,666","4,452","3,506","3,841","4,562","6,011","3,75","4,443","3,814","4,643","4,43421","4,93406","4,711944" -INTL.33-12-ESP-BKWH.A, Spain,"29,16","21,64","25,99","26,696","31,088","30,895","26,105","27,016","34,76","19,046","25,16","27,01","18,731","24,133","27,898","22,881","39,404","34,43","33,665","22,634","29,274","40,617","22,691","40,643","31,359","18,209","25,699","27,036","23,13","26,147","41,576","30,07","20,192","36,45","38,815","27,656","35,77","18,007","33,743","24,23025","30,507","29,626" -INTL.33-12-SWE-BKWH.A, Sweden,"58,133","59,006","54,369","62,801","67,106","70,095","60,134","70,95","69,016","70,911","71,778","62,603","73,588","73,905","58,508","67,421","51,2226","68,365","74,25","70,974","77,798","78,269","65,696","53,005","59,522","72,075","61,106","65,497","68,378","65,193","66,279","66,047","78,333","60,81","63,227","74,734","61,645","64,651","61,79","64,46583","71,6","71,086" -INTL.33-12-CHE-BKWH.A, Switzerland,"32,481","35,13","35,974","35,069","29,871","31,731","32,576","34,328","35,437","29,477","29,497","31,756","32,373","35,416","38,678","34,817","28,458","33,70257","33,136","37,104","33,854","38,29","32,323","31,948","30,938","28,664","28,273","32,362","33,214","32,833","33,261","29,906","35,783","35,628","35,122","35,378","31,984","31,47968","32,095881","35,156989","37,867647","36,964485" -INTL.33-12-TUR-BKWH.A, Turkey,"11,159","12,308","13,81","11,13","13,19","11,822","11,637","18,314","28,447","17,61","22,917","22,456","26,302","33,611","30,28","35,186","40,07","39,41784","41,80671","34,33","30,57","23,77","33,346","34,977","45,623","39,165","43,802","35,492","32,937","35,598","51,423001","51,154999","56,668998","58,225","39,750001","65,856","66,685883","57,823851","59,490211","88,2094218","78,094369","55,1755392" -INTL.33-12-GBR-BKWH.A, United Kingdom,"3,921","4,369","4,543","4,548","3,992","4,08","4,767","4,13","4,915","4,732","5,119","4,534","5,329","4,237","5,043","4,79","3,359","4,127","5,117","5,336","5,085","4,055","4,78787","3,22767","4,844","4,92149","4,59315","5,0773","5,14119","5,22792","3,59138","5,69175","5,30965","4,70147","5,8878","6,29727","5,370412217","5,88187","5,44327","5,84628","6,75391","5,0149" -, Eurasia,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -INTL.33-12-MDA-BKWH.A, Moldova,--,--,--,--,--,--,--,--,--,--,--,--,"0,255","0,371","0,275","0,321","0,362","0,378","0,387","0,363","0,392","0,359","0,348","0,358","0,35","0,359","0,365","0,354","0,385","0,354","0,403","0,348","0,266","0,311","0,317","0,265","0,228","0,282","0,27324","0,29799","0,276","0,316" -INTL.33-12-UKR-BKWH.A, Ukraine,--,--,--,--,--,--,--,--,--,--,--,--,"7,725","10,929","11,997","9,853","8,546","9,757","15,756","14,177","11,161","11,912","9,531","9,146","11,635","12,239","12,757","10,042","11,397","11,817","13,02","10,837","10,374","13,663","8,393","5,343","7,594","8,856","10,32372","6,5083","7,5638","10,3326" +# https://www.eia.gov/international/data/world/electricity/electricity-generation?pd=2&p=000000000000000000000000000000g&u=1&f=A&v=mapbubble&a=-&i=none&vo=value&t=R&g=000000000000002&l=73-1028i008017kg6368g80a4k000e0ag00gg0004g8g0ho00g000400008&l=72-00000000000000000000000000080000000000000000000g&s=315532800000&e=1609459200000&ev=false& +Report generated on: 03-14-2024 13:40:38 +"API","","1980","1981","1982","1983","1984","1985","1986","1987","1988","1989","1990","1991","1992","1993","1994","1995","1996","1997","1998","1999","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2011","2012","2013","2014","2015","2016","2017","2018","2019","2020","2021" +"","hydroelectricity net generation (billion kWh)","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","" +"INTL.33-12-EURO-BKWH.A"," Europe","458.018","464.155","459.881","473.685","481.241","476.739","459.535","491.085","534.517","465.365","474.466","475.47","509.041","526.448","531.815","543.743","529.114164","543.845616","562.491501","566.861453","588.644662","584.806195","539.051405","503.7067","542.112443","542.974669","535.006084","538.449707","565.143111","561.761402","617.547148","540.926277","598.055253","629.44709","617.111295","613.079848","627.720566217","560.362524","616.5081462","606.5997419","644.1106599","628.1390143" +"INTL.33-12-ALB-BKWH.A"," Albania","2.919","3.018","3.093","3.167","3.241","3.315","3.365","3.979","3.713","3.846","2.82","3.483","3.187","3.281","3.733","4.162","5.669","4.978","4.872","5.231","4.548","3.519","3.477","5.117","5.411","5.319","4.951","2.76","3.759","5.201","7.49133","4.09068","4.67775","6.88941","4.67676","5.83605","7.70418","4.47975","8.46648","5.15394","5.281","8.891943" +"INTL.33-12-AUT-BKWH.A"," Austria","28.501","30.008","29.893","29.577","28.384","30.288","30.496","25.401","35.151","34.641","31.179","31.112","34.483","36.336","35.349","36.696","33.874","35.744","36.792","40.292","41.418","40.05","39.825","32.883","36.394","36.31","35.48","36.732","37.969","40.487","36.466","32.511","41.862","40.138","39.001","35.255","37.954","36.462","35.73","40.43655","41.9356096","38.75133" +"INTL.33-12-BEL-BKWH.A"," Belgium","0.274","0.377","0.325","0.331","0.348","0.282","0.339","0.425","0.354","0.3","0.263","0.226","0.338","0.252","0.342","0.335","0.237","0.30195","0.38511","0.338","0.455","0.437","0.356","0.245","0.314","0.285","0.355","0.385","0.406","0.325","0.298","0.193","0.353","0.376","0.289","0.314","0.367","0.268","0.3135","0.302","0.2669","0.3933" +"INTL.33-12-BIH-BKWH.A"," Bosnia and Herzegovina","--","--","--","--","--","--","--","--","--","--","--","--","3.374","2.343","3.424","3.607","5.104","4.608","4.511","5.477","5.043","5.129","5.215","4.456","5.919","5.938","5.798","3.961","4.818","6.177","7.946","4.343","4.173","7.164","5.876","5.495","5.585","3.7521","6.35382","6.02019","4.58","6.722" +"INTL.33-12-BGR-BKWH.A"," Bulgaria","3.674","3.58","3.018","3.318","3.226","2.214","2.302","2.512","2.569","2.662","1.859","2.417","2.042","1.923","1.453","2.291","2.89","2.726","3.066","2.725","2.646","1.72","2.172","2.999","3.136","4.294","4.196","2.845","2.796","3.435","4.98168","2.84328","3.14622","3.99564","4.55598","5.59845","3.8412","2.79972","5.09553","2.929499","2.820398","4.819205" +"INTL.33-12-HRV-BKWH.A"," Croatia","--","--","--","--","--","--","--","--","--","--","--","--","4.298","4.302","4.881","5.212","7.156","5.234","5.403","6.524","5.794","6.482","5.311","4.827","6.888","6.27","5.94","4.194","5.164","6.663","9.035","4.983","4.789","8.536","8.917","6.327","6.784","5.255","7.62399","5.87268","5.6624","7.1277" +"INTL.33-12-CYP-BKWH.A"," Cyprus","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-12-CZE-BKWH.A"," Czechia","--","--","--","--","--","--","--","--","--","--","--","--","--","1.355","1.445","1.982","1.949","1.68201","1.382","1.664","1.7404","2.033","2.467","1.369","1.999","2.356","2.525","2.068","2.004","2.405","2.775","1.95","2.107","2.704","1.909","1.779","1.983","1.852","1.615","1.98792","2.143884","2.40852" +"INTL.33-12-DNK-BKWH.A"," Denmark","0.03","0.031","0.028","0.036","0.028","0.027","0.029","0.029","0.032","0.027","0.027","0.026","0.028","0.027","0.033","0.03","0.019","0.019","0.02673","0.031","0.03","0.028","0.032","0.021","0.027","0.023","0.023","0.028","0.026","0.019","0.021","0.017","0.017","0.013","0.015","0.01803","0.01927","0.017871","0.0148621","0.0172171","0.017064","0.016295" +"INTL.33-12-EST-BKWH.A"," Estonia","--","--","--","--","--","--","--","--","--","--","--","--","0.001","0.001","0.003","0.002","0.002","0.003","0.004","0.004","0.005","0.007","0.006","0.013","0.022","0.022","0.014","0.021","0.028","0.032","0.027","0.029999","0.042","0.026","0.027","0.027","0.035","0.025999","0.0150003","0.0189999","0.03","0.0248" +"INTL.33-12-FRO-BKWH.A"," Faroe Islands","0.049","0.049","0.049","0.049","0.049","0.049","0.049","0.049","0.062","0.071","0.074","0.074","0.083","0.073","0.075","0.075","0.069564","0.075066","0.076501","0.069453","0.075262","0.075195","0.095535","0.08483","0.093443","0.097986","0.099934","0.103407","0.094921","0.091482","0.06676","0.092","0.099","0.091","0.121","0.132","0.105","0.11","0.107","0.102","0.11","0.11" +"INTL.33-12-FIN-BKWH.A"," Finland","10.115","13.518","12.958","13.445","13.115","12.211","12.266","13.658","13.229","12.9","10.75","13.065","14.956","13.341","11.669","12.796","11.742","12.11958","14.9","12.652","14.513","13.073","10.668","9.495","14.919","13.646","11.379","14.035","16.941","12.559","12.743","12.278001","16.666998","12.672","13.240001","16.583999","15.634127","14.609473","13.1369998","12.2454823","15.883","15.766" +"INTL.33-12-CSK-BKWH.A"," Former Czechoslovakia","4.8","4.2","3.7","3.9","3.2","4.3","4","4.853","4.355","4.229","3.919","3.119","3.602","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-SCG-BKWH.A"," Former Serbia and Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","11.23","10.395","11.016","12.071","14.266","12.636","12.763","13.243","11.88","12.326","11.633","9.752","11.01","11.912","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-YUG-BKWH.A"," Former Yugoslavia","27.868","25.044","23.295","21.623","25.645","24.363","27.474","25.98","25.612","23.256","19.601","18.929","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-FRA-BKWH.A"," France","68.253","70.358","68.6","67.515","64.01","60.248","60.953","68.623","73.952","45.744","52.796","56.277","68.313","64.3","78.057","72.196","64.43","63.151","61.479","71.832","66.466","73.888","59.992","58.567","59.276","50.965","55.741","57.029","63.017","56.428","61.945","45.184","59.099","71.042","62.993","54.876","60.094","49.389","64.485","56.913891","62.06191","58.856657" +"INTL.33-12-DEU-BKWH.A"," Germany","--","--","--","--","--","--","--","--","--","--","--","14.742","17.223","17.699","19.731","21.562","21.737","17.18343","17.044","19.451","21.515","22.506","22.893","19.071","20.866","19.442","19.808","20.957","20.239","18.841","20.678","17.323","21.331","22.66","19.31","18.664","20.214","19.985","17.694","19.731","18.322","19.252" +"INTL.33-12-DDR-BKWH.A"," Germany, East","1.658","1.718","1.748","1.683","1.748","1.758","1.767","1.726","1.719","1.551","1.389","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-DEUW-BKWH.A"," Germany, West","17.125","17.889","17.694","16.713","16.434","15.354","16.526","18.36","18.128","16.482","15.769","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-GIB-BKWH.A"," Gibraltar","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-12-GRC-BKWH.A"," Greece","3.396","3.398","3.551","2.331","2.852","2.792","3.222","2.768","2.354","1.888","1.751","3.068","2.181","2.26","2.573","3.494","4.305","3.84318","3.68","4.546","3.656","2.076","2.772","4.718","4.625","4.967","5.806","2.565","3.279","5.32","7.431","3.998","4.387","6.337","4.464","5.782","5.543","3.962","5.035","3.9798","3.343687","5.909225" +"INTL.33-12-HUN-BKWH.A"," Hungary","0.111","0.166","0.158","0.153","0.179","0.153","0.152","0.167","0.167","0.156","0.176","0.192","0.156","0.164","0.159","0.161","0.205","0.21384","0.15345","0.179","0.176","0.184","0.192","0.169","0.203","0.2","0.184","0.208","0.211","0.226","0.184","0.215999","0.205999","0.207999","0.294001","0.226719","0.253308","0.213999","0.216","0.2129999","0.238","0.202379" +"INTL.33-12-ISL-BKWH.A"," Iceland","3.053","3.085","3.407","3.588","3.738","3.667","3.846","3.918","4.169","4.217","4.162","4.162","4.267","4.421","4.47","4.635","4.724","5.15493","5.565","5.987","6.292","6.512","6.907","7.017","7.063","6.949","7.22","8.31","12.303","12.156","12.509999","12.381999","12.213999","12.747001","12.554","13.541","13.091609","13.891929","13.679377","13.32911","12.9196201","13.5746171" +"INTL.33-12-IRL-BKWH.A"," Ireland","0.833","0.855","0.792","0.776","0.68","0.824","0.91","0.673","0.862","0.684","0.69","0.738","0.809","0.757","0.911","0.706","0.715","0.67122","0.907","0.838","0.838","0.59","0.903","0.592","0.624","0.625","0.717","0.66","0.959","0.893","0.593","0.699","0.795","0.593","0.701","0.798","0.674","0.685","0.687","0.87813","0.932656","0.750122" +"INTL.33-12-ITA-BKWH.A"," Italy","44.997","42.782","41.216","40.96","41.923","40.616","40.626","39.05","40.205","33.647","31.31","41.817","41.778","41.011","44.212","37.404","41.617","41.18697","40.808","44.911","43.763","46.343","39.125","33.303","41.915","35.706","36.624","32.488","41.207","48.647","50.506","45.36477","41.45625","52.24626","57.95955","45.08163","42.00768","35.83701","48.29913","45.31824","47.551784","44.739" +"INTL.33-12-XKS-BKWH.A"," Kosovo","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","0.075","0.119","0.154","0.104","0.095","0.142","0.149","0.139","0.243","0.177","0.27027","0.2079","0.262826","0.300635" +"INTL.33-12-LVA-BKWH.A"," Latvia","--","--","--","--","--","--","--","--","--","--","--","--","2.498","2.846","3.272","2.908","1.841","2.922","2.99","2.729","2.791","2.805","2.438","2.243","3.078","3.293","2.671","2.706","3.078","3.422","3.487998","2.8568","3.677","2.838","1.953","1.841","2.522819","4.355513","2.4170639","2.0958919","2.5840101","2.6889293" +"INTL.33-12-LTU-BKWH.A"," Lithuania","--","--","--","--","--","--","--","--","--","--","--","--","0.308","0.389","0.447","0.369","0.323","0.291","0.413","0.409","0.336","0.322","0.35","0.323","0.417","0.446193","0.393","0.417","0.398","0.42","0.535","0.475","0.419","0.516","0.395","0.346","0.45","0.597","0.427","0.34254","0.3006","0.3837" +"INTL.33-12-LUX-BKWH.A"," Luxembourg","0.086","0.095","0.084","0.083","0.088","0.071","0.084","0.101","0.097","0.072","0.07","0.083","0.069","0.066","0.117","0.087","0.059","0.082","0.114","0.084","0.119","0.117","0.098","0.078","0.103","0.093","0.11","0.116","0.131","0.105","0.104","0.061","0.095","0.114","0.104","0.095","0.111","0.082","0.089","0.10593","0.091602","0.1068" +"INTL.33-12-MLT-BKWH.A"," Malta","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-12-MNE-BKWH.A"," Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","1.733","1.271","1.524","2.05","2.723","1.192","1.462","2.479","1.734","1.476","1.825","1.014","1.693443","1.262781","0.867637","1.212652" +"INTL.33-12-NLD-BKWH.A"," Netherlands","0","0","0","0","0","0.003","0.003","0.001","0.002","0.037","0.119","0.079","0.119","0.091","0.1","0.087","0.079","0.09108","0.111","0.089","0.141","0.116","0.109","0.071","0.094","0.087","0.105","0.106","0.101","0.097","0.105","0.057","0.104389","0.11431","0.112202","0.0927","0.100078","0.060759","0.0723481","0.074182","0.0462851","0.0838927" +"INTL.33-12-MKD-BKWH.A"," North Macedonia","--","--","--","--","--","--","--","--","--","--","--","--","0.817","0.517","0.696","0.793","0.842","0.891","1.072","1.375","1.158","0.62","0.749","1.36","1.467","1.477","1.634","1","0.832","1.257","2.407","1.419","1.031","1.568","1.195","1.846","1.878","1.099","1.773","1.15236","1.277144","1.451623" +"INTL.33-12-NOR-BKWH.A"," Norway","82.717","91.876","91.507","104.704","104.895","101.464","95.321","102.341","107.919","117.369","119.933","109.032","115.505","118.024","110.398","120.315","102.823","108.677","114.546","120.237","140.4","119.258","128.078","104.425","107.693","134.331","118.175","132.319","137.654","124.03","116.257","119.78","141.189","127.551","134.844","136.662","142.244","141.651","138.202","123.66288","141.69","144" +"INTL.33-12-POL-BKWH.A"," Poland","2.326","2.116","1.528","1.658","1.394","1.833","1.534","1.644","1.775","1.593","1.403","1.411","1.492","1.473","1.716","1.868","1.912","1.941","2.286","2.133","2.085","2.302","2.256","1.654","2.06","2.179","2.022","2.328","2.13","2.351","2.9","2.313","2.02","2.421","2.165","1.814","2.117","2.552","1.949","1.93842","2.118337","2.339192" +"INTL.33-12-PRT-BKWH.A"," Portugal","7.873","4.934","6.82","7.897","9.609","10.512","8.364","9.005","12.037","5.72","9.065","8.952","4.599","8.453","10.551","8.26","14.613","12.97395","12.853","7.213","11.21","13.894","7.722","15.566","9.77","4.684","10.892","9.991","6.73","8.201","15.954","11.423","5.589","13.652","15.471","8.615","15.608","5.79","12.316","8.6526","12.082581","11.846464" +"INTL.33-12-ROU-BKWH.A"," Romania","12.506","12.605","11.731","9.934","11.208","11.772","10.688","11.084","13.479","12.497","10.87","14.107","11.583","12.64","12.916","16.526","15.597","17.334","18.69","18.107","14.63","14.774","15.886","13.126","16.348","20.005","18.172","15.806","17.023","15.379","19.684","14.581","11.945","14.807","18.618","16.467","17.848","14.349","17.48736","15.580622","15.381243","17.376933" +"INTL.33-12-SRB-BKWH.A"," Serbia","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","10.855","9.937","9.468","10.436","11.772","8.58","9.193","10.101","10.893","9.979","10.684","9.061","10.53261","9.457175","9.034496","11.284232" +"INTL.33-12-SVK-BKWH.A"," Slovakia","--","--","--","--","--","--","--","--","--","--","--","--","--","3.432","4.311","4.831","4.185","4.023","4.224","4.429","4.569","4.878","5.215","3.4452","4.059","4.592","4.355","4.406","4","4.324","5.184","3.211","3.687","4.329","3.762","3.701","4.302","4.321","3.506","4.27383","4.517","4.17" +"INTL.33-12-SVN-BKWH.A"," Slovenia","--","--","--","--","--","--","--","--","--","--","--","--","3.379","2.974","3.348","3.187","3.616","3.046","3.4","3.684","3.771","3.741","3.265","2.916","4.033","3.426","3.555","3.233","3.978","4.666","4.452","3.506","3.841","4.562","6.011","3.75","4.443","3.814","4.643","4.43421","4.93406","4.711944" +"INTL.33-12-ESP-BKWH.A"," Spain","29.16","21.64","25.99","26.696","31.088","30.895","26.105","27.016","34.76","19.046","25.16","27.01","18.731","24.133","27.898","22.881","39.404","34.43","33.665","22.634","29.274","40.617","22.691","40.643","31.359","18.209","25.699","27.036","23.13","26.147","41.576","30.07","20.192","36.45","38.815","27.656","35.77","18.007","33.743","24.23025","30.507","29.626" +"INTL.33-12-SWE-BKWH.A"," Sweden","58.133","59.006","54.369","62.801","67.106","70.095","60.134","70.95","69.016","70.911","71.778","62.603","73.588","73.905","58.508","67.421","51.2226","68.365","74.25","70.974","77.798","78.269","65.696","53.005","59.522","72.075","61.106","65.497","68.378","65.193","66.279","66.047","78.333","60.81","63.227","74.734","61.645","64.651","61.79","64.46583","71.6","71.086" +"INTL.33-12-CHE-BKWH.A"," Switzerland","32.481","35.13","35.974","35.069","29.871","31.731","32.576","34.328","35.437","29.477","29.497","31.756","32.373","35.416","38.678","34.817","28.458","33.70257","33.136","37.104","33.854","38.29","32.323","31.948","30.938","28.664","28.273","32.362","33.214","32.833","33.261","29.906","35.783","35.628","35.122","35.378","31.984","31.47968","32.095881","35.156989","37.867647","36.964485" +"INTL.33-12-TUR-BKWH.A"," Turkiye","11.159","12.308","13.81","11.13","13.19","11.822","11.637","18.314","28.447","17.61","22.917","22.456","26.302","33.611","30.28","35.186","40.07","39.41784","41.80671","34.33","30.57","23.77","33.346","34.977","45.623","39.165","43.802","35.492","32.937","35.598","51.423001","51.154999","56.668998","58.225","39.750001","65.856","66.685883","57.823851","59.490211","88.2094218","78.094369","55.1755392" +"INTL.33-12-GBR-BKWH.A"," United Kingdom","3.921","4.369","4.543","4.548","3.992","4.08","4.767","4.13","4.915","4.732","5.119","4.534","5.329","4.237","5.043","4.79","3.359","4.127","5.117","5.336","5.085","4.055","4.78787","3.22767","4.844","4.92149","4.59315","5.0773","5.14119","5.22792","3.59138","5.69175","5.30965","4.70147","5.8878","6.29727","5.370412217","5.88187","5.44327","5.84628","6.75391","5.0149" +""," Eurasia","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","" +"INTL.33-12-MDA-BKWH.A"," Moldova","--","--","--","--","--","--","--","--","--","--","--","--","0.255","0.371","0.275","0.321","0.362","0.378","0.387","0.363","0.392","0.359","0.348","0.358","0.35","0.359","0.365","0.354","0.385","0.354","0.403","0.348","0.266","0.311","0.317","0.265","0.228","0.282","0.27324","0.29799","0.276","0.316" +"INTL.33-12-UKR-BKWH.A"," Ukraine","--","--","--","--","--","--","--","--","--","--","--","--","7.725","10.929","11.997","9.853","8.546","9.757","15.756","14.177","11.161","11.912","9.531","9.146","11.635","12.239","12.757","10.042","11.397","11.817","13.02","10.837","10.374","13.663","8.393","5.343","7.594","8.856","10.32372","6.5083","7.5638","10.3326" \ No newline at end of file diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index a9e111ea..d922a753 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -344,6 +344,7 @@ rule build_hydro_profile: params: hydro=config_provider("renewable", "hydro"), countries=config_provider("countries"), + snapshots=config_provider("snapshots"), input: country_shapes=resources("country_shapes.geojson"), eia_hydro_generation="data/eia_hydro_annual_generation.csv", diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index ce8f3a4c..99f15427 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -77,6 +77,8 @@ def get_eia_annual_hydro_generation(fn, countries, capacities=False): fn, skiprows=2, index_col=1, na_values=[" ", "--"], decimal="," ).iloc[1:, 1:] df.index = df.index.str.strip() + df.columns = df.columns.astype(int) + df = df.astype(float) former_countries = { "Former Czechoslovakia": dict( @@ -100,7 +102,7 @@ def get_eia_annual_hydro_generation(fn, countries, capacities=False): } for k, v in former_countries.items(): - period = [str(i) for i in range(v["start"], v["end"] + 1)] + period = [i for i in range(v["start"], v["end"] + 1)] ratio = df.loc[v["countries"]].T.dropna().sum() ratio /= ratio.sum() for country in v["countries"]: @@ -128,7 +130,7 @@ def get_eia_annual_hydro_generation(fn, countries, capacities=False): def correct_eia_stats_by_capacity(eia_stats, fn, countries, baseyear=2019): cap = get_eia_annual_hydro_generation(fn, countries, capacities=True) - ratio = cap / cap.loc[str(baseyear)] + ratio = cap / cap.loc[baseyear] eia_stats_corrected = eia_stats / ratio to_keep = ["AL", "AT", "CH", "DE", "GB", "NL", "RS", "RO", "SK"] to_correct = eia_stats_corrected.columns.difference(to_keep) @@ -137,9 +139,10 @@ def correct_eia_stats_by_capacity(eia_stats, fn, countries, baseyear=2019): def approximate_missing_eia_stats(eia_stats, runoff_fn, countries): runoff = pd.read_csv(runoff_fn, index_col=0).T[countries] + runoff.index = runoff.index.astype(int) - # fix ES, PT data points - runoff.loc["1978", ["ES", "PT"]] = runoff.loc["1979", ["ES", "PT"]] + # fix outliers; exceptional floods in 1977-1979 in ES & PT + runoff.loc[1978, ["ES", "PT"]] = runoff.loc[1979, ["ES", "PT"]] runoff_eia = runoff.loc[eia_stats.index] From 8aa4b5b558a2976781de2b307d450f0d68c45eaf Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 14:09:39 +0100 Subject: [PATCH 118/173] build_heat_totals: use more years for regression --- scripts/build_heat_totals.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/build_heat_totals.py b/scripts/build_heat_totals.py index e49ae635..d0dab987 100644 --- a/scripts/build_heat_totals.py +++ b/scripts/build_heat_totals.py @@ -15,17 +15,22 @@ idx = pd.IndexSlice def approximate_heat_demand(energy_totals, hdd): - if isinstance(hdd, str): - hdd = pd.read_csv(hdd, index_col=0).T - hdd.index = hdd.index.astype(int) + + countries = hdd.columns demands = {} for kind, sector in product(["total", "electricity"], ["services", "residential"]): - row = idx[:, 2007:2015] + # reduced number years (2007-2021) for regression because it implicitly + # assumes a constant building stock + row = idx[:, 2007:2021] col = f"{kind} {sector} space" demand = energy_totals.loc[row, col].unstack(0) + # ffill for GB in 2020- and bfill for CH 2007-2009 + # compromise to have more years available for the fit + demand = demand.ffill(axis=0).bfill(axis=0) + demand_approx = {} for c in countries: @@ -62,16 +67,15 @@ def approximate_heat_demand(energy_totals, hdd): if __name__ == "__main__": if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake - snakemake = mock_snakemake("build_energy_totals") + snakemake = mock_snakemake("build_heat_totals") hdd = pd.read_csv(snakemake.input.hdd, index_col=0).T + hdd.index = hdd.index.astype(int) energy_totals = pd.read_csv(snakemake.input.energy_totals, index_col=[0, 1]) - countries = hdd.columns - heat_demand = approximate_heat_demand(energy_totals, hdd) heat_demand.to_csv(snakemake.output.heat_totals) From 090792c9ec7583cf2f2f8ad7ee839da636f7a411 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:10:21 +0000 Subject: [PATCH 119/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- data/eia_hydro_annual_capacity.csv | 2 +- data/eia_hydro_annual_generation.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/eia_hydro_annual_capacity.csv b/data/eia_hydro_annual_capacity.csv index 8092096d..42c0cc6f 100644 --- a/data/eia_hydro_annual_capacity.csv +++ b/data/eia_hydro_annual_capacity.csv @@ -50,4 +50,4 @@ Report generated on: 03-14-2024 13:39:49 "INTL.33-7-GBR-MK.A"," United Kingdom","2.451","2.451","2.451","2.721","4.188","4.19","4.192","4.197","4.196","1.424","1.11","1.415","1.423","1.425","1.425","1.432","1.455","1.488","1.475","1.477","1.485","1.629","1.59","1.486","1.499","1.501","1.515","1.522","1.626","1.638","1.637","1.673","1.693","1.709","1.73","1.777","1.836","1.873","1.878","1.878","1.879","1.88" ""," Eurasia","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","" "INTL.33-7-MDA-MK.A"," Moldova","--","--","--","--","--","--","--","--","--","--","--","--","0.064","0.064","0.064","0.056","0.056","0.064","0.064","0.064","0.064","0.06","0.06","0.06","0.059","0.059","0.056","0.056","0.064","0.064","0.064","0.064","0.064","0.064","0.064","0.064","0.064","0.064","0.064","0.064","0.076","0.076" -"INTL.33-7-UKR-MK.A"," Ukraine","--","--","--","--","--","--","--","--","--","--","--","--","4.705","4.706","4.706","4.706","4.706","4.706","4.706","4.7","4.7","4.731","4.758","4.766","4.781","4.717","4.746","4.731","4.798","4.795","4.596","4.607","4.608","4.632","4.665","4.697","4.658","4.668","4.668","4.668","4.666","4.43" \ No newline at end of file +"INTL.33-7-UKR-MK.A"," Ukraine","--","--","--","--","--","--","--","--","--","--","--","--","4.705","4.706","4.706","4.706","4.706","4.706","4.706","4.7","4.7","4.731","4.758","4.766","4.781","4.717","4.746","4.731","4.798","4.795","4.596","4.607","4.608","4.632","4.665","4.697","4.658","4.668","4.668","4.668","4.666","4.43" diff --git a/data/eia_hydro_annual_generation.csv b/data/eia_hydro_annual_generation.csv index d4e64a31..b228ddc6 100644 --- a/data/eia_hydro_annual_generation.csv +++ b/data/eia_hydro_annual_generation.csv @@ -50,4 +50,4 @@ Report generated on: 03-14-2024 13:40:38 "INTL.33-12-GBR-BKWH.A"," United Kingdom","3.921","4.369","4.543","4.548","3.992","4.08","4.767","4.13","4.915","4.732","5.119","4.534","5.329","4.237","5.043","4.79","3.359","4.127","5.117","5.336","5.085","4.055","4.78787","3.22767","4.844","4.92149","4.59315","5.0773","5.14119","5.22792","3.59138","5.69175","5.30965","4.70147","5.8878","6.29727","5.370412217","5.88187","5.44327","5.84628","6.75391","5.0149" ""," Eurasia","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","" "INTL.33-12-MDA-BKWH.A"," Moldova","--","--","--","--","--","--","--","--","--","--","--","--","0.255","0.371","0.275","0.321","0.362","0.378","0.387","0.363","0.392","0.359","0.348","0.358","0.35","0.359","0.365","0.354","0.385","0.354","0.403","0.348","0.266","0.311","0.317","0.265","0.228","0.282","0.27324","0.29799","0.276","0.316" -"INTL.33-12-UKR-BKWH.A"," Ukraine","--","--","--","--","--","--","--","--","--","--","--","--","7.725","10.929","11.997","9.853","8.546","9.757","15.756","14.177","11.161","11.912","9.531","9.146","11.635","12.239","12.757","10.042","11.397","11.817","13.02","10.837","10.374","13.663","8.393","5.343","7.594","8.856","10.32372","6.5083","7.5638","10.3326" \ No newline at end of file +"INTL.33-12-UKR-BKWH.A"," Ukraine","--","--","--","--","--","--","--","--","--","--","--","--","7.725","10.929","11.997","9.853","8.546","9.757","15.756","14.177","11.161","11.912","9.531","9.146","11.635","12.239","12.757","10.042","11.397","11.817","13.02","10.837","10.374","13.663","8.393","5.343","7.594","8.856","10.32372","6.5083","7.5638","10.3326" From 5b1575cb50e690e7130190bca22adba9ef568772 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 14:18:26 +0100 Subject: [PATCH 120/173] build_hydro: change decimal to '.' from ',' with updated data --- scripts/build_hydro_profile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 99f15427..ebadd047 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -74,11 +74,10 @@ cc = coco.CountryConverter() def get_eia_annual_hydro_generation(fn, countries, capacities=False): # in billion kWh/a = TWh/a df = pd.read_csv( - fn, skiprows=2, index_col=1, na_values=[" ", "--"], decimal="," + fn, skiprows=2, index_col=1, na_values=[" ", "--"] ).iloc[1:, 1:] df.index = df.index.str.strip() df.columns = df.columns.astype(int) - df = df.astype(float) former_countries = { "Former Czechoslovakia": dict( From f84707ba38ee5fdfd9b95a278f00f7ec3035a883 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 14:30:50 +0100 Subject: [PATCH 121/173] drop_leap_days already in renewable profiles --- rules/build_electricity.smk | 4 ++++ scripts/build_hydro_profile.py | 7 ++++++- scripts/build_line_rating.py | 7 ++++--- scripts/build_renewable_profiles.py | 6 ++++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index d922a753..edf5fca3 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -267,6 +267,7 @@ def input_ua_md_availability_matrix(w): rule build_renewable_profiles: params: snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), renewable=config_provider("renewable"), input: unpack(input_ua_md_availability_matrix), @@ -345,6 +346,7 @@ rule build_hydro_profile: hydro=config_provider("renewable", "hydro"), countries=config_provider("countries"), snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), input: country_shapes=resources("country_shapes.geojson"), eia_hydro_generation="data/eia_hydro_annual_generation.csv", @@ -370,6 +372,7 @@ rule build_hydro_profile: rule build_line_rating: params: snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), input: base_network=resources("networks/base.nc"), cutout=lambda w: "cutouts/" @@ -413,6 +416,7 @@ rule add_electricity: length_factor=config_provider("lines", "length_factor"), scaling_factor=config_provider("load", "scaling_factor"), countries=config_provider("countries"), + snapshots=config_provider("snapshots"), renewable=config_provider("renewable"), electricity=config_provider("electricity"), conventional=config_provider("conventional"), diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index ebadd047..510bc6fa 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -174,7 +174,12 @@ if __name__ == "__main__": set_scenario_config(snakemake) params_hydro = snakemake.params.hydro - cutout = atlite.Cutout(snakemake.input.cutout) + + time = pd.date_range(freq="h", **snakemake.params.snapshots) + if snakemake.params.drop_leap_day: + time = time[~((time.month == 2) & (time.day == 29))] + + cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) countries = snakemake.params.countries country_shapes = ( diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py index 5b4642d1..794638d2 100755 --- a/scripts/build_line_rating.py +++ b/scripts/build_line_rating.py @@ -146,10 +146,11 @@ if __name__ == "__main__": configure_logging(snakemake) set_scenario_config(snakemake) - snapshots = snakemake.params.snapshots - n = pypsa.Network(snakemake.input.base_network) - time = pd.date_range(freq="h", **snapshots) + time = pd.date_range(freq="h", **snakemake.params.snapshots) + if snakemake.params.drop_leap_day: + time = time[~((time.month == 2) & (time.day == 29))] + cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) da = calculate_line_rating(n, cutout) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index e1576850..88057fb6 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -227,9 +227,11 @@ if __name__ == "__main__": else: client = None - sns = pd.date_range(freq="h", **snakemake.params.snapshots) + time = pd.date_range(freq="h", **snakemake.params.snapshots) + if snakemake.params.drop_leap_day: + time = time[~((time.month == 2) & (time.day == 29))] - cutout = atlite.Cutout(snakemake.input.cutout).sel(time=sns) + cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) regions = gpd.read_file(snakemake.input.regions) assert not regions.empty, ( f"List of regions in {snakemake.input.regions} is empty, please " From f695b63732d6e8a56716775c8b5c3a361ba36a53 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 14:31:18 +0100 Subject: [PATCH 122/173] add more documentation --- config/config.default.yaml | 2 +- doc/configtables/load.csv | 1 + rules/build_sector.smk | 1 - scripts/build_energy_totals.py | 2 +- ...build_population_weighted_energy_totals.py | 3 +- scripts/build_shipping_demand.py | 2 +- scripts/new-energy-totals.csv | 34 +++++++++++++++++++ 7 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 scripts/new-energy-totals.csv diff --git a/config/config.default.yaml b/config/config.default.yaml index ae45234c..da2e20f1 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -239,7 +239,7 @@ renewable: flatten_dispatch: false flatten_dispatch_buffer: 0.2 clip_min_inflow: 1.0 - eia_norm_year: 2013 + eia_norm_year: false eia_correct_by_capacity: false eia_approximate_missing: false diff --git a/doc/configtables/load.csv b/doc/configtables/load.csv index 0a62aeef..b245268f 100644 --- a/doc/configtables/load.csv +++ b/doc/configtables/load.csv @@ -4,3 +4,4 @@ time_shift_for_large_gaps,string,string,"Periods which are used for copying time manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`." scaling_factor,--,float,"Global correction factor for the load time series." fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year" +supplement_missing_data_artificially,bool,"{true, false}","Whether to supplement missing data for selected time period should be supplemented by artificial data from https://zenodo.org/record/7070438/files/demand_hourly.csv." \ No newline at end of file diff --git a/rules/build_sector.smk b/rules/build_sector.smk index e41d0224..14226ef8 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -908,7 +908,6 @@ rule prepare_sector_network: else [] ), network=resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), - energy_totals_name=resources("energy_totals.csv"), eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition", pop_weighted_energy_totals=resources( "pop_weighted_energy_totals_s{simpl}_{clusters}.csv" diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 48df4625..01532872 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -750,7 +750,7 @@ def build_transport_data(countries, population, idees): missing = transport_data.index[transport_data["average fuel efficiency"].isna()] if not missing.empty: logger.info( - f"Missing data on fuel efficiency from:\n{list(missing)}\nFilling gapswith averaged data." + f"Missing data on fuel efficiency from:\n{list(missing)}\nFilling gaps with averaged data." ) fill_values = transport_data["average fuel efficiency"].mean() diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index b26f289d..c2be19a9 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -15,8 +15,9 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_population_weighted_energy_totals", + kind='energy', simpl="", - clusters=48, + clusters=60, ) set_scenario_config(snakemake) diff --git a/scripts/build_shipping_demand.py b/scripts/build_shipping_demand.py index 58def027..b50cd316 100644 --- a/scripts/build_shipping_demand.py +++ b/scripts/build_shipping_demand.py @@ -18,7 +18,7 @@ if __name__ == "__main__": from _helpers import mock_snakemake snakemake = mock_snakemake( - "build_shipping_demand_per_node", + "build_shipping_demand", simpl="", clusters=48, ) diff --git a/scripts/new-energy-totals.csv b/scripts/new-energy-totals.csv new file mode 100644 index 00000000..78ba6736 --- /dev/null +++ b/scripts/new-energy-totals.csv @@ -0,0 +1,34 @@ +country,electricity residential space,total residential space,total residential water,electricity residential water,total residential cooking,electricity residential cooking,total residential,electricity residential,derived heat residential,thermal uses residential,total services space,electricity services space,total services water,electricity services water,total services cooking,electricity services cooking,total services,electricity services,derived heat services,thermal uses services,total agriculture electricity,total agriculture heat,total agriculture machinery,total agriculture,total road,electricity road,total two-wheel,total passenger cars,electricity passenger cars,total other road passenger,electricity other road passenger,total light duty road freight,electricity light duty road freight,total heavy duty road freight,total rail,electricity rail,total rail passenger,electricity rail passenger,total rail freight,electricity rail freight,total aviation passenger,total aviation freight,total domestic aviation passenger,total international aviation passenger,total domestic aviation freight,total international aviation freight,total domestic aviation,total international aviation,total domestic navigation,total international navigation +AL,0.5570465743624181,2.868681023831782,0.9246366122728509,0.4701405463928116,0.7560298250278052,0.606309006828873,6.745434890000001,3.8599970000000003,,,0.8492947400612103,0.24188377504306646,0.19063435423096936,0.1082113698760542,0.24955086644902696,0.190340216176473,2.2211439200000003,1.4779985500000001,,,,,,,8.933037890000001,0.0,0.11476078758153294,5.255099996394696,0.0,0.43826272462667554,0.0,1.0066258503715193,0.0,2.1182885310255757,0.03566921000000001,0.0,0.026867054339929548,0.0,0.008802155660070457,0.0,0.23558664812723476,0.015249191872765277,0.0,0.23558664812723476,0.0,0.015249191872765277,0.0,0.25083584000000003,0.38266189000000006,0.23777535000000002 +AT,3.8347980364339187,52.728795606720304,9.042762942245817,1.6580654856850694,3.332605848035772,2.2140159394525,74.4128059879979,17.31873449075142,8.985794565462129,65.16773835369318,17.423168106040677,1.362361756513761,3.4703813442929237,0.2967617406747894,3.4964403032216045,2.3458829314999434,33.49193062767192,13.244905252570849,9.882817057376736,24.839638062603143,0.6497150410629878,3.8719030223442585,1.7388785775882416,6.260496640995489,88.04632651098169,0.035087453918020005,0.6026064240302123,47.145477443307264,0.010621583280449667,3.067508471104808,0.02013826679946008,5.385434496257479,0.003947974868709491,31.84529967628193,3.384639661479662,2.8930267787407162,2.332832737694036,1.971964325107237,1.0518069237856262,0.9210624536334792,7.604475261435951,0.3130344308815537,0.26550445600104483,7.338970805434905,0.07206205482497326,0.24097237605658042,0.3375665108260181,7.579943181491485,0.058055683482739553,0.23600759000000002 +BA,0.9551423259051094,8.062615381913417,2.2035460750967655,0.8061285279384439,1.4999498844734496,1.039610369496632,15.49027994803239,6.61856060561295,,,2.075880307094079,0.37511972750507755,0.3986027383060623,0.16781704177400622,0.46097540282364163,0.2951846191943365,4.372938190104563,2.2921190692934506,,,,,,,10.403943245213432,0.0,0.1336571875633523,6.120399665114565,0.0,0.5104266398122862,0.0,1.172375886612297,0.0,2.4670838661109293,0.2691038049081697,0.22364645608190642,0.20269657078330564,0.1718117268416388,0.06640723412486403,0.051834729240267635,0.22862718311632196,0.01479871550610001,0.0,0.22862718311632196,0.0,0.01479871550610001,0.0,0.24342589862242198,0.05081711523682938,0.0511435984699603 +BE,2.815051564451901,72.00315684834005,14.758606504598777,1.9245397162358575,5.61762658049057,2.123765047409,104.53126486079485,19.81013300307489,0.048333322074790706,92.47214768445166,35.55025666623682,6.159088599308961,4.553234776118736,1.3005782934779981,4.296286602056132,1.7511206036264235,57.027872149578414,22.25434577090179,0.7699977716616743,45.32121766089195,1.2652057097802312,6.510654609861979,2.1504989713552063,9.926359290997418,93.22107248215386,0.01074409225014649,0.6477429921079886,54.179087820705185,0.006636212751625691,3.954966803176898,0.001991249371656604,10.47276574878302,0.0015859584461520306,23.966509117380756,2.2359228430787947,1.596953398545387,1.7366935264699335,1.306970119159899,0.4992293166088615,0.2899832793854877,11.903404936264774,2.9443700871166114,0.0,11.903404936264774,0.5794491728255937,2.3649209142910177,0.5794491728255937,14.268325850555792,1.822780948768488,71.59805975 +BG,1.0366899653005102,13.774445984268725,5.032599359348841,2.50096431277225,2.8263907739201346,2.446346362124563,26.126347522623533,10.536998125106878,3.70225381151711,21.738366989691684,3.6161635069149147,1.317290338927257,1.4876301353881975,0.8459255010716765,1.614166101394505,1.2802729496644545,11.214107236902933,7.954580847599141,1.1947324162166162,7.013424737247175,0.2157562658354375,1.238388279337861,0.8037223126895519,2.257866857862851,27.913508544418903,0.05240401618919764,0.2678003482560851,18.16930433603284,0.0,2.6753054767672606,0.05240401618919764,1.723251062925399,0.0,5.077847320437322,0.36999955326892575,0.19499962550233102,0.2595248907228583,0.1337123426290272,0.11047466254606741,0.06128728287330383,1.9735489165680373,0.0214510834319626,0.06685311111111157,1.9066958054569259,0.01455716249528969,0.0068939209366729115,0.08141027360640127,1.9135897263935986,0.0,1.0494446800000001 +CH,4.388888888888889,51.52777777777778,8.944444444444445,2.388888888888889,2.611111111111111,1.3611111111111112,72.58333333333333,18.777777777777775,,,21.5,1.1666666666666667,2.8055555555555554,0.19444444444444442,0.6388888888888888,0.6388888888888888,40.0,17.055555555555554,,,,,,,55.888888888888886,0.0,0.7179923543283743,32.87814329403942,0.0,2.7419582254567123,0.0,6.297879959411784,0.0,13.25291505565259,3.1666666666666665,3.1666666666666665,2.3852225886072858,2.432725641458858,0.7814440780593807,0.7339410252078085,17.617703597787695,1.1600741799900811,0.8685212443581152,16.749182353429582,0.07592320008632922,1.0841509799037519,0.9444444444444444,17.833333333333332,0.4444444444444445,0.0 +CZ,2.5725233632931075,62.57273499598758,10.288206369245467,2.9111285854172344,5.349291874222479,2.9169812116183698,83.99741666322201,14.715973413943496,13.888879856159699,78.24427707959727,20.246784369915833,4.388583048588386,3.0235451499346557,0.9434718391306539,3.505353453229548,1.7737748387302745,34.29840057384516,14.78664045488378,5.310847457419239,27.053057440415177,0.6941273932306512,4.054135056204236,2.4003802728676513,7.148642722302538,63.503988188281525,0.08911029892052649,0.6865202837197137,30.92628881993422,0.0007819669138069432,3.866800834894914,0.08831777854219773,5.7476679804286555,0.0,22.276710269304015,2.5098763617700213,1.491101348077621,1.957212181581117,1.1043425069361408,0.5526641801889038,0.38675884114148024,3.4758788630067836,0.048572629407252134,0.056765346606964515,3.419113516399819,0.019945792240296925,0.028626837166955205,0.07671113884726144,3.447740353566774,0.023888852908381327,0.0 +DE,23.552538648485736,482.88953556791756,88.86181482888813,9.341725122373857,41.094774831600795,14.516493828941883,694.2861367421355,136.00002715231648,51.24612134236524,614.0643012925772,200.1083899771467,8.177528100639465,38.567939444757855,8.148137272139993,48.02923585230634,20.88733594674748,388.35703665770825,141.5218730593723,16.87389225775031,291.39267527735336,7.53469558411125,5.474391021838289,1.4190403036639783,14.42812690961352,600.5438784206764,0.11224182554399796,5.370903791988889,380.78585774165197,0.0804737905883431,18.892615204428267,0.014714641214532026,28.660484589683342,0.015353321394055243,166.83401709292394,15.93416884756733,11.885002372832927,11.472824304223883,8.279888081535278,4.461344543343446,3.605114291297649,91.38069880171732,13.41045647623245,7.896669996323121,83.4840288053942,2.19924733222008,11.211209144012372,10.095917328543202,94.69523794940658,3.3722220527550077,26.697188350000005 +DK,0.7013376012077638,33.93012445800281,7.368540509780002,0.5229211600849933,3.1224115802390884,2.437647648641987,50.639691342206994,10.307073119763176,19.141944444444412,44.46958545008379,11.568860496650082,0.7165794765906534,1.9268567416141313,0.4139126319754398,2.107350856421504,1.6568129470533832,22.96597151274313,10.221030677425023,8.886388888888847,15.964433033531657,1.6065828331780032,4.6006732485303825,2.958335976083448,9.165592057791836,42.126010180586306,0.006041443591563428,0.26907068768936326,23.38606046077933,0.005426936489976022,3.9158027200781587,0.0,5.764536143667914,0.0006145071015874062,8.79054016837154,1.3112062796097734,0.3861092120959323,1.201322441895927,0.35393692637112534,0.10988383771384651,0.03217228572480684,10.062240435459456,0.3052602112787246,0.5934322985915401,9.468808136867919,0.10206631066816502,0.20319390061055956,0.6954986092597052,9.672002037478478,1.978361573681062,7.339855820000001 +EE,0.11231606763657928,7.615319503233344,1.5105449935831652,0.06137687617864419,0.7305183637351055,0.6392485833424982,10.871312066259437,1.8649831545295248,3.8389405197191633,9.856556620105813,2.8051300129906593,0.7294467788998327,0.3759461883671719,0.16897479263456983,0.43229161868604543,0.3654118725947719,4.8619944419222145,2.5288861940995093,1.1860593158990447,3.6224949714927197,0.17269894009588554,0.7062927576664825,0.4019989426098384,1.2809906403722064,8.143920002436563,0.02196621718242599,0.02187321389494755,5.041673827103431,0.0035133698706650515,0.9394047336082759,0.018423149598824944,0.6543028641167076,2.969771293599253e-05,1.4866653637132001,0.3473335032636153,0.04203126411907797,0.08800246779439479,0.04203126411907797,0.2593310354692205,0.0,0.32494722922299063,0.009489420040482168,0.0,0.32494722922299063,0.0036550739486526196,0.005834346091829548,0.0036550739486526196,0.33078157531482016,0.04694670715427129,4.740620600000001 +ES,10.794125129618664,71.26452543322868,39.68527650785515,7.710514219460869,19.26597649255715,8.828493562332296,177.196549581737,74.63098166927618,0.0,131.8776230960182,34.390226550633635,5.784912430114415,14.00729551399568,6.8886787516288885,17.98020571548081,13.01444004144452,115.94028841537886,75.5600175062191,0.0,75.67658371390513,3.798490176657542,20.504156811235546,10.361953239126692,34.664600227019776,295.6964784476745,0.008958749867787469,5.271025477457057,177.9796367179715,0.006198765274153721,10.784071123614806,0.0,20.279070030269107,0.0027599845936337485,81.38267509836203,8.270129202825443,4.551941091567911,7.174411141940354,4.18254627330064,1.095718060885089,0.36939481826726994,58.78529713592237,1.0458142160063002,18.097222294891036,40.68807484103132,0.33629527208914706,0.7095189439171532,18.433517566980182,41.39759378494848,5.86721442260112,84.04543941000001 +FI,7.848535711327018,41.84350237788999,9.048520920884483,3.443099269456141,2.9395617816882234,2.8792251386947934,60.826832141117144,21.50989869058375,18.587994476407626,53.92184520851802,21.23055041955415,6.199986054777693,2.553779457989475,0.9989674641957111,2.737139467312127,2.6900543568450637,34.04765319297288,17.50312586133877,12.247279309978593,26.808838116560455,1.2798819960077403,7.441003309894633,1.9312393542030517,10.652124660105425,45.56549617623142,0.0019365717691190252,0.5223209306867633,26.118742999170177,0.0007992579455364848,3.2601013702649593,0.00016885440537935242,5.613312604425437,0.0006476900733945322,10.051018271684086,1.1047159761512422,0.7330519889163917,0.6084665070352008,0.5419802858183081,0.4962494691160414,0.19107170309808352,7.731241694301903,0.3876090573520712,1.4438906650320935,6.287351029269809,0.11591836963879651,0.2716906877132747,1.55980903467089,6.559041716983084,1.725813385303544,1.5172963200000003 +FR,36.34821295388092,304.3090830974361,74.10444863335236,20.89120031251567,42.04728767502008,19.96798113754325,503.553879238341,167.90380216073933,14.323611111111111,421.0509096549809,149.2072481280514,29.02123818481345,25.328465674571646,12.10488257238001,28.51040521266642,17.00244127422052,285.9163571402345,142.24734181479553,8.505277777777804,216.87393093116245,7.478237257126606,32.74985084813704,20.230248718717654,60.4583368239813,477.2711666499798,0.17960126429470955,6.464992469847664,252.13117399671086,0.07795214668519337,21.88511370979885,0.039237383416606084,91.90030834156023,0.061631171267770395,104.88957813206223,12.134370941891715,10.168257793403882,10.262575388236112,8.653518950028449,1.8717955536556061,1.5147388433754336,71.22571967077627,3.5990187474665083,19.982431345628665,51.2432883251476,1.4447624710282905,2.154256276438218,21.427193816656956,53.397544601585814,5.826683163334846,26.200994400000006 +GB,32.35436698812953,318.9482399097387,79.0068899383846,11.692921151505214,20.777464730965892,3.4829778546191315,480.6012101626519,113.45010172801696,0.603890281538269,419.06137321331204,111.80304633929093,16.901263938070077,21.670713737923748,9.433929352734362,25.136145179539948,11.42081966062867,217.22824016750891,97.6278626859339,4.641112795545814,162.61403696376797,3.2166577319321257,5.143442067057801,2.6425900548485233,11.00268985383845,430.58518629173466,0.046201812762040446,1.9853577192078395,261.8768230625204,0.017483920193319114,13.879178870387758,0.01238603617527179,62.912474806224225,0.01625588985728355,89.93135183339443,11.756950007831975,4.3188908694754105,10.388956712485863,4.00601448166063,1.3679932953461098,0.3128763878147804,130.47903983498458,6.901249527740472,9.80331472977463,120.67572510520993,0.7901245563584364,6.1111249713820355,10.593439286133068,126.78685007659197,8.07111335225377,31.552050440000002 +GR,0.5656086758341576,21.217682502692107,8.293810066447499,3.7413645529326613,6.278738103001239,4.769683859069138,43.76514999884091,17.446059785604763,0.48277777777777753,37.40069475869229,5.125290380611692,2.6778373331934873,3.162988904204836,2.5497865662833865,3.05774002743643,1.940386753884961,21.204518154450987,17.03005713656159,0.0,12.844248401137325,2.309274089698663,3.4548066758354046,0.9812352669719969,6.745316032506065,57.6786974096937,7.609582935570777e-07,3.1869261083333336,29.7721921076304,0.0,5.078601781049937,0.0,7.226214055206279,0.0,12.414763357473749,0.4563904473396791,0.24500010394974975,0.413371507694714,0.23811802896220835,0.04301893964496518,0.006882074987541386,9.810799671681195,0.0797370737941638,1.3944121003233676,8.416387571357827,0.05929988423846293,0.02043718955570087,1.4537119845618307,8.436824760913527,5.0180753923393295, +HR,0.1580787880948285,18.34128786679964,4.389380056414181,1.0090057071271274,2.5476989978731437,1.318689059616855,28.847515888791193,6.237006781452905,1.6066742865068224,26.00131795404627,3.113211034470075,1.0345479474956152,0.9497163154353965,0.48100473034340274,1.0016020047136875,0.6411008353971066,8.272536625407751,5.38680865605112,0.4341687257624747,5.496189318262362,0.052814361197001926,1.8898328107958695,0.6988726986738213,2.6415198706666922,21.188430385465697,0.00018962982230924554,0.09159965703986905,13.436827525272498,0.00015599173040538647,1.8326831729199144,0.0,1.2565648624838692,3.363809190385904e-05,4.570755167749542,0.49776885579754093,0.21305012359963119,0.3313779723176993,0.15395303355572673,0.16639088347984168,0.05909709004390443,1.5213591752527897,0.005307491413879789,0.12226864940430518,1.3990905258484845,0.002843300346906715,0.002464191066973074,0.1251119497512119,1.4015547169154576,0.4627721052114323,0.04553145000000001 +HU,0.34544664961757376,51.71232046242127,9.058565274870105,2.140328474053002,5.714062255524212,2.0028906881072146,72.21082944708841,10.580042271441457,6.0852699910027175,66.5433537234814,14.356335066067029,0.16475011915334667,2.671442732381201,0.01669481559397995,3.150014039363932,0.1438143642750518,27.204656665349653,7.520276095678086,1.6199979270337255,20.586618985218344,0.5989606036781462,3.545410576534174,1.7932330983579277,5.937604278570248,38.02445666587427,0.004754012401710198,0.21252554633366075,20.378331083907437,0.0013669723377535844,3.8333953670014473,0.003313731799269432,4.637708110268266,7.330826468718214e-05,8.962496558363458,1.7569522475334471,1.1969520519059795,1.2917234069105592,0.9310232589824952,0.4652288406228879,0.2659287929234843,1.8353907709400215,0.054606555289952,0.0,1.8353907709400215,0.03396721087675474,0.020639344413197255,0.03396721087675474,1.8560301153532188,0.05833335371119447,0.0 +IE,1.1552139973989726,19.793109005806674,6.515397123991764,1.161434517801231,1.8421621718709775,0.8957692904350855,32.59953066332596,7.947985647128668,0.0,28.169618920079486,6.393256702138997,0.28535728735816485,1.6984446799638506,0.1477063007761994,1.856921735238021,0.6416420635559278,15.16447753566351,6.3577829109494655,0.0,10.2117126360729,0.47032508536436934,1.3904538819426195,0.862566607567254,2.723345574874243,40.68108342426441,0.008272205986726226,0.04426990840125408,24.407852018798376,0.007543433011813816,2.192367837492994,0.00016737978187553794,5.312891952707581,0.0005613931930368736,8.723701706864198,0.5233030297833083,0.04194485894551323,0.5099352244962404,0.04194485894551323,0.013367805287068027,0.0,7.3776188484574154,0.18094217829083742,0.018085728645439496,7.359533119811976,0.06395847585112205,0.11698370243971537,0.08204420449656154,7.476516822251692,0.2341743960240419,1.6970496000000002 +IT,1.0468206851697905,244.9130987767243,64.70816273756871,3.5257726860039496,33.711739880385224,5.298056156106979,398.1019762913365,66.98298367503608,10.821944449588157,350.87847290630515,79.03400365830159,10.16304834908569,19.572715587485437,6.461518895607246,23.010958076787716,7.348694836331663,185.82190964755122,88.9193830223242,2.2208333343889035,131.777336675836,4.840172338839531,14.844562085255594,12.704831814002475,32.38956623809759,382.16562754905397,0.12857527719351874,13.026200965629071,228.24478422974195,0.04308195860999806,25.177330023583888,0.06296249137904908,56.86449956932279,0.0016822306906727807,58.8528127607762,10.728613644333404,10.27889174413704,9.369662030497414,8.968468758742143,1.3589516138359898,1.3104229853948974,41.63152311753265,1.21375688128503,11.652418521864087,29.97910459566857,0.22427118935269547,0.9894856919323345,11.876689711216782,30.968590287600907,11.450287803031888,25.3443286 +LT,0.054268380164312624,11.715350602163216,2.5431037955938787,0.11347525483697869,1.2898906463385191,0.729637351593262,17.15320056639655,2.5910416440334036,5.404240349484153,15.549737784710798,3.6858966350176123,0.35403649213535254,0.646954859642566,0.2813772376756209,0.6491284395948164,0.5147296157717044,6.910102062603848,3.0984102546581362,1.9902252700580714,5.0040026932093715,0.14678373132394648,0.8607001311237008,0.3067145284231102,1.3141983908707575,16.47794344391527,0.07138098387942092,0.04742286954576074,11.759333998588998,2.367015705875897e-05,0.5515959910278992,0.06929945666578705,0.6262110477409603,0.0,3.49337953701165,0.6408426868412563,0.010000099960345853,0.11889710881126903,0.010000099960345853,0.5219455780299873,0.0,0.8025997207548139,0.012700327855460105,0.0,0.8025997207548139,0.007917059144158009,0.0047832687113020965,0.007917059144158009,0.807382989466116,0.05972313763724503,1.0061694500000002 +LU,0.09102207030796113,4.4582528460800175,0.5408506509033512,0.028752011719132938,0.1842589136749026,0.04414639513556391,5.784651145195065,0.9800439009093757,0.0,5.200337291213116,2.7416121696376643,0.42964741234496207,0.38754337834907376,0.13872110809105936,0.48879748143703383,0.21746057846312994,4.799602678258059,1.9924408503041833,0.7100000000000004,3.7584075376545165,0.03410552775164625,0.18906254463135402,0.05933199882242604,0.2825000712054263,25.012907547031013,0.006537582503034108,0.026090732142124796,9.61195035452163,0.0032742520029449287,0.34688378849703594,0.000672576010893665,0.4835316107788431,0.00048316442385610544,14.544451061091378,0.18787070506432504,0.12889466270463407,0.17086480068620152,0.11850807896971455,0.01700590437812352,0.010386583734919526,1.3692647775079168,2.9770867847386207,0.0,1.3692647775079168,0.14195883953447136,2.835127945204149,0.14195883953447136,4.204392722712066,0.0,0.0 +LV,0.04837569074377474,10.384613116949835,2.340644963567189,0.021113172384594966,0.6724545809684214,0.32652826749923863,14.743124497004171,1.7869997346257898,4.284217511442863,13.398134099393324,4.150995102858243,0.421817413562573,0.6219309895211804,0.18546538917869623,0.6530449380559203,0.4981204820144212,7.015731142346706,2.723629714011616,1.544950201464294,5.448903057406379,0.12993097414003488,1.024067185252895,0.6283022133780052,1.782300372770935,9.923692225555435,0.04430668304451984,0.026413105447034027,5.546359480279278,6.36983085949211e-05,0.853156946404734,0.04422967733254512,0.7124780217945105,1.0080120181086826e-05,2.785284671629878,0.8872377538316613,0.03889163480458043,0.12718689197385732,0.038891634804580424,0.7600508618578039,0.0,1.423451394923915,0.01682638285385829,0.0,1.423451394923915,0.011383099264030286,0.005443283589828004,0.011383099264030286,1.428894678513743,0.09444732504491289,2.7937818600000006 +ME,0.18428691762019353,1.6699110387853566,0.4476280894929783,0.1555359212866029,0.2968058048131801,0.20058433735409437,3.13138913,1.2769972600000001,,,0.08420605652273316,0.011619785550127245,0.01504797135378126,0.005198335075684077,0.01621943636276206,0.009143699254493983,0.15980783,0.07100115000000001,,,,,,,1.8195018700000003,0.0,0.023374743304404807,1.0703709519894518,0.0,0.08926636792867462,0.0,0.20503188721405052,0.0,0.43145791956341834,0.031866200000000004,0.020003600000000003,0.024002520016761314,0.015367348623627292,0.007863679983238687,0.004636251376372712,0.14909856022518334,0.009650939774816675,0.0,0.14909856022518334,0.0,0.009650939774816675,0.0,0.15874950000000002,0.07118723000000002,0.0 +MK,0.4423191483796166,2.8226495861922793,0.8413145426825128,0.37331199161795947,0.6356058031547073,0.48143565708557307,6.036190970000001,3.0650050900000005,,,0.9885463470402409,0.23288296163331187,0.20672486528543757,0.10418468247672413,0.2569197585883557,0.1832574063853998,2.3475271300000005,1.4230002800000001,,,,,,,5.933567850000001,0.0,0.07622725079860408,3.4905810062720652,0.0,0.29110607664715116,0.0,0.6686283945386197,0.0,1.4070251217435599,0.05284672,0.01700306,0.03980563903509613,0.013062246330083196,0.013041080964903865,0.003940813669916804,0.11322752199957879,0.00732905800042122,0.0,0.11322752199957879,0.0,0.00732905800042122,0.0,0.12055658000000001,0.0,0.0 +NL,1.6298586976689946,84.68942813303583,21.893183941383803,1.7773949260589401,9.357113200850907,1.3562324711811131,132.6887280464084,23.45301684196818,3.7844426867074854,116.07576375448231,41.256701585982235,3.4487947887252113,8.74386555749784,3.3685105382303595,9.870204039831533,4.025718619627854,84.81143946306823,36.32155823061464,1.7366658600480913,62.05804646052415,6.7784528108968365,31.684305638933232,6.298820126624233,44.7615785764543,121.86847257103743,0.048381893076100455,1.861204605302346,65.71183538795913,0.022202016959228868,3.939807489700578,0.009819762152609613,18.451060419932258,0.0038408434784715697,31.904564668143138,2.034727691797239,1.716116333742955,1.882559071473059,1.5955246718273612,0.1521686203241799,0.12059166191559394,34.92419971151198,5.748609775635073,0.0,34.92419971151198,0.3822845604365483,5.366325215198524,0.3822845604365483,40.2905249267105,4.103892187419366,152.76315521 +NO,28.804401730577776,39.45808456243531,7.757949880399734,5.663303412691806,2.5556362108232924,1.8656144339010035,52.202185400000005,38.90400146,,,26.130029515723944,19.07492154647848,3.545729560386932,2.5883825790824604,2.547165145793553,1.8594305564292937,35.12762416,26.495954460000007,,,,,,,39.489374250000004,0.028005040000000002,0.5073113699769504,23.230687369087583,0.00599875137002298,1.9373835604102068,0.018863239609742802,4.449888763994197,0.0031430490202342173,9.364103186531059,0.7961898000000001,0.6040040500000001,0.5997125986669634,0.4640135178884206,0.19647720133303667,0.13999053211157952,9.53481903369833,0.7042329663016708,3.837156822578874,5.697662211119456,0.335431317421127,0.3688016488805438,4.172588140000001,6.06646386,14.428050070000001,4.840906090000001 +PL,0.719707263764176,162.69384493310605,37.066477418290695,0.6321409759629364,19.48056221983485,7.554412836839432,237.44274247070248,28.441898996790353,48.88889712347745,219.28587265019746,51.955260494442626,8.819077017180906,8.441123332415255,3.5593519543311287,8.505633232810988,5.38704327166558,93.81952067315392,43.08232151710042,12.791391043403525,69.7380338434871,1.3424590215351815,34.01913900064919,6.315156248520575,41.67675427070494,174.29796279485242,0.06566503664190335,1.115185972116605,96.59925173251918,0.0006594983968755774,9.455957193306968,0.06495879022996191,24.85035470838785,4.674801506585862e-05,42.27721318852181,4.043880652573237,2.8530468343039725,2.283560723689119,1.8658111396792856,1.760319928884118,0.9872356946246871,6.230789077111895,0.07698608946551269,0.5481001624237407,5.682688914688154,0.03306317767518576,0.043922911790326924,0.5811633400989265,5.726611826478481,0.03611112581642298,1.6363293700000003 +PT,0.16853103420639537,6.167189443761765,6.844749403026448,0.6192555042551876,8.657967119181887,2.513361052425902,30.644647379477153,12.31396861546159,0.011388980962399788,21.88441540460838,5.314540992166446,1.8223271167970734,2.3288462896580606,1.389450501188967,2.7089567112102446,1.677792763857291,21.132756291752937,15.70110751612303,0.2966663759252526,11.94361641655254,0.8077702887437023,1.9424746346398076,2.1680579713654864,4.918302894748996,60.07163884638256,0.005796247464804267,0.6943809466998286,37.48714626049493,0.004144788310373135,1.8392478706029092,0.0014471892405585024,8.822663675857115,0.00015068153169906648,11.228200092727777,0.4119341087857363,0.29221113468922777,0.3542766997705545,0.261975695831413,0.0576574090151818,0.03023543885781476,12.04189939469445,0.24947267240484822,1.7711568990199187,10.270742495674531,0.1107532522332049,0.13871942017164332,1.8819101512531238,10.409461915846174,1.4175055568193127,7.483056010000001 +RO,0.8164119937824674,59.96132511343599,12.535594689161648,0.5966601803368272,8.287391484240173,0.6268225706774041,90.0416838937321,11.895960635306784,10.526118400572317,81.04442001456628,11.472370482472948,0.31126932980776745,2.499852646666194,0.43035561975633935,2.7719690913846016,1.1178608185175691,22.759444431153444,8.001025371554253,2.5952795750395103,16.898643459088195,0.7007809689311082,2.719627160661053,2.094159726588078,5.514567856180239,56.27724851083044,0.006095761615848004,0.21744941693675585,30.680317400032756,0.005639041540646845,11.646944255137056,0.00015561928868615372,7.116715973219791,4.505132889433588e-05,6.615821465504074,2.9624978308219267,1.0611074233348035,2.1230366871828705,0.7659474400326133,0.839461143639056,0.2951599833021901,2.424579253795485,0.029546566493058167,0.20490687455205012,2.2196723792434345,0.02497642466045021,0.004570141832607959,0.2298832992125003,2.2242425210760426,0.48333372495510224,0.48328465000000004 +RS,2.041447300754024,17.23241025134951,4.7096888757710875,1.7229567393124234,3.2058768205282546,2.221982761190753,33.10772576,14.145999310000002,,,4.436825941714843,0.8017518796993448,0.8519426499259911,0.3586791598052431,0.985253156817818,0.6309047643840355,9.346379720000002,4.898997940000001,,,,,,,22.236583260000003,0.0,0.285668530623453,13.081268662284423,0.0,1.0909464043385495,0.0,2.5057455043946524,0.0,5.272954158358922,0.5751616500000001,0.47800463000000004,0.43322796621493825,0.3672170905695961,0.14193368378506177,0.11078753943040395,0.4886500505665789,0.03162962943342115,0.0,0.4886500505665789,0.0,0.03162962943342115,0.0,0.52027968,0.10861257000000002,0.10931037 +SE,18.4203419908313,55.67334748926343,14.09700520336014,7.62928350560303,5.841710107199814,5.785689042843419,86.98471532467524,43.504196153945145,30.748061957160978,75.74261755247736,22.700405098892183,4.608878048490316,4.269153420938776,1.7706726927590173,4.374008646562861,4.286688107484913,46.05946171198871,25.491266437692087,15.6636143722038,32.15077398576433,1.0361667627692726,2.0898260280447953,1.010335904191686,4.136328695005755,83.04667870338166,0.01280258345751495,0.34320531162223467,51.05330259569601,0.0061524965671318635,6.1507365387542245,0.001514571605766908,7.009050732526921,0.002657098759479214,18.490383524782263,2.761947422261534,2.750002981435833,2.0247546969142176,2.0184609369237854,0.7371927253473161,0.7315420445120471,10.241309021150325,0.18676454919898827,1.9390076232799187,8.302301397870405,0.06947218543730693,0.11729236376168135,2.0084798087172255,8.419593761632086,0.7977769681471846,18.578494690000003 +SI,0.34227433442339894,8.376475064050604,2.9220707579747884,0.453818512537254,0.8699951748952809,0.5648060322082612,13.995062824999088,3.2289764898849858,0.9472293430810873,12.239285152226612,2.570621989760942,0.6007517589071416,0.5133561218151605,0.18953808270509473,0.647701991765941,0.48074261627130094,5.65765380957795,3.2179778886185555,0.5774928791411378,3.880746457492038,0.0,0.549946125291897,0.2641860340673345,0.8141321593592316,20.80700782789383,0.00010110043502486967,0.09327831893642177,11.880939237809546,8.371504104872192e-05,0.7193026858478071,0.0,0.9960279918658297,0.0,7.117459593434226,0.27222489410492606,0.15389153928786237,0.14979465172003809,0.07389407828151527,0.12243024238488796,0.0799974610063471,0.3099300394503224,0.004512821170390541,0.0,0.3099300394503224,0.0028361068079113625,0.0016767143624791794,0.0028361068079113625,0.3116067538128016,0.0,0.7076273500000001 +SK,0.7110244219256427,16.322019247760963,3.9420025260381517,0.4761557307462169,1.9667320357371187,0.7020969879798182,24.976139305348433,4.930015666329864,5.806117943917119,22.24202848390687,12.395946919792488,1.59350805554285,1.5814611823385478,0.7318096909411619,1.591685689729668,0.771687906762327,19.888551870977174,7.539329752327933,1.1225013209917258,15.78104264329495,0.21327589995852728,0.8694320240429981,0.44148890889017306,1.5241968328916984,23.792769864038295,0.03763800097049386,0.03401005608446167,9.914745300376103,6.220464867834378e-05,1.7151976456935867,0.0375611283634322,2.3748436290651527,1.0622037973027887e-05,9.753973232818993,0.5388822140190512,0.5388822140190512,0.37230773469849826,0.37230773469849826,0.166574479320553,0.166574479320553,0.46763946179091403,0.01347991726152616,0.0,0.46763946179091403,0.010194585747323313,0.003285331514202844,0.010194585747323313,0.4709247933051169,0.18577762000000003,0.0 From ad9da6118757ccfc44dc92cac00b7748fe7f383a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:34:15 +0000 Subject: [PATCH 123/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/configtables/load.csv | 2 +- scripts/build_hydro_profile.py | 4 +--- scripts/build_population_weighted_energy_totals.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/doc/configtables/load.csv b/doc/configtables/load.csv index b245268f..02e7b97b 100644 --- a/doc/configtables/load.csv +++ b/doc/configtables/load.csv @@ -4,4 +4,4 @@ time_shift_for_large_gaps,string,string,"Periods which are used for copying time manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`." scaling_factor,--,float,"Global correction factor for the load time series." fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year" -supplement_missing_data_artificially,bool,"{true, false}","Whether to supplement missing data for selected time period should be supplemented by artificial data from https://zenodo.org/record/7070438/files/demand_hourly.csv." \ No newline at end of file +supplement_missing_data_artificially,bool,"{true, false}","Whether to supplement missing data for selected time period should be supplemented by artificial data from https://zenodo.org/record/7070438/files/demand_hourly.csv." diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 510bc6fa..76ffc496 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -73,9 +73,7 @@ cc = coco.CountryConverter() def get_eia_annual_hydro_generation(fn, countries, capacities=False): # in billion kWh/a = TWh/a - df = pd.read_csv( - fn, skiprows=2, index_col=1, na_values=[" ", "--"] - ).iloc[1:, 1:] + df = pd.read_csv(fn, skiprows=2, index_col=1, na_values=[" ", "--"]).iloc[1:, 1:] df.index = df.index.str.strip() df.columns = df.columns.astype(int) diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index c2be19a9..70105a06 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -15,7 +15,7 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_population_weighted_energy_totals", - kind='energy', + kind="energy", simpl="", clusters=60, ) From 0d6ba9fbd1d998e73b7a82f8941f516c9e18f49a Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 15:15:56 +0100 Subject: [PATCH 124/173] move leap year handling into helper function --- doc/configtables/load.csv | 2 +- rules/build_electricity.smk | 2 ++ scripts/_helpers.py | 12 ++++++++++ scripts/add_electricity.py | 22 +++++++------------ scripts/base_network.py | 6 ++--- scripts/build_daily_heat_demand.py | 13 ++++++----- scripts/build_electricity_demand.py | 6 +++-- scripts/build_hourly_heat_demand.py | 8 +++---- scripts/build_hydro_profile.py | 10 +++------ scripts/build_line_rating.py | 6 ++--- ...build_population_weighted_energy_totals.py | 2 +- scripts/build_renewable_profiles.py | 9 +++----- scripts/build_solar_thermal_profiles.py | 7 ++---- scripts/build_temperature_profiles.py | 7 ++---- scripts/build_transport_demand.py | 14 +++++++----- scripts/make_summary.py | 5 +++-- scripts/prepare_sector_network.py | 4 ---- 17 files changed, 66 insertions(+), 69 deletions(-) diff --git a/doc/configtables/load.csv b/doc/configtables/load.csv index b245268f..02e7b97b 100644 --- a/doc/configtables/load.csv +++ b/doc/configtables/load.csv @@ -4,4 +4,4 @@ time_shift_for_large_gaps,string,string,"Periods which are used for copying time manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`." scaling_factor,--,float,"Global correction factor for the load time series." fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year" -supplement_missing_data_artificially,bool,"{true, false}","Whether to supplement missing data for selected time period should be supplemented by artificial data from https://zenodo.org/record/7070438/files/demand_hourly.csv." \ No newline at end of file +supplement_missing_data_artificially,bool,"{true, false}","Whether to supplement missing data for selected time period should be supplemented by artificial data from https://zenodo.org/record/7070438/files/demand_hourly.csv." diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index edf5fca3..6db8bcd7 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -21,6 +21,7 @@ if config["enable"].get("prepare_links_p_nom", False): rule build_electricity_demand: params: snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), countries=config_provider("countries"), load=config_provider("load"), input: @@ -68,6 +69,7 @@ rule base_network: params: countries=config_provider("countries"), snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), lines=config_provider("lines"), links=config_provider("links"), transformers=config_provider("transformers"), diff --git a/scripts/_helpers.py b/scripts/_helpers.py index a1504c3c..09687a11 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -724,3 +724,15 @@ def validate_checksum(file_path, zenodo_url=None, checksum=None): assert ( calculated_checksum == checksum ), "Checksum is invalid. This may be due to an incomplete download. Delete the file and re-execute the rule." + + +def get_snapshots(snapshots, drop_leap_day=False, freq="h", **kwargs): + """ + Returns pandas DateTimeIndex potentially without leap days. + """ + + time = pd.date_range(freq=freq, **snapshots, **kwargs) + if drop_leap_day and time.is_leap_year.any(): + time = time[~((time.month == 2) & (time.day == 29))] + + return time diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index a0d41e1d..ac73cb1f 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -93,7 +93,12 @@ import powerplantmatching as pm import pypsa import scipy.sparse as sparse import xarray as xr -from _helpers import configure_logging, set_scenario_config, update_p_nom_max +from _helpers import ( + configure_logging, + get_snapshots, + set_scenario_config, + update_p_nom_max, +) from powerplantmatching.export import map_country_bus from shapely.prepared import prep @@ -760,15 +765,6 @@ def estimate_renewable_capacities( ) -def drop_leap_day(n): - if not n.snapshots.is_leap_year.any(): - return - leap_days = (n.snapshots.day == 29) & (n.snapshots.month == 2) - n.set_snapshots(n.snapshots[~leap_days]) - n.snapshot_weightings[:] = 8760 / len(n.snapshots) - logger.info("Dropped February 29 from leap year.") - - def attach_line_rating( n, rating, s_max_pu, correction_factor, max_voltage_difference, max_line_rating ): @@ -805,7 +801,8 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.base_network) - n.set_snapshots(pd.date_range(freq="h", **snakemake.params.snapshots)) + time = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) + n.set_snapshots(time) Nyears = n.snapshot_weightings.objective.sum() / 8760.0 @@ -916,8 +913,5 @@ if __name__ == "__main__": sanitize_carriers(n, snakemake.config) - if snakemake.params.drop_leap_day: - drop_leap_day(n) - n.meta = snakemake.config n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/base_network.py b/scripts/base_network.py index 66438994..346f99a5 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -75,7 +75,7 @@ import shapely import shapely.prepared import shapely.wkt import yaml -from _helpers import configure_logging, set_scenario_config +from _helpers import configure_logging, get_snapshots, set_scenario_config from packaging.version import Version, parse from scipy import spatial from scipy.sparse import csgraph @@ -730,12 +730,12 @@ def base_network( transformers = _set_electrical_parameters_transformers(transformers, config) links = _set_electrical_parameters_links(links, config, links_p_nom) converters = _set_electrical_parameters_converters(converters, config) - snapshots = snakemake.params.snapshots n = pypsa.Network() n.name = "PyPSA-Eur" - n.set_snapshots(pd.date_range(freq="h", **snapshots)) + time = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) + n.set_snapshots(time) n.madd("Carrier", ["AC", "DC"]) n.import_components_from_dataframe(buses, "Bus") diff --git a/scripts/build_daily_heat_demand.py b/scripts/build_daily_heat_demand.py index c08cfe33..54c5c386 100644 --- a/scripts/build_daily_heat_demand.py +++ b/scripts/build_daily_heat_demand.py @@ -11,7 +11,7 @@ import geopandas as gpd import numpy as np import pandas as pd import xarray as xr -from _helpers import set_scenario_config +from _helpers import get_snapshots, set_scenario_config from dask.distributed import Client, LocalCluster if __name__ == "__main__": @@ -32,11 +32,12 @@ if __name__ == "__main__": cutout_name = snakemake.input.cutout - time = pd.date_range(freq="h", **snakemake.params.snapshots) - daily = pd.date_range(freq="D", **snakemake.params.snapshots) - if snakemake.params.drop_leap_day: - time = time[~((time.month == 2) & (time.day == 29))] - daily = daily[~((daily.month == 2) & (daily.day == 29))] + time = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) + daily = get_snapshots( + snakemake.params.snapshots, + snakemake.params.drop_leap_day, + freq="D", + ) cutout = atlite.Cutout(cutout_name).sel(time=time) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index 68df4bac..2d3da224 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -39,7 +39,7 @@ import logging import numpy as np import pandas as pd -from _helpers import configure_logging, set_scenario_config +from _helpers import configure_logging, get_snapshots, set_scenario_config from pandas import Timedelta as Delta logger = logging.getLogger(__name__) @@ -263,7 +263,9 @@ if __name__ == "__main__": configure_logging(snakemake) set_scenario_config(snakemake) - snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) + snapshots = get_snapshots( + snakemake.params.snapshots, snakemake.params.drop_leap_day + ) fixed_year = snakemake.config["load"].get("fixed_year", False) years = ( diff --git a/scripts/build_hourly_heat_demand.py b/scripts/build_hourly_heat_demand.py index c3916b54..1fb4f5a4 100644 --- a/scripts/build_hourly_heat_demand.py +++ b/scripts/build_hourly_heat_demand.py @@ -10,7 +10,7 @@ from itertools import product import pandas as pd import xarray as xr -from _helpers import generate_periodic_profiles, set_scenario_config +from _helpers import generate_periodic_profiles, get_snapshots, set_scenario_config if __name__ == "__main__": if "snakemake" not in globals(): @@ -24,9 +24,9 @@ if __name__ == "__main__": ) set_scenario_config(snakemake) - snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) - if snakemake.params.drop_leap_day: - snapshots = snapshots[~((snapshots.month == 2) & (snapshots.day == 29))] + snapshots = get_snapshots( + snakemake.params.snapshots, snakemake.params.drop_leap_day + ) daily_space_heat_demand = ( xr.open_dataarray(snakemake.input.heat_demand) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 510bc6fa..b7f270b3 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -65,7 +65,7 @@ import atlite import country_converter as coco import geopandas as gpd import pandas as pd -from _helpers import configure_logging, set_scenario_config +from _helpers import configure_logging, get_snapshots, set_scenario_config from numpy.polynomial import Polynomial cc = coco.CountryConverter() @@ -73,9 +73,7 @@ cc = coco.CountryConverter() def get_eia_annual_hydro_generation(fn, countries, capacities=False): # in billion kWh/a = TWh/a - df = pd.read_csv( - fn, skiprows=2, index_col=1, na_values=[" ", "--"] - ).iloc[1:, 1:] + df = pd.read_csv(fn, skiprows=2, index_col=1, na_values=[" ", "--"]).iloc[1:, 1:] df.index = df.index.str.strip() df.columns = df.columns.astype(int) @@ -175,9 +173,7 @@ if __name__ == "__main__": params_hydro = snakemake.params.hydro - time = pd.date_range(freq="h", **snakemake.params.snapshots) - if snakemake.params.drop_leap_day: - time = time[~((time.month == 2) & (time.day == 29))] + time = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py index 794638d2..f9c71ea3 100755 --- a/scripts/build_line_rating.py +++ b/scripts/build_line_rating.py @@ -58,7 +58,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import configure_logging, set_scenario_config +from _helpers import configure_logging, get_snapshots, set_scenario_config from shapely.geometry import LineString as Line from shapely.geometry import Point @@ -147,9 +147,7 @@ if __name__ == "__main__": set_scenario_config(snakemake) n = pypsa.Network(snakemake.input.base_network) - time = pd.date_range(freq="h", **snakemake.params.snapshots) - if snakemake.params.drop_leap_day: - time = time[~((time.month == 2) & (time.day == 29))] + time = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index c2be19a9..70105a06 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -15,7 +15,7 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_population_weighted_energy_totals", - kind='energy', + kind="energy", simpl="", clusters=60, ) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 88057fb6..f1eb5e15 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -186,9 +186,8 @@ import time import atlite import geopandas as gpd import numpy as np -import pandas as pd import xarray as xr -from _helpers import configure_logging, set_scenario_config +from _helpers import configure_logging, get_snapshots, set_scenario_config from dask.distributed import Client from pypsa.geo import haversine from shapely.geometry import LineString @@ -227,11 +226,9 @@ if __name__ == "__main__": else: client = None - time = pd.date_range(freq="h", **snakemake.params.snapshots) - if snakemake.params.drop_leap_day: - time = time[~((time.month == 2) & (time.day == 29))] + sns = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) - cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) + cutout = atlite.Cutout(snakemake.input.cutout).sel(time=sns) regions = gpd.read_file(snakemake.input.regions) assert not regions.empty, ( f"List of regions in {snakemake.input.regions} is empty, please " diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index c9eaffcc..bb5180b9 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -9,9 +9,8 @@ Build solar thermal collector time series. import atlite import geopandas as gpd import numpy as np -import pandas as pd import xarray as xr -from _helpers import set_scenario_config +from _helpers import get_snapshots, set_scenario_config from dask.distributed import Client, LocalCluster if __name__ == "__main__": @@ -32,9 +31,7 @@ if __name__ == "__main__": config = snakemake.params.solar_thermal config.pop("cutout", None) - time = pd.date_range(freq="h", **snakemake.params.snapshots) - if snakemake.params.drop_leap_day: - time = time[~((time.month == 2) & (time.day == 29))] + time = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index e5fa0b38..00c88b5b 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -9,9 +9,8 @@ Build time series for air and soil temperatures per clustered model region. import atlite import geopandas as gpd import numpy as np -import pandas as pd import xarray as xr -from _helpers import set_scenario_config +from _helpers import get_snapshots, set_scenario_config from dask.distributed import Client, LocalCluster if __name__ == "__main__": @@ -29,9 +28,7 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - time = pd.date_range(freq="h", **snakemake.params.snapshots) - if snakemake.params.drop_leap_day: - time = time[~((time.month == 2) & (time.day == 29))] + time = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index d3c740be..e39774fa 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -13,7 +13,12 @@ import logging import numpy as np import pandas as pd import xarray as xr -from _helpers import configure_logging, generate_periodic_profiles, set_scenario_config +from _helpers import ( + configure_logging, + generate_periodic_profiles, + get_snapshots, + set_scenario_config, +) logger = logging.getLogger(__name__) @@ -183,10 +188,9 @@ if __name__ == "__main__": options = snakemake.params.sector - snapshots = pd.date_range(freq="h", **snakemake.params.snapshots, tz="UTC") - if snakemake.params.drop_leap_day: - leap_day = (snapshots.month == 2) & (snapshots.day == 29) - snapshots = snapshots[~leap_day] + snapshots = get_snapshots( + snakemake.params.snapshots, snakemake.params.drop_leap_day, tz="UTC" + ) nyears = len(snapshots) / 8760 diff --git a/scripts/make_summary.py b/scripts/make_summary.py index b9e1a8af..8c2a1aea 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -13,7 +13,7 @@ import sys import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging, set_scenario_config +from _helpers import configure_logging, get_snapshots, set_scenario_config from prepare_sector_network import prepare_costs idx = pd.IndexSlice @@ -690,7 +690,8 @@ if __name__ == "__main__": for planning_horizon in snakemake.params.scenario["planning_horizons"] } - Nyears = len(pd.date_range(freq="h", **snakemake.params.snapshots)) / 8760 + time = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) + Nyears = len(time) / 8760 costs_db = prepare_costs( snakemake.input.costs, diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 9c4404f2..5757a88b 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3507,10 +3507,6 @@ def set_temporal_aggregation(n, resolution, solver_name, drop_leap_day=False): logger.info("Aggregate to frequency %s", resolution) n = average_every_nhours(n, resolution) - if drop_leap_day: - sns = n.snapshots[~((n.snapshots.month == 2) & (n.snapshots.day == 29))] - n.set_snapshots(sns) - return n From 6a3cf1245970996ae42ee0153613cc67ce30cb8b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 15:18:05 +0100 Subject: [PATCH 125/173] remove accidental file addition --- scripts/new-energy-totals.csv | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 scripts/new-energy-totals.csv diff --git a/scripts/new-energy-totals.csv b/scripts/new-energy-totals.csv deleted file mode 100644 index 78ba6736..00000000 --- a/scripts/new-energy-totals.csv +++ /dev/null @@ -1,34 +0,0 @@ -country,electricity residential space,total residential space,total residential water,electricity residential water,total residential cooking,electricity residential cooking,total residential,electricity residential,derived heat residential,thermal uses residential,total services space,electricity services space,total services water,electricity services water,total services cooking,electricity services cooking,total services,electricity services,derived heat services,thermal uses services,total agriculture electricity,total agriculture heat,total agriculture machinery,total agriculture,total road,electricity road,total two-wheel,total passenger cars,electricity passenger cars,total other road passenger,electricity other road passenger,total light duty road freight,electricity light duty road freight,total heavy duty road freight,total rail,electricity rail,total rail passenger,electricity rail passenger,total rail freight,electricity rail freight,total aviation passenger,total aviation freight,total domestic aviation passenger,total international aviation passenger,total domestic aviation freight,total international aviation freight,total domestic aviation,total international aviation,total domestic navigation,total international navigation -AL,0.5570465743624181,2.868681023831782,0.9246366122728509,0.4701405463928116,0.7560298250278052,0.606309006828873,6.745434890000001,3.8599970000000003,,,0.8492947400612103,0.24188377504306646,0.19063435423096936,0.1082113698760542,0.24955086644902696,0.190340216176473,2.2211439200000003,1.4779985500000001,,,,,,,8.933037890000001,0.0,0.11476078758153294,5.255099996394696,0.0,0.43826272462667554,0.0,1.0066258503715193,0.0,2.1182885310255757,0.03566921000000001,0.0,0.026867054339929548,0.0,0.008802155660070457,0.0,0.23558664812723476,0.015249191872765277,0.0,0.23558664812723476,0.0,0.015249191872765277,0.0,0.25083584000000003,0.38266189000000006,0.23777535000000002 -AT,3.8347980364339187,52.728795606720304,9.042762942245817,1.6580654856850694,3.332605848035772,2.2140159394525,74.4128059879979,17.31873449075142,8.985794565462129,65.16773835369318,17.423168106040677,1.362361756513761,3.4703813442929237,0.2967617406747894,3.4964403032216045,2.3458829314999434,33.49193062767192,13.244905252570849,9.882817057376736,24.839638062603143,0.6497150410629878,3.8719030223442585,1.7388785775882416,6.260496640995489,88.04632651098169,0.035087453918020005,0.6026064240302123,47.145477443307264,0.010621583280449667,3.067508471104808,0.02013826679946008,5.385434496257479,0.003947974868709491,31.84529967628193,3.384639661479662,2.8930267787407162,2.332832737694036,1.971964325107237,1.0518069237856262,0.9210624536334792,7.604475261435951,0.3130344308815537,0.26550445600104483,7.338970805434905,0.07206205482497326,0.24097237605658042,0.3375665108260181,7.579943181491485,0.058055683482739553,0.23600759000000002 -BA,0.9551423259051094,8.062615381913417,2.2035460750967655,0.8061285279384439,1.4999498844734496,1.039610369496632,15.49027994803239,6.61856060561295,,,2.075880307094079,0.37511972750507755,0.3986027383060623,0.16781704177400622,0.46097540282364163,0.2951846191943365,4.372938190104563,2.2921190692934506,,,,,,,10.403943245213432,0.0,0.1336571875633523,6.120399665114565,0.0,0.5104266398122862,0.0,1.172375886612297,0.0,2.4670838661109293,0.2691038049081697,0.22364645608190642,0.20269657078330564,0.1718117268416388,0.06640723412486403,0.051834729240267635,0.22862718311632196,0.01479871550610001,0.0,0.22862718311632196,0.0,0.01479871550610001,0.0,0.24342589862242198,0.05081711523682938,0.0511435984699603 -BE,2.815051564451901,72.00315684834005,14.758606504598777,1.9245397162358575,5.61762658049057,2.123765047409,104.53126486079485,19.81013300307489,0.048333322074790706,92.47214768445166,35.55025666623682,6.159088599308961,4.553234776118736,1.3005782934779981,4.296286602056132,1.7511206036264235,57.027872149578414,22.25434577090179,0.7699977716616743,45.32121766089195,1.2652057097802312,6.510654609861979,2.1504989713552063,9.926359290997418,93.22107248215386,0.01074409225014649,0.6477429921079886,54.179087820705185,0.006636212751625691,3.954966803176898,0.001991249371656604,10.47276574878302,0.0015859584461520306,23.966509117380756,2.2359228430787947,1.596953398545387,1.7366935264699335,1.306970119159899,0.4992293166088615,0.2899832793854877,11.903404936264774,2.9443700871166114,0.0,11.903404936264774,0.5794491728255937,2.3649209142910177,0.5794491728255937,14.268325850555792,1.822780948768488,71.59805975 -BG,1.0366899653005102,13.774445984268725,5.032599359348841,2.50096431277225,2.8263907739201346,2.446346362124563,26.126347522623533,10.536998125106878,3.70225381151711,21.738366989691684,3.6161635069149147,1.317290338927257,1.4876301353881975,0.8459255010716765,1.614166101394505,1.2802729496644545,11.214107236902933,7.954580847599141,1.1947324162166162,7.013424737247175,0.2157562658354375,1.238388279337861,0.8037223126895519,2.257866857862851,27.913508544418903,0.05240401618919764,0.2678003482560851,18.16930433603284,0.0,2.6753054767672606,0.05240401618919764,1.723251062925399,0.0,5.077847320437322,0.36999955326892575,0.19499962550233102,0.2595248907228583,0.1337123426290272,0.11047466254606741,0.06128728287330383,1.9735489165680373,0.0214510834319626,0.06685311111111157,1.9066958054569259,0.01455716249528969,0.0068939209366729115,0.08141027360640127,1.9135897263935986,0.0,1.0494446800000001 -CH,4.388888888888889,51.52777777777778,8.944444444444445,2.388888888888889,2.611111111111111,1.3611111111111112,72.58333333333333,18.777777777777775,,,21.5,1.1666666666666667,2.8055555555555554,0.19444444444444442,0.6388888888888888,0.6388888888888888,40.0,17.055555555555554,,,,,,,55.888888888888886,0.0,0.7179923543283743,32.87814329403942,0.0,2.7419582254567123,0.0,6.297879959411784,0.0,13.25291505565259,3.1666666666666665,3.1666666666666665,2.3852225886072858,2.432725641458858,0.7814440780593807,0.7339410252078085,17.617703597787695,1.1600741799900811,0.8685212443581152,16.749182353429582,0.07592320008632922,1.0841509799037519,0.9444444444444444,17.833333333333332,0.4444444444444445,0.0 -CZ,2.5725233632931075,62.57273499598758,10.288206369245467,2.9111285854172344,5.349291874222479,2.9169812116183698,83.99741666322201,14.715973413943496,13.888879856159699,78.24427707959727,20.246784369915833,4.388583048588386,3.0235451499346557,0.9434718391306539,3.505353453229548,1.7737748387302745,34.29840057384516,14.78664045488378,5.310847457419239,27.053057440415177,0.6941273932306512,4.054135056204236,2.4003802728676513,7.148642722302538,63.503988188281525,0.08911029892052649,0.6865202837197137,30.92628881993422,0.0007819669138069432,3.866800834894914,0.08831777854219773,5.7476679804286555,0.0,22.276710269304015,2.5098763617700213,1.491101348077621,1.957212181581117,1.1043425069361408,0.5526641801889038,0.38675884114148024,3.4758788630067836,0.048572629407252134,0.056765346606964515,3.419113516399819,0.019945792240296925,0.028626837166955205,0.07671113884726144,3.447740353566774,0.023888852908381327,0.0 -DE,23.552538648485736,482.88953556791756,88.86181482888813,9.341725122373857,41.094774831600795,14.516493828941883,694.2861367421355,136.00002715231648,51.24612134236524,614.0643012925772,200.1083899771467,8.177528100639465,38.567939444757855,8.148137272139993,48.02923585230634,20.88733594674748,388.35703665770825,141.5218730593723,16.87389225775031,291.39267527735336,7.53469558411125,5.474391021838289,1.4190403036639783,14.42812690961352,600.5438784206764,0.11224182554399796,5.370903791988889,380.78585774165197,0.0804737905883431,18.892615204428267,0.014714641214532026,28.660484589683342,0.015353321394055243,166.83401709292394,15.93416884756733,11.885002372832927,11.472824304223883,8.279888081535278,4.461344543343446,3.605114291297649,91.38069880171732,13.41045647623245,7.896669996323121,83.4840288053942,2.19924733222008,11.211209144012372,10.095917328543202,94.69523794940658,3.3722220527550077,26.697188350000005 -DK,0.7013376012077638,33.93012445800281,7.368540509780002,0.5229211600849933,3.1224115802390884,2.437647648641987,50.639691342206994,10.307073119763176,19.141944444444412,44.46958545008379,11.568860496650082,0.7165794765906534,1.9268567416141313,0.4139126319754398,2.107350856421504,1.6568129470533832,22.96597151274313,10.221030677425023,8.886388888888847,15.964433033531657,1.6065828331780032,4.6006732485303825,2.958335976083448,9.165592057791836,42.126010180586306,0.006041443591563428,0.26907068768936326,23.38606046077933,0.005426936489976022,3.9158027200781587,0.0,5.764536143667914,0.0006145071015874062,8.79054016837154,1.3112062796097734,0.3861092120959323,1.201322441895927,0.35393692637112534,0.10988383771384651,0.03217228572480684,10.062240435459456,0.3052602112787246,0.5934322985915401,9.468808136867919,0.10206631066816502,0.20319390061055956,0.6954986092597052,9.672002037478478,1.978361573681062,7.339855820000001 -EE,0.11231606763657928,7.615319503233344,1.5105449935831652,0.06137687617864419,0.7305183637351055,0.6392485833424982,10.871312066259437,1.8649831545295248,3.8389405197191633,9.856556620105813,2.8051300129906593,0.7294467788998327,0.3759461883671719,0.16897479263456983,0.43229161868604543,0.3654118725947719,4.8619944419222145,2.5288861940995093,1.1860593158990447,3.6224949714927197,0.17269894009588554,0.7062927576664825,0.4019989426098384,1.2809906403722064,8.143920002436563,0.02196621718242599,0.02187321389494755,5.041673827103431,0.0035133698706650515,0.9394047336082759,0.018423149598824944,0.6543028641167076,2.969771293599253e-05,1.4866653637132001,0.3473335032636153,0.04203126411907797,0.08800246779439479,0.04203126411907797,0.2593310354692205,0.0,0.32494722922299063,0.009489420040482168,0.0,0.32494722922299063,0.0036550739486526196,0.005834346091829548,0.0036550739486526196,0.33078157531482016,0.04694670715427129,4.740620600000001 -ES,10.794125129618664,71.26452543322868,39.68527650785515,7.710514219460869,19.26597649255715,8.828493562332296,177.196549581737,74.63098166927618,0.0,131.8776230960182,34.390226550633635,5.784912430114415,14.00729551399568,6.8886787516288885,17.98020571548081,13.01444004144452,115.94028841537886,75.5600175062191,0.0,75.67658371390513,3.798490176657542,20.504156811235546,10.361953239126692,34.664600227019776,295.6964784476745,0.008958749867787469,5.271025477457057,177.9796367179715,0.006198765274153721,10.784071123614806,0.0,20.279070030269107,0.0027599845936337485,81.38267509836203,8.270129202825443,4.551941091567911,7.174411141940354,4.18254627330064,1.095718060885089,0.36939481826726994,58.78529713592237,1.0458142160063002,18.097222294891036,40.68807484103132,0.33629527208914706,0.7095189439171532,18.433517566980182,41.39759378494848,5.86721442260112,84.04543941000001 -FI,7.848535711327018,41.84350237788999,9.048520920884483,3.443099269456141,2.9395617816882234,2.8792251386947934,60.826832141117144,21.50989869058375,18.587994476407626,53.92184520851802,21.23055041955415,6.199986054777693,2.553779457989475,0.9989674641957111,2.737139467312127,2.6900543568450637,34.04765319297288,17.50312586133877,12.247279309978593,26.808838116560455,1.2798819960077403,7.441003309894633,1.9312393542030517,10.652124660105425,45.56549617623142,0.0019365717691190252,0.5223209306867633,26.118742999170177,0.0007992579455364848,3.2601013702649593,0.00016885440537935242,5.613312604425437,0.0006476900733945322,10.051018271684086,1.1047159761512422,0.7330519889163917,0.6084665070352008,0.5419802858183081,0.4962494691160414,0.19107170309808352,7.731241694301903,0.3876090573520712,1.4438906650320935,6.287351029269809,0.11591836963879651,0.2716906877132747,1.55980903467089,6.559041716983084,1.725813385303544,1.5172963200000003 -FR,36.34821295388092,304.3090830974361,74.10444863335236,20.89120031251567,42.04728767502008,19.96798113754325,503.553879238341,167.90380216073933,14.323611111111111,421.0509096549809,149.2072481280514,29.02123818481345,25.328465674571646,12.10488257238001,28.51040521266642,17.00244127422052,285.9163571402345,142.24734181479553,8.505277777777804,216.87393093116245,7.478237257126606,32.74985084813704,20.230248718717654,60.4583368239813,477.2711666499798,0.17960126429470955,6.464992469847664,252.13117399671086,0.07795214668519337,21.88511370979885,0.039237383416606084,91.90030834156023,0.061631171267770395,104.88957813206223,12.134370941891715,10.168257793403882,10.262575388236112,8.653518950028449,1.8717955536556061,1.5147388433754336,71.22571967077627,3.5990187474665083,19.982431345628665,51.2432883251476,1.4447624710282905,2.154256276438218,21.427193816656956,53.397544601585814,5.826683163334846,26.200994400000006 -GB,32.35436698812953,318.9482399097387,79.0068899383846,11.692921151505214,20.777464730965892,3.4829778546191315,480.6012101626519,113.45010172801696,0.603890281538269,419.06137321331204,111.80304633929093,16.901263938070077,21.670713737923748,9.433929352734362,25.136145179539948,11.42081966062867,217.22824016750891,97.6278626859339,4.641112795545814,162.61403696376797,3.2166577319321257,5.143442067057801,2.6425900548485233,11.00268985383845,430.58518629173466,0.046201812762040446,1.9853577192078395,261.8768230625204,0.017483920193319114,13.879178870387758,0.01238603617527179,62.912474806224225,0.01625588985728355,89.93135183339443,11.756950007831975,4.3188908694754105,10.388956712485863,4.00601448166063,1.3679932953461098,0.3128763878147804,130.47903983498458,6.901249527740472,9.80331472977463,120.67572510520993,0.7901245563584364,6.1111249713820355,10.593439286133068,126.78685007659197,8.07111335225377,31.552050440000002 -GR,0.5656086758341576,21.217682502692107,8.293810066447499,3.7413645529326613,6.278738103001239,4.769683859069138,43.76514999884091,17.446059785604763,0.48277777777777753,37.40069475869229,5.125290380611692,2.6778373331934873,3.162988904204836,2.5497865662833865,3.05774002743643,1.940386753884961,21.204518154450987,17.03005713656159,0.0,12.844248401137325,2.309274089698663,3.4548066758354046,0.9812352669719969,6.745316032506065,57.6786974096937,7.609582935570777e-07,3.1869261083333336,29.7721921076304,0.0,5.078601781049937,0.0,7.226214055206279,0.0,12.414763357473749,0.4563904473396791,0.24500010394974975,0.413371507694714,0.23811802896220835,0.04301893964496518,0.006882074987541386,9.810799671681195,0.0797370737941638,1.3944121003233676,8.416387571357827,0.05929988423846293,0.02043718955570087,1.4537119845618307,8.436824760913527,5.0180753923393295, -HR,0.1580787880948285,18.34128786679964,4.389380056414181,1.0090057071271274,2.5476989978731437,1.318689059616855,28.847515888791193,6.237006781452905,1.6066742865068224,26.00131795404627,3.113211034470075,1.0345479474956152,0.9497163154353965,0.48100473034340274,1.0016020047136875,0.6411008353971066,8.272536625407751,5.38680865605112,0.4341687257624747,5.496189318262362,0.052814361197001926,1.8898328107958695,0.6988726986738213,2.6415198706666922,21.188430385465697,0.00018962982230924554,0.09159965703986905,13.436827525272498,0.00015599173040538647,1.8326831729199144,0.0,1.2565648624838692,3.363809190385904e-05,4.570755167749542,0.49776885579754093,0.21305012359963119,0.3313779723176993,0.15395303355572673,0.16639088347984168,0.05909709004390443,1.5213591752527897,0.005307491413879789,0.12226864940430518,1.3990905258484845,0.002843300346906715,0.002464191066973074,0.1251119497512119,1.4015547169154576,0.4627721052114323,0.04553145000000001 -HU,0.34544664961757376,51.71232046242127,9.058565274870105,2.140328474053002,5.714062255524212,2.0028906881072146,72.21082944708841,10.580042271441457,6.0852699910027175,66.5433537234814,14.356335066067029,0.16475011915334667,2.671442732381201,0.01669481559397995,3.150014039363932,0.1438143642750518,27.204656665349653,7.520276095678086,1.6199979270337255,20.586618985218344,0.5989606036781462,3.545410576534174,1.7932330983579277,5.937604278570248,38.02445666587427,0.004754012401710198,0.21252554633366075,20.378331083907437,0.0013669723377535844,3.8333953670014473,0.003313731799269432,4.637708110268266,7.330826468718214e-05,8.962496558363458,1.7569522475334471,1.1969520519059795,1.2917234069105592,0.9310232589824952,0.4652288406228879,0.2659287929234843,1.8353907709400215,0.054606555289952,0.0,1.8353907709400215,0.03396721087675474,0.020639344413197255,0.03396721087675474,1.8560301153532188,0.05833335371119447,0.0 -IE,1.1552139973989726,19.793109005806674,6.515397123991764,1.161434517801231,1.8421621718709775,0.8957692904350855,32.59953066332596,7.947985647128668,0.0,28.169618920079486,6.393256702138997,0.28535728735816485,1.6984446799638506,0.1477063007761994,1.856921735238021,0.6416420635559278,15.16447753566351,6.3577829109494655,0.0,10.2117126360729,0.47032508536436934,1.3904538819426195,0.862566607567254,2.723345574874243,40.68108342426441,0.008272205986726226,0.04426990840125408,24.407852018798376,0.007543433011813816,2.192367837492994,0.00016737978187553794,5.312891952707581,0.0005613931930368736,8.723701706864198,0.5233030297833083,0.04194485894551323,0.5099352244962404,0.04194485894551323,0.013367805287068027,0.0,7.3776188484574154,0.18094217829083742,0.018085728645439496,7.359533119811976,0.06395847585112205,0.11698370243971537,0.08204420449656154,7.476516822251692,0.2341743960240419,1.6970496000000002 -IT,1.0468206851697905,244.9130987767243,64.70816273756871,3.5257726860039496,33.711739880385224,5.298056156106979,398.1019762913365,66.98298367503608,10.821944449588157,350.87847290630515,79.03400365830159,10.16304834908569,19.572715587485437,6.461518895607246,23.010958076787716,7.348694836331663,185.82190964755122,88.9193830223242,2.2208333343889035,131.777336675836,4.840172338839531,14.844562085255594,12.704831814002475,32.38956623809759,382.16562754905397,0.12857527719351874,13.026200965629071,228.24478422974195,0.04308195860999806,25.177330023583888,0.06296249137904908,56.86449956932279,0.0016822306906727807,58.8528127607762,10.728613644333404,10.27889174413704,9.369662030497414,8.968468758742143,1.3589516138359898,1.3104229853948974,41.63152311753265,1.21375688128503,11.652418521864087,29.97910459566857,0.22427118935269547,0.9894856919323345,11.876689711216782,30.968590287600907,11.450287803031888,25.3443286 -LT,0.054268380164312624,11.715350602163216,2.5431037955938787,0.11347525483697869,1.2898906463385191,0.729637351593262,17.15320056639655,2.5910416440334036,5.404240349484153,15.549737784710798,3.6858966350176123,0.35403649213535254,0.646954859642566,0.2813772376756209,0.6491284395948164,0.5147296157717044,6.910102062603848,3.0984102546581362,1.9902252700580714,5.0040026932093715,0.14678373132394648,0.8607001311237008,0.3067145284231102,1.3141983908707575,16.47794344391527,0.07138098387942092,0.04742286954576074,11.759333998588998,2.367015705875897e-05,0.5515959910278992,0.06929945666578705,0.6262110477409603,0.0,3.49337953701165,0.6408426868412563,0.010000099960345853,0.11889710881126903,0.010000099960345853,0.5219455780299873,0.0,0.8025997207548139,0.012700327855460105,0.0,0.8025997207548139,0.007917059144158009,0.0047832687113020965,0.007917059144158009,0.807382989466116,0.05972313763724503,1.0061694500000002 -LU,0.09102207030796113,4.4582528460800175,0.5408506509033512,0.028752011719132938,0.1842589136749026,0.04414639513556391,5.784651145195065,0.9800439009093757,0.0,5.200337291213116,2.7416121696376643,0.42964741234496207,0.38754337834907376,0.13872110809105936,0.48879748143703383,0.21746057846312994,4.799602678258059,1.9924408503041833,0.7100000000000004,3.7584075376545165,0.03410552775164625,0.18906254463135402,0.05933199882242604,0.2825000712054263,25.012907547031013,0.006537582503034108,0.026090732142124796,9.61195035452163,0.0032742520029449287,0.34688378849703594,0.000672576010893665,0.4835316107788431,0.00048316442385610544,14.544451061091378,0.18787070506432504,0.12889466270463407,0.17086480068620152,0.11850807896971455,0.01700590437812352,0.010386583734919526,1.3692647775079168,2.9770867847386207,0.0,1.3692647775079168,0.14195883953447136,2.835127945204149,0.14195883953447136,4.204392722712066,0.0,0.0 -LV,0.04837569074377474,10.384613116949835,2.340644963567189,0.021113172384594966,0.6724545809684214,0.32652826749923863,14.743124497004171,1.7869997346257898,4.284217511442863,13.398134099393324,4.150995102858243,0.421817413562573,0.6219309895211804,0.18546538917869623,0.6530449380559203,0.4981204820144212,7.015731142346706,2.723629714011616,1.544950201464294,5.448903057406379,0.12993097414003488,1.024067185252895,0.6283022133780052,1.782300372770935,9.923692225555435,0.04430668304451984,0.026413105447034027,5.546359480279278,6.36983085949211e-05,0.853156946404734,0.04422967733254512,0.7124780217945105,1.0080120181086826e-05,2.785284671629878,0.8872377538316613,0.03889163480458043,0.12718689197385732,0.038891634804580424,0.7600508618578039,0.0,1.423451394923915,0.01682638285385829,0.0,1.423451394923915,0.011383099264030286,0.005443283589828004,0.011383099264030286,1.428894678513743,0.09444732504491289,2.7937818600000006 -ME,0.18428691762019353,1.6699110387853566,0.4476280894929783,0.1555359212866029,0.2968058048131801,0.20058433735409437,3.13138913,1.2769972600000001,,,0.08420605652273316,0.011619785550127245,0.01504797135378126,0.005198335075684077,0.01621943636276206,0.009143699254493983,0.15980783,0.07100115000000001,,,,,,,1.8195018700000003,0.0,0.023374743304404807,1.0703709519894518,0.0,0.08926636792867462,0.0,0.20503188721405052,0.0,0.43145791956341834,0.031866200000000004,0.020003600000000003,0.024002520016761314,0.015367348623627292,0.007863679983238687,0.004636251376372712,0.14909856022518334,0.009650939774816675,0.0,0.14909856022518334,0.0,0.009650939774816675,0.0,0.15874950000000002,0.07118723000000002,0.0 -MK,0.4423191483796166,2.8226495861922793,0.8413145426825128,0.37331199161795947,0.6356058031547073,0.48143565708557307,6.036190970000001,3.0650050900000005,,,0.9885463470402409,0.23288296163331187,0.20672486528543757,0.10418468247672413,0.2569197585883557,0.1832574063853998,2.3475271300000005,1.4230002800000001,,,,,,,5.933567850000001,0.0,0.07622725079860408,3.4905810062720652,0.0,0.29110607664715116,0.0,0.6686283945386197,0.0,1.4070251217435599,0.05284672,0.01700306,0.03980563903509613,0.013062246330083196,0.013041080964903865,0.003940813669916804,0.11322752199957879,0.00732905800042122,0.0,0.11322752199957879,0.0,0.00732905800042122,0.0,0.12055658000000001,0.0,0.0 -NL,1.6298586976689946,84.68942813303583,21.893183941383803,1.7773949260589401,9.357113200850907,1.3562324711811131,132.6887280464084,23.45301684196818,3.7844426867074854,116.07576375448231,41.256701585982235,3.4487947887252113,8.74386555749784,3.3685105382303595,9.870204039831533,4.025718619627854,84.81143946306823,36.32155823061464,1.7366658600480913,62.05804646052415,6.7784528108968365,31.684305638933232,6.298820126624233,44.7615785764543,121.86847257103743,0.048381893076100455,1.861204605302346,65.71183538795913,0.022202016959228868,3.939807489700578,0.009819762152609613,18.451060419932258,0.0038408434784715697,31.904564668143138,2.034727691797239,1.716116333742955,1.882559071473059,1.5955246718273612,0.1521686203241799,0.12059166191559394,34.92419971151198,5.748609775635073,0.0,34.92419971151198,0.3822845604365483,5.366325215198524,0.3822845604365483,40.2905249267105,4.103892187419366,152.76315521 -NO,28.804401730577776,39.45808456243531,7.757949880399734,5.663303412691806,2.5556362108232924,1.8656144339010035,52.202185400000005,38.90400146,,,26.130029515723944,19.07492154647848,3.545729560386932,2.5883825790824604,2.547165145793553,1.8594305564292937,35.12762416,26.495954460000007,,,,,,,39.489374250000004,0.028005040000000002,0.5073113699769504,23.230687369087583,0.00599875137002298,1.9373835604102068,0.018863239609742802,4.449888763994197,0.0031430490202342173,9.364103186531059,0.7961898000000001,0.6040040500000001,0.5997125986669634,0.4640135178884206,0.19647720133303667,0.13999053211157952,9.53481903369833,0.7042329663016708,3.837156822578874,5.697662211119456,0.335431317421127,0.3688016488805438,4.172588140000001,6.06646386,14.428050070000001,4.840906090000001 -PL,0.719707263764176,162.69384493310605,37.066477418290695,0.6321409759629364,19.48056221983485,7.554412836839432,237.44274247070248,28.441898996790353,48.88889712347745,219.28587265019746,51.955260494442626,8.819077017180906,8.441123332415255,3.5593519543311287,8.505633232810988,5.38704327166558,93.81952067315392,43.08232151710042,12.791391043403525,69.7380338434871,1.3424590215351815,34.01913900064919,6.315156248520575,41.67675427070494,174.29796279485242,0.06566503664190335,1.115185972116605,96.59925173251918,0.0006594983968755774,9.455957193306968,0.06495879022996191,24.85035470838785,4.674801506585862e-05,42.27721318852181,4.043880652573237,2.8530468343039725,2.283560723689119,1.8658111396792856,1.760319928884118,0.9872356946246871,6.230789077111895,0.07698608946551269,0.5481001624237407,5.682688914688154,0.03306317767518576,0.043922911790326924,0.5811633400989265,5.726611826478481,0.03611112581642298,1.6363293700000003 -PT,0.16853103420639537,6.167189443761765,6.844749403026448,0.6192555042551876,8.657967119181887,2.513361052425902,30.644647379477153,12.31396861546159,0.011388980962399788,21.88441540460838,5.314540992166446,1.8223271167970734,2.3288462896580606,1.389450501188967,2.7089567112102446,1.677792763857291,21.132756291752937,15.70110751612303,0.2966663759252526,11.94361641655254,0.8077702887437023,1.9424746346398076,2.1680579713654864,4.918302894748996,60.07163884638256,0.005796247464804267,0.6943809466998286,37.48714626049493,0.004144788310373135,1.8392478706029092,0.0014471892405585024,8.822663675857115,0.00015068153169906648,11.228200092727777,0.4119341087857363,0.29221113468922777,0.3542766997705545,0.261975695831413,0.0576574090151818,0.03023543885781476,12.04189939469445,0.24947267240484822,1.7711568990199187,10.270742495674531,0.1107532522332049,0.13871942017164332,1.8819101512531238,10.409461915846174,1.4175055568193127,7.483056010000001 -RO,0.8164119937824674,59.96132511343599,12.535594689161648,0.5966601803368272,8.287391484240173,0.6268225706774041,90.0416838937321,11.895960635306784,10.526118400572317,81.04442001456628,11.472370482472948,0.31126932980776745,2.499852646666194,0.43035561975633935,2.7719690913846016,1.1178608185175691,22.759444431153444,8.001025371554253,2.5952795750395103,16.898643459088195,0.7007809689311082,2.719627160661053,2.094159726588078,5.514567856180239,56.27724851083044,0.006095761615848004,0.21744941693675585,30.680317400032756,0.005639041540646845,11.646944255137056,0.00015561928868615372,7.116715973219791,4.505132889433588e-05,6.615821465504074,2.9624978308219267,1.0611074233348035,2.1230366871828705,0.7659474400326133,0.839461143639056,0.2951599833021901,2.424579253795485,0.029546566493058167,0.20490687455205012,2.2196723792434345,0.02497642466045021,0.004570141832607959,0.2298832992125003,2.2242425210760426,0.48333372495510224,0.48328465000000004 -RS,2.041447300754024,17.23241025134951,4.7096888757710875,1.7229567393124234,3.2058768205282546,2.221982761190753,33.10772576,14.145999310000002,,,4.436825941714843,0.8017518796993448,0.8519426499259911,0.3586791598052431,0.985253156817818,0.6309047643840355,9.346379720000002,4.898997940000001,,,,,,,22.236583260000003,0.0,0.285668530623453,13.081268662284423,0.0,1.0909464043385495,0.0,2.5057455043946524,0.0,5.272954158358922,0.5751616500000001,0.47800463000000004,0.43322796621493825,0.3672170905695961,0.14193368378506177,0.11078753943040395,0.4886500505665789,0.03162962943342115,0.0,0.4886500505665789,0.0,0.03162962943342115,0.0,0.52027968,0.10861257000000002,0.10931037 -SE,18.4203419908313,55.67334748926343,14.09700520336014,7.62928350560303,5.841710107199814,5.785689042843419,86.98471532467524,43.504196153945145,30.748061957160978,75.74261755247736,22.700405098892183,4.608878048490316,4.269153420938776,1.7706726927590173,4.374008646562861,4.286688107484913,46.05946171198871,25.491266437692087,15.6636143722038,32.15077398576433,1.0361667627692726,2.0898260280447953,1.010335904191686,4.136328695005755,83.04667870338166,0.01280258345751495,0.34320531162223467,51.05330259569601,0.0061524965671318635,6.1507365387542245,0.001514571605766908,7.009050732526921,0.002657098759479214,18.490383524782263,2.761947422261534,2.750002981435833,2.0247546969142176,2.0184609369237854,0.7371927253473161,0.7315420445120471,10.241309021150325,0.18676454919898827,1.9390076232799187,8.302301397870405,0.06947218543730693,0.11729236376168135,2.0084798087172255,8.419593761632086,0.7977769681471846,18.578494690000003 -SI,0.34227433442339894,8.376475064050604,2.9220707579747884,0.453818512537254,0.8699951748952809,0.5648060322082612,13.995062824999088,3.2289764898849858,0.9472293430810873,12.239285152226612,2.570621989760942,0.6007517589071416,0.5133561218151605,0.18953808270509473,0.647701991765941,0.48074261627130094,5.65765380957795,3.2179778886185555,0.5774928791411378,3.880746457492038,0.0,0.549946125291897,0.2641860340673345,0.8141321593592316,20.80700782789383,0.00010110043502486967,0.09327831893642177,11.880939237809546,8.371504104872192e-05,0.7193026858478071,0.0,0.9960279918658297,0.0,7.117459593434226,0.27222489410492606,0.15389153928786237,0.14979465172003809,0.07389407828151527,0.12243024238488796,0.0799974610063471,0.3099300394503224,0.004512821170390541,0.0,0.3099300394503224,0.0028361068079113625,0.0016767143624791794,0.0028361068079113625,0.3116067538128016,0.0,0.7076273500000001 -SK,0.7110244219256427,16.322019247760963,3.9420025260381517,0.4761557307462169,1.9667320357371187,0.7020969879798182,24.976139305348433,4.930015666329864,5.806117943917119,22.24202848390687,12.395946919792488,1.59350805554285,1.5814611823385478,0.7318096909411619,1.591685689729668,0.771687906762327,19.888551870977174,7.539329752327933,1.1225013209917258,15.78104264329495,0.21327589995852728,0.8694320240429981,0.44148890889017306,1.5241968328916984,23.792769864038295,0.03763800097049386,0.03401005608446167,9.914745300376103,6.220464867834378e-05,1.7151976456935867,0.0375611283634322,2.3748436290651527,1.0622037973027887e-05,9.753973232818993,0.5388822140190512,0.5388822140190512,0.37230773469849826,0.37230773469849826,0.166574479320553,0.166574479320553,0.46763946179091403,0.01347991726152616,0.0,0.46763946179091403,0.010194585747323313,0.003285331514202844,0.010194585747323313,0.4709247933051169,0.18577762000000003,0.0 From a5d0f07fe13f39cbc437c218324df90195646fb4 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 15:29:27 +0100 Subject: [PATCH 126/173] build_cutout: revert to master version --- scripts/build_cutout.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 80ddae34..1edb18ce 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -109,11 +109,9 @@ if __name__ == "__main__": cutout_params = snakemake.params.cutouts[snakemake.wildcards.cutout] - if "time" not in cutout_params: - snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) - cutout_params["time"] = [snapshots[0], snapshots[-1]] - - cutout_params["time"] = slice(*cutout_params["time"]) + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) + time = [snapshots[0], snapshots[-1]] + cutout_params["time"] = slice(*cutout_params.get("time", time)) if {"x", "y", "bounds"}.isdisjoint(cutout_params): # Determine the bounds from bus regions with a buffer of two grid cells From 57b487f1638dbf998927d5fdb4a3afaf5b94db64 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 15:38:36 +0100 Subject: [PATCH 127/173] handle leap days in temporal clustering --- rules/build_electricity.smk | 1 + scripts/prepare_network.py | 3 +++ scripts/prepare_sector_network.py | 7 +++++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 6db8bcd7..2b4d9af6 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -588,6 +588,7 @@ rule prepare_network: costs=config_provider("costs"), adjustments=config_provider("adjustments", "electricity"), autarky=config_provider("electricity", "autarky", default={}), + drop_leap_day=config_provider("enable", "drop_leap_day"), input: resources("networks/elec_s{simpl}_{clusters}_ec.nc"), tech_costs=lambda w: resources( diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index f38a3619..f7a3f6b0 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -204,6 +204,9 @@ def average_every_nhours(n, offset): m = n.copy(with_time=False) snapshot_weightings = n.snapshot_weightings.resample(offset).sum() + sns = snapshot_weightings.index + if snakemake.params.drop_leap_days: + sns = sns[~((sns.month == 2) & (sns.day == 29))] m.set_snapshots(snapshot_weightings.index) m.snapshot_weightings = snapshot_weightings diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 5757a88b..e5f0dac8 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -812,6 +812,10 @@ def average_every_nhours(n, offset): m = n.copy(with_time=False) snapshot_weightings = n.snapshot_weightings.resample(offset).sum() + sns = snapshot_weightings.index + if snakemake.params.drop_leap_days: + sns = sns[~((sns.month == 2) & (sns.day == 29))] + snapshot_weightings = snapshot_weightings.loc[sns] m.set_snapshots(snapshot_weightings.index) m.snapshot_weightings = snapshot_weightings @@ -3660,8 +3664,7 @@ if __name__ == "__main__": solver_name = snakemake.config["solving"]["solver"]["name"] resolution = snakemake.params.time_resolution - drop_leap_day = snakemake.params.drop_leap_day - n = set_temporal_aggregation(n, resolution, solver_name, drop_leap_day) + n = set_temporal_aggregation(n, resolution, solver_name) co2_budget = snakemake.params.co2_budget if isinstance(co2_budget, str) and co2_budget.startswith("cb"): From 4fbb3c81c4ddb589614f67dc01c885c9b3806bbe Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 16:48:32 +0100 Subject: [PATCH 128/173] fix build_district_heat_share and build_transport_demand for multiyear --- rules/build_sector.smk | 4 ++++ scripts/build_district_heat_share.py | 7 +++---- scripts/build_energy_totals.py | 19 +++++++++++++++++-- ...build_population_weighted_energy_totals.py | 2 +- scripts/build_transport_demand.py | 10 ++++++---- scripts/prepare_network.py | 2 +- scripts/prepare_sector_network.py | 2 +- 7 files changed, 33 insertions(+), 13 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 14226ef8..e1575a78 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -729,6 +729,8 @@ rule build_retro_cost: rule build_population_weighted_energy_totals: + params: + snapshots=config_provider("snapshots"), input: energy_totals=resources("{kind}_totals.csv"), clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), @@ -771,6 +773,7 @@ rule build_transport_demand: snapshots=config_provider("snapshots"), drop_leap_day=config_provider("enable", "drop_leap_day"), sector=config_provider("sector"), + energy_totals_year=config_provider("energy", "energy_totals_year"), input: clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), pop_weighted_energy_totals=resources( @@ -799,6 +802,7 @@ rule build_transport_demand: rule build_district_heat_share: params: sector=config_provider("sector"), + energy_totals_year=config_provider("energy", "energy_totals_year"), input: district_heat_share=resources("district_heat_share.csv"), clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), diff --git a/scripts/build_district_heat_share.py b/scripts/build_district_heat_share.py index 3cf88466..10a30915 100644 --- a/scripts/build_district_heat_share.py +++ b/scripts/build_district_heat_share.py @@ -22,7 +22,7 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_district_heat_share", simpl="", - clusters=48, + clusters=60, planning_horizons="2050", ) configure_logging(snakemake) @@ -32,9 +32,8 @@ if __name__ == "__main__": pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) - district_heat_share = pd.read_csv(snakemake.input.district_heat_share, index_col=0)[ - "district heat share" - ] + year = str(snakemake.params.energy_totals_year) + district_heat_share = pd.read_csv(snakemake.input.district_heat_share, index_col=0)[year] # make ct-based share nodal district_heat_share = district_heat_share.reindex(pop_layout.ct).fillna(0) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 01532872..a53c6208 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -721,10 +721,18 @@ def build_transport_data(countries, population, idees): transport_data = pd.DataFrame(idees["passenger cars"]) + countries_without_ch = set(countries) - {"CH"} + new_index = pd.MultiIndex.from_product( + [countries_without_ch, transport_data.index.levels[1]], + names=["country", "year"] + ) + + transport_data = transport_data.reindex(index=new_index) + # https://www.bfs.admin.ch/bfs/en/home/statistics/mobility-transport/transport-infrastructure-vehicles/vehicles/road-vehicles-stock-level-motorisation.html if "CH" in countries: fn = snakemake.input.swiss_transport - swiss_cars = pd.read_csv(fn, index_col=0).loc[1990:2021, ["passenger cars"]] + swiss_cars = pd.read_csv(fn, index_col=0).loc[2000:2015, ["passenger cars"]] swiss_cars.index = pd.MultiIndex.from_product( [["CH"], swiss_cars.index], names=["country", "year"] @@ -741,7 +749,14 @@ def build_transport_data(countries, population, idees): ) cars_pp = transport_data["number cars"] / population - transport_data.loc[missing, "number cars"] = cars_pp.mean() * population + + fill_values = {year: cars_pp.mean() * population for year in transport_data.index.levels[1]} + fill_values = pd.DataFrame(fill_values).stack() + fill_values = pd.DataFrame(fill_values, columns=["number cars"]) + fill_values.index.names = ["country", "year"] + fill_values = fill_values.reindex(transport_data.index) + + transport_data = transport_data.combine_first(fill_values) # collect average fuel efficiency in kWh/km diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index 70105a06..69d4c681 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -15,7 +15,7 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_population_weighted_energy_totals", - kind="energy", + kind="heat", simpl="", clusters=60, ) diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index e39774fa..085a0954 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -23,8 +23,9 @@ from _helpers import ( logger = logging.getLogger(__name__) -def build_nodal_transport_data(fn, pop_layout): - transport_data = pd.read_csv(fn, index_col=0) +def build_nodal_transport_data(fn, pop_layout, year): + transport_data = pd.read_csv(fn, index_col=[0, 1]) + transport_data = transport_data.xs(min(2015, year), level='year') nodal_transport_data = transport_data.loc[pop_layout.ct].fillna(0.0) nodal_transport_data.index = pop_layout.index @@ -173,7 +174,7 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_transport_demand", simpl="", - clusters=48, + clusters=60, ) configure_logging(snakemake) set_scenario_config(snakemake) @@ -194,8 +195,9 @@ if __name__ == "__main__": nyears = len(snapshots) / 8760 + energy_totals_year = snakemake.params.energy_totals_year nodal_transport_data = build_nodal_transport_data( - snakemake.input.transport_data, pop_layout + snakemake.input.transport_data, pop_layout, energy_totals_year ) transport_demand = build_transport_demand( diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index f7a3f6b0..00cb00bf 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -205,7 +205,7 @@ def average_every_nhours(n, offset): snapshot_weightings = n.snapshot_weightings.resample(offset).sum() sns = snapshot_weightings.index - if snakemake.params.drop_leap_days: + if snakemake.params.drop_leap_day: sns = sns[~((sns.month == 2) & (sns.day == 29))] m.set_snapshots(snapshot_weightings.index) m.snapshot_weightings = snapshot_weightings diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index e5f0dac8..ceb7764e 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -813,7 +813,7 @@ def average_every_nhours(n, offset): snapshot_weightings = n.snapshot_weightings.resample(offset).sum() sns = snapshot_weightings.index - if snakemake.params.drop_leap_days: + if snakemake.params.drop_leap_day: sns = sns[~((sns.month == 2) & (sns.day == 29))] snapshot_weightings = snapshot_weightings.loc[sns] m.set_snapshots(snapshot_weightings.index) From 1496c585d03b60a1992a350a942872eb867a76a9 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 14 Mar 2024 16:56:00 +0100 Subject: [PATCH 129/173] Fix duplicated years in add_land_use_constraint_m --- doc/release_notes.rst | 2 ++ scripts/solve_network.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index ad1bbf4f..5682d788 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -106,6 +106,8 @@ Upcoming Release * Adapt the disabling of transmission expansion in myopic foresight optimisations when limit is already reached to also handle cost limits. +* Fix duplicated years in `add_land_use_constraint_m`. + PyPSA-Eur 0.10.0 (19th February 2024) ===================================== diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 7e53e606..7adf492a 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -165,7 +165,7 @@ def _add_land_use_constraint_m(n, planning_horizons, config): previous_years = [ str(y) - for y in planning_horizons + grouping_years + for y in set(planning_horizons + grouping_years) if y < int(snakemake.wildcards.planning_horizons) ] From 115dd8d11643225a73f39616991f61f123de2e3e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 17:15:24 +0100 Subject: [PATCH 130/173] handle case when only artifical load must be loaded --- scripts/build_electricity_demand.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index 2d3da224..8d61e770 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -183,7 +183,7 @@ def manual_adjustment(load, fn_load, countries): elif "MK" in load: load["AL"] = load["MK"] * (4.1 / 7.4) - if "MK" in countries: + if "MK" in countries and "MK" in countries: if "MK" not in load or load.MK.isnull().sum() > len(load) / 2: if "ME" in load: load["MK"] = load.ME * (6.7 / 2.9) @@ -192,7 +192,7 @@ def manual_adjustment(load, fn_load, countries): if "ME" in load: load["BA"] = load.HR * (11.0 / 16.2) - if "KV" not in load or load.KV.isnull().values.all(): + if ("KV" not in load or load.KV.isnull().values.all()) and "KV" in countries: if "RS" in load: load["KV"] = load["RS"] * (4.8 / 27.0) @@ -267,7 +267,7 @@ if __name__ == "__main__": snakemake.params.snapshots, snakemake.params.drop_leap_day ) - fixed_year = snakemake.config["load"].get("fixed_year", False) + fixed_year = snakemake.params["load"].get("fixed_year", False) years = ( slice(str(fixed_year), str(fixed_year)) if fixed_year @@ -281,6 +281,8 @@ if __name__ == "__main__": load = load_timeseries(snakemake.input.reported, years, countries) + load = load.reindex(index=snapshots) + if "UA" in countries: # attach load of UA (best data only for entsoe transparency) load_ua = load_timeseries(snakemake.input.reported, "2018", ["UA"]) @@ -297,9 +299,6 @@ if __name__ == "__main__": if snakemake.params.load["manual_adjustments"]: load = manual_adjustment(load, snakemake.input[0], countries) - if load.empty: - logger.warning("Build electricity demand time series is empty.") - logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.") load = load.interpolate(method="linear", limit=interpolate_limit) From 88dba985121847067317a9e423f63043b2d32140 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 18:11:02 +0100 Subject: [PATCH 131/173] fix for CI --- scripts/build_energy_totals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index a53c6208..19fc5a8e 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -419,7 +419,7 @@ def build_energy_totals(countries, eurostat, swiss, idees): # add swiss energy data - df = pd.concat([df.drop("CH"), swiss]).sort_index() + df = pd.concat([df.drop("CH", errors='ignore'), swiss]).sort_index() # get values for missing countries based on Eurostat EnergyBalances # divide cooking/space/water according to averages in EU28 From a6ea15ea4a811a4874209c780c5ea42958092d4c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 17:11:26 +0000 Subject: [PATCH 132/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_district_heat_share.py | 4 +++- scripts/build_energy_totals.py | 8 +++++--- scripts/build_transport_demand.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/build_district_heat_share.py b/scripts/build_district_heat_share.py index 10a30915..178f2c0d 100644 --- a/scripts/build_district_heat_share.py +++ b/scripts/build_district_heat_share.py @@ -33,7 +33,9 @@ if __name__ == "__main__": pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) year = str(snakemake.params.energy_totals_year) - district_heat_share = pd.read_csv(snakemake.input.district_heat_share, index_col=0)[year] + district_heat_share = pd.read_csv(snakemake.input.district_heat_share, index_col=0)[ + year + ] # make ct-based share nodal district_heat_share = district_heat_share.reindex(pop_layout.ct).fillna(0) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 19fc5a8e..dda84a53 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -419,7 +419,7 @@ def build_energy_totals(countries, eurostat, swiss, idees): # add swiss energy data - df = pd.concat([df.drop("CH", errors='ignore'), swiss]).sort_index() + df = pd.concat([df.drop("CH", errors="ignore"), swiss]).sort_index() # get values for missing countries based on Eurostat EnergyBalances # divide cooking/space/water according to averages in EU28 @@ -724,7 +724,7 @@ def build_transport_data(countries, population, idees): countries_without_ch = set(countries) - {"CH"} new_index = pd.MultiIndex.from_product( [countries_without_ch, transport_data.index.levels[1]], - names=["country", "year"] + names=["country", "year"], ) transport_data = transport_data.reindex(index=new_index) @@ -750,7 +750,9 @@ def build_transport_data(countries, population, idees): cars_pp = transport_data["number cars"] / population - fill_values = {year: cars_pp.mean() * population for year in transport_data.index.levels[1]} + fill_values = { + year: cars_pp.mean() * population for year in transport_data.index.levels[1] + } fill_values = pd.DataFrame(fill_values).stack() fill_values = pd.DataFrame(fill_values, columns=["number cars"]) fill_values.index.names = ["country", "year"] diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index 085a0954..35f22a80 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) def build_nodal_transport_data(fn, pop_layout, year): transport_data = pd.read_csv(fn, index_col=[0, 1]) - transport_data = transport_data.xs(min(2015, year), level='year') + transport_data = transport_data.xs(min(2015, year), level="year") nodal_transport_data = transport_data.loc[pop_layout.ct].fillna(0.0) nodal_transport_data.index = pop_layout.index From 8d85acc61aa0252181c2d8da5f7702242dd1bbad Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 18:19:25 +0100 Subject: [PATCH 133/173] remove unused keyword argument in temporal aggregation [no ci] --- scripts/prepare_sector_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index ceb7764e..4e7e0374 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3486,7 +3486,7 @@ def apply_time_segmentation( return n -def set_temporal_aggregation(n, resolution, solver_name, drop_leap_day=False): +def set_temporal_aggregation(n, resolution, solver_name): """ Aggregate network temporally. """ From 02fc334a2e6bd0669babc1d817026c0d0102e702 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 18:46:45 +0100 Subject: [PATCH 134/173] handle leap days in add_brownfield --- rules/solve_myopic.smk | 1 + scripts/add_brownfield.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 57b8a9d3..eec1aca1 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -70,6 +70,7 @@ rule add_brownfield: ), threshold_capacity=config_provider("existing_capacities", " threshold_capacity"), snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), carriers=config_provider("electricity", "renewable_carriers"), input: unpack(input_profile_tech_brownfield), diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 1bccf563..c607132c 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -14,6 +14,7 @@ import pypsa import xarray as xr from _helpers import ( configure_logging, + get_snapshots set_scenario_config, update_config_from_wildcards, ) @@ -192,7 +193,7 @@ def adjust_renewable_profiles(n, input_profiles, params, year): clustermaps.index = clustermaps.index.astype(str) # temporal clustering - dr = pd.date_range(**params["snapshots"], freq="h") + dr = get_snapshots(params["snapshots"], params["drop_leap_day"]) snapshotmaps = ( pd.Series(dr, index=dr).where(lambda x: x.isin(n.snapshots), pd.NA).ffill() ) From f74e2f1bb7b7fee97a7d1c748a3ea3ef03baf5c7 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 18:47:09 +0100 Subject: [PATCH 135/173] build_heat_totals: allow subset of countries --- scripts/build_heat_totals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_heat_totals.py b/scripts/build_heat_totals.py index d0dab987..9bee63e5 100644 --- a/scripts/build_heat_totals.py +++ b/scripts/build_heat_totals.py @@ -16,7 +16,7 @@ idx = pd.IndexSlice def approximate_heat_demand(energy_totals, hdd): - countries = hdd.columns + countries = hdd.columns.intersection(energy_totals.index.levels[0]) demands = {} From f7e617befd090545ad5ecfb8169324d65fa67873 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 18:52:25 +0100 Subject: [PATCH 136/173] make_summary: add drop_leap_day param --- rules/postprocess.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 1b188829..6d239a54 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -156,6 +156,7 @@ rule make_summary: foresight=config_provider("foresight"), costs=config_provider("costs"), snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), scenario=config_provider("scenario"), RDIR=RDIR, input: From 7587f2c6becb6d2bca565f3aacccdb1a16b27d3f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 18:52:59 +0100 Subject: [PATCH 137/173] pop_weighted_energy_totals: feasible for sub-annual temporal scope --- scripts/build_population_weighted_energy_totals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index 69d4c681..60af66aa 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -24,7 +24,7 @@ if __name__ == "__main__": config = snakemake.config["energy"] if snakemake.wildcards.kind == "heat": - years = pd.date_range(freq="YE", **snakemake.params.snapshots).year + years = pd.date_range(freq="h", **snakemake.params.snapshots).year.unique() assert len(years) == 1, "Currently only works for single year." data_year = years[0] else: From 0bbda371d4a789a2e70a6d074a3d426842672450 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 17:54:14 +0000 Subject: [PATCH 138/173] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_brownfield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index c607132c..82a40e57 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -14,7 +14,7 @@ import pypsa import xarray as xr from _helpers import ( configure_logging, - get_snapshots + get_snapshots, set_scenario_config, update_config_from_wildcards, ) From 63072f2224eb1424cd1cdb9739de6ac00ad41b9d Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 19:06:21 +0100 Subject: [PATCH 139/173] fix syntax mistake --- scripts/add_brownfield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index c607132c..82a40e57 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -14,7 +14,7 @@ import pypsa import xarray as xr from _helpers import ( configure_logging, - get_snapshots + get_snapshots, set_scenario_config, update_config_from_wildcards, ) From a2f97ad1b38d6d3ddeccbc4d7ee665549a211cb1 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 14 Mar 2024 19:24:37 +0100 Subject: [PATCH 140/173] make build_eurostat independent of snakemake object to be imported --- scripts/build_energy_totals.py | 14 ++++++++------ scripts/prepare_sector_network.py | 15 +++++++++------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index dda84a53..abab06e4 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -84,22 +84,19 @@ def eurostat_per_country(input_eurostat, country): return pd.concat(sheet) -def build_eurostat(input_eurostat, countries): +def build_eurostat(input_eurostat, countries, nprocesses=1, disable_progressbar=False): """ Return multi-index for all countries' energy data in TWh/a. """ countries = {idees_rename.get(country, country) for country in countries} - {"CH"} - nprocesses = snakemake.threads - disable_progress = snakemake.config["run"].get("disable_progressbar", False) - func = partial(eurostat_per_country, input_eurostat) tqdm_kwargs = dict( ascii=False, unit=" country", total=len(countries), desc="Build from eurostat database", - disable=disable_progress, + disable=disable_progressbar, ) with mute_print(): with mp.Pool(processes=nprocesses) as pool: @@ -953,7 +950,12 @@ if __name__ == "__main__": idees_countries = pd.Index(countries).intersection(eu28) input_eurostat = snakemake.input.eurostat - eurostat = build_eurostat(input_eurostat, countries) + eurostat = build_eurostat( + input_eurostat, + countries, + nprocesses=snakemake.threads, + disable_progressbar=snakemake.config["run"].get("disable_progressbar", False), + ) swiss = build_swiss() idees = build_idees(idees_countries) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 4e7e0374..b6ec98b2 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -23,7 +23,12 @@ from _helpers import ( update_config_from_wildcards, ) from add_electricity import calculate_annuity, sanitize_carriers, sanitize_locations -from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2 +from build_energy_totals import ( + build_co2_totals, + build_eea_co2, + build_eurostat, + build_eurostat_co2, +) from networkx.algorithms import complement from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation from prepare_network import maybe_adjust_costs_and_potentials @@ -255,12 +260,10 @@ def co2_emissions_year( """ eea_co2 = build_eea_co2(input_co2, year, emissions_scope) - # TODO: read Eurostat data from year > 2014 + eurostat = build_eurostat(input_eurostat, countries) + # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK - if year > 2014: - eurostat_co2 = build_eurostat_co2(input_eurostat, countries, 2014) - else: - eurostat_co2 = build_eurostat_co2(input_eurostat, countries, year) + eurostat_co2 = build_eurostat_co2(eurostat, year) co2_totals = build_co2_totals(countries, eea_co2, eurostat_co2) From 97d876d4e5dce2c2b7eaad452e159a918a9fd8aa Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 15 Mar 2024 11:44:41 +0100 Subject: [PATCH 141/173] update synthetic load data to cover 1941-2023 --- rules/retrieve.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 76069c6f..e6f46aaa 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -199,7 +199,7 @@ if config["enable"]["retrieve"]: rule retrieve_artificial_load_data: input: storage( - "https://zenodo.org/record/7070438/files/demand_hourly.csv", + "https://zenodo.org/records/10820928/files/demand_hourly.csv", keep_local=True, ), output: From ee55bba56827fa03105d79beda9b87f785cb0f1e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 15 Mar 2024 12:04:25 +0100 Subject: [PATCH 142/173] omit synthetic load data md5 validation --- rules/retrieve.smk | 2 -- 1 file changed, 2 deletions(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index e6f46aaa..4724d8db 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -200,7 +200,6 @@ if config["enable"]["retrieve"]: input: storage( "https://zenodo.org/records/10820928/files/demand_hourly.csv", - keep_local=True, ), output: "data/load_artificial_raw.csv", @@ -211,7 +210,6 @@ if config["enable"]["retrieve"]: retries: 2 run: move(input[0], output[0]) - validate_checksum(output[0], input[0]) if config["enable"]["retrieve"]: From 3a5465371114627377a6ac03817dde05f16bd2a0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 15 Mar 2024 13:50:27 +0100 Subject: [PATCH 143/173] fix shared resources for scenario management with {kind} wildcard --- scripts/_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 09687a11..d03a306b 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -60,7 +60,7 @@ def get_run_path(fn, dir, rdir, shared_resources): if shared_resources == "base": pattern = r"\{([^{}]+)\}" existing_wildcards = set(re.findall(pattern, fn)) - irrelevant_wildcards = {"technology", "year", "scope"} + irrelevant_wildcards = {"technology", "year", "scope", "kind"} no_relevant_wildcards = not existing_wildcards - irrelevant_wildcards no_elec_rule = not fn.startswith("networks/elec") and not fn.startswith( "add_electricity" @@ -68,7 +68,7 @@ def get_run_path(fn, dir, rdir, shared_resources): is_shared = no_relevant_wildcards and no_elec_rule elif isinstance(shared_resources, str): rdir = shared_resources + "/" - is_shared = True + is_shared = False elif isinstance(shared_resources, bool): is_shared = shared_resources else: From f50ee2f22551ba1dc73f32fa3b774853a1ebc960 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 15 Mar 2024 13:50:53 +0100 Subject: [PATCH 144/173] do not use log() for config independent retrieve rules --- rules/retrieve.smk | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 4724d8db..0a7cdf6a 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -168,7 +168,7 @@ if config["enable"]["retrieve"]: output: expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles), log: - logs("retrieve_gas_infrastructure_data.log"), + "logs/retrieve_gas_infrastructure_data.log", retries: 2 conda: "../envs/retrieve.yaml" @@ -184,7 +184,7 @@ if config["enable"]["retrieve"]: output: "data/electricity_demand_raw.csv", log: - logs("retrieve_electricity_demand.log"), + "logs/retrieve_electricity_demand.log", resources: mem_mb=5000, retries: 2 @@ -204,7 +204,7 @@ if config["enable"]["retrieve"]: output: "data/load_artificial_raw.csv", log: - logs("retrieve_artificial_load_data.log"), + "logs/retrieve_artificial_load_data.log", resources: mem_mb=5000, retries: 2 @@ -223,7 +223,7 @@ if config["enable"]["retrieve"]: output: protected("data/shipdensity_global.zip"), log: - logs("retrieve_ship_raster.log"), + "logs/retrieve_ship_raster.log", resources: mem_mb=5000, retries: 2 @@ -351,7 +351,7 @@ if config["enable"]["retrieve"]: output: "data/validation/emission-spot-primary-market-auction-report-2019-data.xls", log: - logs("retrieve_monthly_co2_prices.log"), + "logs/retrieve_monthly_co2_prices.log", resources: mem_mb=5000, retries: 2 @@ -365,7 +365,7 @@ if config["enable"]["retrieve"]: output: "data/validation/energy-price-trends-xlsx-5619002.xlsx", log: - logs("retrieve_monthly_fuel_prices.log"), + "logs/retrieve_monthly_fuel_prices.log", resources: mem_mb=5000, retries: 2 From e459c3c89d11a7ff159de97718ce2807b689d27a Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 15 Mar 2024 14:28:01 +0100 Subject: [PATCH 145/173] add release notes; rename artificial to synthetic --- config/config.default.yaml | 2 +- doc/configtables/load.csv | 2 +- doc/release_notes.rst | 46 +++++++++++++++++++++++++++++ doc/sector.rst | 5 ++++ rules/build_electricity.smk | 6 ++-- rules/retrieve.smk | 6 ++-- scripts/build_electricity_demand.py | 12 ++++---- 7 files changed, 65 insertions(+), 14 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index da2e20f1..a8cd195c 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -292,7 +292,7 @@ load: manual_adjustments: true # false scaling_factor: 1.0 fixed_year: false # false or year (e.g. 2013) - supplement_missing_data_artificially: true + supplement_synthetic: true # docs # TODO: PyPSA-Eur merge issue in prepare_sector_network.py diff --git a/doc/configtables/load.csv b/doc/configtables/load.csv index 02e7b97b..34d73dc5 100644 --- a/doc/configtables/load.csv +++ b/doc/configtables/load.csv @@ -4,4 +4,4 @@ time_shift_for_large_gaps,string,string,"Periods which are used for copying time manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`." scaling_factor,--,float,"Global correction factor for the load time series." fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year" -supplement_missing_data_artificially,bool,"{true, false}","Whether to supplement missing data for selected time period should be supplemented by artificial data from https://zenodo.org/record/7070438/files/demand_hourly.csv." +supplement_synthetic,bool,"{true, false}","Whether to supplement missing data for selected time period should be supplemented by synthetic data from https://zenodo.org/record/10820928." diff --git a/doc/release_notes.rst b/doc/release_notes.rst index ad1bbf4f..a5cb9273 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,52 @@ Release Notes Upcoming Release ================ +* Enhanced support for choosing different weather years + (https://github.com/PyPSA/pypsa-eur/pull/204): + + - Processed energy statistics from eurostat (1990-2021) and IDEES (2000-2015) + are now initially stored for all available years and filtered by the year + given in ``energy: energy_totals_year:``. + + - Added option to supplement electricity load data with synthetic time series + for years not contained in OPSD (from https://zenodo.org/records/10820928, + ``load: supplement_synthetic:``). + + - The total annual heat demand for years not contained in the energy + statistics by eurostat (1990-2021) or IDEES (2000-2015) are scaled based on + a regression between the total number of heating degree days and the total + annual heat demand between the years 2007-2021, assuming a similar building + stock. + + - Added option to scale annual hydro-electricity generation data for years not + contained in the in EIA (1980-2021) based on a regression between annual + generation and total runoff per country for the years 1980-2021 + (``renewable: hydro: eia_approximate_missing:``) + + - Added option to normalize annual hydro generation data by the associated + installed capacity reported by EIA (1980-2021) in order to eliminate changes + in generation due to newly built capacity (``renewable: hydro: + eia_approximate_missing: eia_correct_by_capacity:``). + + - Added option to make hydro generation data independent of weather year + (``renewable: hydro: eia_approximate_missing: eia_norm_year:``). + + - Added option to drop leap days (``enable: drop_leap_day:``). + + - Added option to make electric load data independent of weather year + (``load: fixed_year:``). + + - Include time series of Swiss number of passenger vehicles from the `Swiss + Federal Statistical Office + `_. + + - Updated hydro-electricity generation and capacity data from EIA. + + - The easiest way to sweep over multiple weather years is to use the new + scenario management. An example for the necessary `create_scenarios.py` + script can be found in this `Github gist + `_. + * Upgrade to Snakemake v8.5+. This version is the new minimum version required. To upgrade an existing environment, run ``conda install -c bioconda snakemake-minimal">=8.5"`` and ``pip install snakemake-storage-plugin-http`` diff --git a/doc/sector.rst b/doc/sector.rst index 84a5fdc9..7a04a580 100644 --- a/doc/sector.rst +++ b/doc/sector.rst @@ -56,6 +56,11 @@ Rule ``build_energy_totals`` .. automodule:: build_energy_totals +Rule ``build_heat_totals`` +============================================================================== + +.. automodule:: build_heat_totals + Rule ``build_gas_input_locations`` ============================================================================== diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 2b4d9af6..f84a9969 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -26,9 +26,9 @@ rule build_electricity_demand: load=config_provider("load"), input: reported=ancient("data/electricity_demand_raw.csv"), - artificial=lambda w: ( - ancient("data/load_artificial_raw.csv") - if config_provider("load", "supplement_missing_data_artificially")(w) + synthetic=lambda w: ( + ancient("data/load_synthetic_raw.csv") + if config_provider("load", "supplement_synthetic")(w) else [] ), output: diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 0a7cdf6a..b2c1e8d6 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -196,15 +196,15 @@ if config["enable"]["retrieve"]: if config["enable"]["retrieve"]: - rule retrieve_artificial_load_data: + rule retrieve_synthetic_electricity_demand: input: storage( "https://zenodo.org/records/10820928/files/demand_hourly.csv", ), output: - "data/load_artificial_raw.csv", + "data/load_synthetic_raw.csv", log: - "logs/retrieve_artificial_load_data.log", + "logs/retrieve_synthetic_electricity_demand.log", resources: mem_mb=5000, retries: 2 diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index 8d61e770..9174672d 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -307,12 +307,12 @@ if __name__ == "__main__": ) load = load.apply(fill_large_gaps, shift=time_shift) - if snakemake.params.load["supplement_missing_data_artificially"]: - logger.info("Supplement missing data with artificial data.") - fn = snakemake.input.artificial - artificial_load = pd.read_csv(fn, index_col=0, parse_dates=True) - artificial_load = artificial_load.loc[snapshots, countries] - load = load.combine_first(artificial_load) + if snakemake.params.load["supplement_synthetic"]: + logger.info("Supplement missing data with synthetic data.") + fn = snakemake.input.synthetic + synthetic_load = pd.read_csv(fn, index_col=0, parse_dates=True) + synthetic_load = synthetic_load.loc[snapshots, countries] + load = load.combine_first(synthetic_load) assert not load.isna().any().any(), ( "Load data contains nans. Adjust the parameters " From 941de04859e15a97a8db3dc464731bf1a5bd96af Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Mon, 18 Mar 2024 10:37:08 +0100 Subject: [PATCH 146/173] Don't use log path function in IRENA retrieval rule The `logs` function returns a path with a {run} wildcard which is not present in the output of the retrieval rule. Follows f50ee2f2. --- rules/retrieve.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index b2c1e8d6..ac296f79 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -50,7 +50,7 @@ if config["enable"].get("retrieve_irena"): onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv", solar="data/existing_infrastructure/solar_capacity_IRENA.csv", log: - logs("retrieve_irena.log"), + "logs/retrieve_irena.log", resources: mem_mb=1000, retries: 2 From ce549cd9494228998ee6689d095ca06c171fff11 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Mon, 18 Mar 2024 10:00:42 +0100 Subject: [PATCH 147/173] Allow the retrieval of cost data from forks of technology-data --- scripts/retrieve_cost_data.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/retrieve_cost_data.py b/scripts/retrieve_cost_data.py index eb1ef041..39fcab7c 100644 --- a/scripts/retrieve_cost_data.py +++ b/scripts/retrieve_cost_data.py @@ -25,9 +25,10 @@ if __name__ == "__main__": set_scenario_config(snakemake) version = snakemake.params.version - baseurl = ( - f"https://raw.githubusercontent.com/PyPSA/technology-data/{version}/outputs/" - ) + if "/" in version: + baseurl = f"https://raw.githubusercontent.com/{version}/outputs" + else: + baseurl = f"https://raw.githubusercontent.com/PyPSA/technology-data/{version}/outputs/" filepath = Path(snakemake.output[0]) url = baseurl + filepath.name From 1f499eda6ffb20ed81c15a1b6a3a8557a53e18d4 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Mon, 18 Mar 2024 13:31:04 +0100 Subject: [PATCH 148/173] Document new handling of costs version config option --- doc/configtables/costs.csv | 2 +- doc/release_notes.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/configtables/costs.csv b/doc/configtables/costs.csv index b69c0bf9..03933c18 100644 --- a/doc/configtables/costs.csv +++ b/doc/configtables/costs.csv @@ -1,6 +1,6 @@ ,Unit,Values,Description year,--,YYYY; e.g. '2030',Year for which to retrieve cost assumptions of ``resources/costs.csv``. -version,--,vX.X.X; e.g. 'v0.5.0',Version of ``technology-data`` repository to use. +version,--,vX.X.X or //vX.X.X; e.g. 'v0.5.0',Version of ``technology-data`` repository to use. If this string is of the form // then costs are instead retrieved from ``github.com//`` at the tag. rooftop_share,--,float,Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV). social_discountrate,p.u.,float,Social discount rate to compare costs in different investment periods. 0.02 corresponds to a social discount rate of 2%. fill_values,--,float,Default values if not specified for a technology in ``resources/costs.csv``. diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 11fd3df7..21094250 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -82,6 +82,8 @@ Upcoming Release * Upgrade default techno-economic assumptions to ``technology-data`` v0.8.1. +* Add possibility to download cost data from custom fork of ``technology-data``. + * Linearly interpolate missing investment periods in year-dependent configuration options. From e03fbf152c88ecff3aca56cc62ca4ac7794d6319 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Mon, 18 Mar 2024 13:37:58 +0000 Subject: [PATCH 149/173] Only sanitize locations when there are buses with a location Fixes an edge case when no buses with a location are added in add_extra_components. --- scripts/add_electricity.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ac73cb1f..7e60203f 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -184,12 +184,13 @@ def sanitize_carriers(n, config): def sanitize_locations(n): - n.buses["x"] = n.buses.x.where(n.buses.x != 0, n.buses.location.map(n.buses.x)) - n.buses["y"] = n.buses.y.where(n.buses.y != 0, n.buses.location.map(n.buses.y)) - n.buses["country"] = n.buses.country.where( - n.buses.country.ne("") & n.buses.country.notnull(), - n.buses.location.map(n.buses.country), - ) + if "location" in n.buses.columns: + n.buses["x"] = n.buses.x.where(n.buses.x != 0, n.buses.location.map(n.buses.x)) + n.buses["y"] = n.buses.y.where(n.buses.y != 0, n.buses.location.map(n.buses.y)) + n.buses["country"] = n.buses.country.where( + n.buses.country.ne("") & n.buses.country.notnull(), + n.buses.location.map(n.buses.country), + ) def add_co2_emissions(n, costs, carriers): From c983aa7d2dcdee44d805d9c820f416870110e14d Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 7 Mar 2024 10:22:28 +0100 Subject: [PATCH 150/173] prepare_perfect: ensure network.meta assignment --- scripts/prepare_perfect_foresight.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index f7e8495e..fea0cef4 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -537,5 +537,8 @@ if __name__ == "__main__": # set carbon constraints n = set_carbon_constraints(n) + # update meta + n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) + # export network n.export_to_netcdf(snakemake.output[0]) From f3eae37a02ebf974228279777082d180f97c4a8a Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 14 Mar 2024 12:52:25 +0100 Subject: [PATCH 151/173] scneario management: reenable shared resources in one folder --- scripts/_helpers.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index d03a306b..5884d908 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -66,20 +66,17 @@ def get_run_path(fn, dir, rdir, shared_resources): "add_electricity" ) is_shared = no_relevant_wildcards and no_elec_rule + rdir = "" if is_shared else rdir elif isinstance(shared_resources, str): rdir = shared_resources + "/" - is_shared = False elif isinstance(shared_resources, bool): - is_shared = shared_resources + rdir = "" else: raise ValueError( "shared_resources must be a boolean, str, or 'base' for special handling." ) - if is_shared: - return f"{dir}{fn}" - else: - return f"{dir}{rdir}{fn}" + return f"{dir}{rdir}{fn}" def path_provider(dir, rdir, shared_resources): From 1388104996d8794a6b345296386800eb43388a06 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 15 Mar 2024 13:02:27 +0100 Subject: [PATCH 152/173] test.sh: set continuous chain of && commands and print out commands --- test.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test.sh b/test.sh index d6007750..a3dfb65f 100755 --- a/test.sh +++ b/test.sh @@ -2,8 +2,12 @@ # # SPDX-License-Identifier: CC0-1.0 +set -x && \ + snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime && \ snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime && \ snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime && \ snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime && \ -snakemake -call all --configfile config/test/config.scenarios.yaml --rerun-triggers=mtime -n +snakemake -call all --configfile config/test/config.scenarios.yaml --rerun-triggers=mtime -n && \ + +set +x From 04d4d9849d6ab91133250d091cce22d1e128d5c6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 19:45:01 +0000 Subject: [PATCH 153/173] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black-pre-commit-mirror: 24.2.0 → 24.3.0](https://github.com/psf/black-pre-commit-mirror/compare/24.2.0...24.3.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 28d0278a..2fb51c54 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,7 +51,7 @@ repos: # Formatting with "black" coding style - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.2.0 + rev: 24.3.0 hooks: # Format Python files - id: black From 8d9afb0eb048f6d0d3c4370c84c29159862b4bec Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Tue, 19 Mar 2024 08:28:27 +0100 Subject: [PATCH 154/173] solve_electricity.smk:replace "minimal" by "shallow" --- rules/solve_electricity.smk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index f24119b9..389687a0 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -31,7 +31,7 @@ rule solve_network: mem_mb=memory, runtime=config_provider("solving", "runtime", default="6h"), shadow: - "minimal" + "shallow" conda: "../envs/environment.yaml" script: @@ -62,7 +62,7 @@ rule solve_operations_network: mem_mb=(lambda w: 10000 + 372 * int(w.clusters)), runtime=config_provider("solving", "runtime", default="6h"), shadow: - "minimal" + "shallow" conda: "../envs/environment.yaml" script: From 2565a7db4f283317a33e3a10c09b03463d2cb1fb Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 Mar 2024 08:20:23 +0100 Subject: [PATCH 155/173] snakefile: move copy_default_files and process_run_config to helpers --- Snakefile | 30 ++++++------------------------ scripts/_helpers.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/Snakefile b/Snakefile index 58cb6677..4a41622e 100644 --- a/Snakefile +++ b/Snakefile @@ -2,26 +2,18 @@ # # SPDX-License-Identifier: MIT -from os.path import normpath, exists -from shutil import copyfile, move, rmtree from pathlib import Path import yaml - +from os.path import normpath +from shutil import move, rmtree from snakemake.utils import min_version min_version("8.5") -from scripts._helpers import path_provider +from scripts._helpers import path_provider, copy_default_files, process_run_config -default_files = { - "config/config.default.yaml": "config/config.yaml", - "config/scenarios.template.yaml": "config/scenarios.yaml", -} -for template, target in default_files.items(): - target = os.path.join(workflow.current_basedir, target) - template = os.path.join(workflow.current_basedir, template) - if not exists(target) and exists(template): - copyfile(template, target) + +copy_default_files(workflow) configfile: "config/config.default.yaml" @@ -29,17 +21,7 @@ configfile: "config/config.yaml" run = config["run"] -scenarios = run.get("scenarios", {}) -if run["name"] and scenarios.get("enable"): - fn = Path(scenarios["file"]) - scenarios = yaml.safe_load(fn.read_text()) - RDIR = "{run}/" - if run["name"] == "all": - config["run"]["name"] = list(scenarios.keys()) -elif run["name"]: - RDIR = run["name"] + "/" -else: - RDIR = "" +RDIR = process_run_config(run) logs = path_provider("logs/", RDIR, run["shared_resources"]) benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"]) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index d03a306b..2bacf5cd 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -11,7 +11,9 @@ import os import re import urllib from functools import partial +from os.path import exists from pathlib import Path +from shutil import copyfile import pandas as pd import pytz @@ -25,6 +27,33 @@ logger = logging.getLogger(__name__) REGION_COLS = ["geometry", "name", "x", "y", "country"] +def copy_default_files(workflow): + default_files = { + "config/config.default.yaml": "config/config.yaml", + "config/scenarios.template.yaml": "config/scenarios.yaml", + } + for template, target in default_files.items(): + target = os.path.join(workflow.current_basedir, target) + template = os.path.join(workflow.current_basedir, template) + if not exists(target) and exists(template): + copyfile(template, target) + + +def process_run_config(run): + scenarios = run.get("scenarios", {}) + if run["name"] and scenarios.get("enable"): + fn = Path(scenarios["file"]) + scenarios = yaml.safe_load(fn.read_text()) + RDIR = "{run}/" + if run["name"] == "all": + run["name"] = list(scenarios.keys()) + elif run["name"]: + RDIR = run["name"] + "/" + else: + RDIR = "" + return RDIR + + def get_run_path(fn, dir, rdir, shared_resources): """ Dynamically provide paths based on shared resources and filename. From a8454fd8305dff8b5360536903f3d581125b062d Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Mon, 18 Mar 2024 16:16:30 +0000 Subject: [PATCH 156/173] Fill in missing eurostat data on domestic aviation energy demand --- scripts/build_energy_totals.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index abab06e4..b56d3294 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -124,6 +124,17 @@ def build_eurostat(input_eurostat, countries, nprocesses=1, disable_progressbar= ) df = pd.concat([temp, df.loc[~int_avia]]) + # Fill in missing data on "Domestic aviation" for each country. + domestic_avia = df.index.get_level_values(4) == "Domestic aviation" + for country in countries: + slicer = idx[country, :, :, :, "Domestic aviation"] + # For the Total and Fossil energy columns, fill in zeros with + # the closest non-zero value in the year index. + for col in ["Total", "Fossil energy"]: + df.loc[slicer, col] = ( + df.loc[slicer, col].replace(0.0, np.nan).ffill().bfill() + ) + # Renaming some indices index_rename = { "Households": "Residential", From 65af55c22b48e305a6baa30f265054429ba79a55 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Tue, 19 Mar 2024 08:54:58 +0100 Subject: [PATCH 157/173] Fix typos --- scripts/add_brownfield.py | 2 +- scripts/retrieve_cost_data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 82a40e57..7e49031b 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -145,7 +145,7 @@ def disable_grid_expansion_if_limit_hit(n): for name, glc in glcs.iterrows(): total_expansion = ( ( - n.lines.query("p_nom_extendable") + n.lines.query("s_nom_extendable") .eval(f"s_nom_min * {cols[limit_type]}") .sum() ) diff --git a/scripts/retrieve_cost_data.py b/scripts/retrieve_cost_data.py index 39fcab7c..e236fbfd 100644 --- a/scripts/retrieve_cost_data.py +++ b/scripts/retrieve_cost_data.py @@ -26,7 +26,7 @@ if __name__ == "__main__": version = snakemake.params.version if "/" in version: - baseurl = f"https://raw.githubusercontent.com/{version}/outputs" + baseurl = f"https://raw.githubusercontent.com/{version}/outputs/" else: baseurl = f"https://raw.githubusercontent.com/PyPSA/technology-data/{version}/outputs/" filepath = Path(snakemake.output[0]) From 22d1bf5dd1a35554d476263c91d9bb390881dc2f Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Mon, 18 Mar 2024 11:58:39 +0100 Subject: [PATCH 158/173] Bugfix for missing eia years First, cannot cast pd index to boolean. Second, fix logic error of mistakenly swapped difference in calculating missing years. --- scripts/build_hydro_profile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index b7f270b3..c4e9701e 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -202,9 +202,10 @@ if __name__ == "__main__": contained_years = pd.date_range(freq="YE", **snakemake.params.snapshots).year norm_year = config_hydro.get("eia_norm_year") + missing_years = contained_years.difference(eia_stats.index) if norm_year: eia_stats.loc[contained_years] = eia_stats.loc[norm_year] - elif missing_years := eia_stats.index.difference(contained_years): + elif missing_years.any(): eia_stats.loc[missing_years] = eia_stats.median() inflow = cutout.runoff( From fcf773c003254975740340bb8cb21827aef7fc8b Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Tue, 19 Mar 2024 08:28:14 +0000 Subject: [PATCH 159/173] Copy, don't move natura.tiff to resources Preventing problems when parallel processes attempt to move the same file to different resources directories. --- rules/retrieve.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index ac296f79..4b244483 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -115,7 +115,7 @@ if config["enable"]["retrieve"] and config["enable"].get( mem_mb=5000, retries: 2 run: - move(input[0], output[0]) + copyfile(input[0], output[0]) validate_checksum(output[0], input[0]) From 23e1139c212aa449059ae82f975555142e20e5ec Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 Mar 2024 09:39:35 +0100 Subject: [PATCH 160/173] snakefile + helpers: separate scenario and rdir getter --- Snakefile | 5 +++-- scripts/_helpers.py | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Snakefile b/Snakefile index 4a41622e..3e8875d3 100644 --- a/Snakefile +++ b/Snakefile @@ -10,7 +10,7 @@ from snakemake.utils import min_version min_version("8.5") -from scripts._helpers import path_provider, copy_default_files, process_run_config +from scripts._helpers import path_provider, copy_default_files, get_scenarios, get_rdir copy_default_files(workflow) @@ -21,7 +21,8 @@ configfile: "config/config.yaml" run = config["run"] -RDIR = process_run_config(run) +scenarios = get_scenarios(run) +RDIR = get_rdir(run) logs = path_provider("logs/", RDIR, run["shared_resources"]) benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"]) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 2bacf5cd..9b390380 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -39,14 +39,21 @@ def copy_default_files(workflow): copyfile(template, target) -def process_run_config(run): - scenarios = run.get("scenarios", {}) - if run["name"] and scenarios.get("enable"): - fn = Path(scenarios["file"]) +def get_scenarios(run): + scenario_config = run.get("scenarios", {}) + if run["name"] and scenario_config.get("enable"): + fn = Path(scenario_config["file"]) scenarios = yaml.safe_load(fn.read_text()) - RDIR = "{run}/" if run["name"] == "all": run["name"] = list(scenarios.keys()) + return scenarios + return {} + + +def get_rdir(run): + scenario_config = run.get("scenarios", {}) + if run["name"] and scenario_config.get("enable"): + RDIR = "{run}/" elif run["name"]: RDIR = run["name"] + "/" else: From 8190fde9cee654a8dafd1fd2df09c18e116e4372 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 Mar 2024 09:48:52 +0100 Subject: [PATCH 161/173] Snakefile: reinsert shutil and os functions --- Snakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 3e8875d3..4f55085f 100644 --- a/Snakefile +++ b/Snakefile @@ -4,8 +4,8 @@ from pathlib import Path import yaml -from os.path import normpath -from shutil import move, rmtree +from os.path import normpath, exists +from shutil import copyfile, move, rmtree from snakemake.utils import min_version min_version("8.5") From 86a6c3223def9ef304988eb9b0a53c973005bbf3 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 Mar 2024 11:26:36 +0100 Subject: [PATCH 162/173] perfect foresight: store configs in results dir --- rules/solve_perfect.smk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 215757fe..d4cbd6f3 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -110,7 +110,8 @@ rule solve_sector_network_perfect: output: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", - config="configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.yaml", + config=RESULTS + + "configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.yaml", threads: solver_threads resources: mem_mb=config_provider("solving", "mem"), From 651e28775ecbe50f9918ea18bbb06edc57d58ad9 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Tue, 19 Mar 2024 13:00:16 +0000 Subject: [PATCH 163/173] Use raw strings to avoid illegal backslash warnings python 12 --- Snakefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Snakefile b/Snakefile index 58cb6677..ca1b72cc 100644 --- a/Snakefile +++ b/Snakefile @@ -56,9 +56,9 @@ localrules: wildcard_constraints: simpl="[a-zA-Z0-9]*", clusters="[0-9]+(m|c)?|all", - ll="(v|c)([0-9\.]+|opt)", - opts="[-+a-zA-Z0-9\.]*", - sector_opts="[-+a-zA-Z0-9\.\s]*", + ll=r"(v|c)([0-9\.]+|opt)", + opts=r"[-+a-zA-Z0-9\.]*", + sector_opts=r"[-+a-zA-Z0-9\.\s]*", include: "rules/common.smk" @@ -127,7 +127,7 @@ rule dag: conda: "envs/environment.yaml" shell: - """ + r""" snakemake --rulegraph all | sed -n "/digraph/,\$p" > {output.dot} dot -Tpdf -o {output.pdf} {output.dot} dot -Tpng -o {output.png} {output.dot} From c5026aaf9b24a4ec17049e03cf2f6953b7059f02 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 20 Mar 2024 13:56:48 +0100 Subject: [PATCH 164/173] add draft HVDC projects from TYNDP 2024 --- data/links_tyndp.csv | 13 +++++++++++++ doc/release_notes.rst | 3 +++ 2 files changed, 16 insertions(+) diff --git a/data/links_tyndp.csv b/data/links_tyndp.csv index a0603120..43030be5 100644 --- a/data/links_tyndp.csv +++ b/data/links_tyndp.csv @@ -26,3 +26,16 @@ NordBalt,Klaipeda (LT),Nybro (SE),450,,700,built,,https://en.wikipedia.org/wiki/ Estlink 1,Harku (EE),Espoo (FI),105,,350,built,,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889 Greenlink,Waterford (IE),Pembroke (UK),,180,500,under construction,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/286,-6.987,52.260,-4.986,51.686 Celtic Interconnector,Aghada (IE),La Martyre (FR),,572,700,under consideration,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/107,-8.16642,51.91413,-4.184,48.459 +GiLA,Bordeaux (FR),Nantes (FR),,312,640,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-1.209,46.901,-0.576,44.960 +HG North Tyrrhenian Corridor,Milan (IT),Viterbo (IT),,500,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,9.409,45.553,12.015,42.244 +HG Adriatic Corridor,Ferrara (IT),Foggia (IT),,582,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,11.661,44.855,15.550,41.513 +SAPEI 2,Fioumesanto (IT),Montalto (IT),,390,1000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,8.283,40.790,11.602,42.331 +HG Ionian-Tyrrhenian Corridor,Rossano (IT),Latina (IT),,496,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,16.629,39.568,12.779,41.430 +HG Ionian-Tyrrhenian Corridor 2,Rossano (IT),Catania (IT),,330,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,16.629,39.568,15.049,37.408 +Germany-UK Hybrid Interconnector,Fetteresso (UK),Emden (DE),800,,2000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-2.383,56.991,7.207,53.376 +NU-Link Interconnector,Hornsea (UK),Moerdijk (NL),,460,1200,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-0.261,53.655,4.586,51.661 +APOLLO-LINK,La Farga (ES),La Spezia (IT),,725,2091,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,2.883,42.062,9.884,44.107 +Baltic WindConnector (BWC),Lubmin (DE),Lihula (EE),,960,2000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,13.686,54.139,23.818,58.675 +High-Voltage Direct Current Interconnector Project Romania-Hungary,Constanta (RO),Albertirsa (HU),,930,2500,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,28.588,44.201,19.584,47.224 +Rhine-Main-Link,Ovelgönne (DE),Marxheim (DE),,433,4000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,8.379,53.315,8.435,50.078 +Green Aegean Interconnector,Arachthos (GR),Ottenhofen (DE),,600,3000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,20.967,39.185,11.868,48.207 diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 21094250..e081ce35 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -59,6 +59,9 @@ Upcoming Release * Removed rule ``copy_config``. Instead, a config file is created for each network output of the ``solve_*`` rules, with the same content as ``n.meta``. +* Added new HVDC transmission projects from `TYNDP 2024 draft projects + `_. + * Upgrade to Snakemake v8.5+. This version is the new minimum version required. To upgrade an existing environment, run ``conda install -c bioconda snakemake-minimal">=8.5"`` and ``pip install snakemake-storage-plugin-http`` From bb898bf69519c1b3c6454ff3248f2d42df543c2d Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Wed, 20 Mar 2024 15:40:07 +0100 Subject: [PATCH 165/173] Fix typo in reading input to build_sequestration_potentials rule Presumably needed since the transition to snakemake 8 storage providers --- scripts/build_sequestration_potentials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 106c1271..2a362a77 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -44,7 +44,7 @@ if __name__ == "__main__": cf = snakemake.params.sequestration_potential - gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) + gdf = gpd.read_file(snakemake.input.sequestration_potential) regions = gpd.read_file(snakemake.input.regions_offshore) if cf["include_onshore"]: From ab8336d6a891c89b2154a540ab10edcb61c73a06 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Thu, 21 Mar 2024 09:55:48 +0100 Subject: [PATCH 166/173] disable windows machines in CI The CI on windows machines keeps failing due to some weird syntax problem in snakefiles, which we can't solve on our end it seems. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c17c0425..7aabf0e6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,7 +31,7 @@ jobs: os: - ubuntu-latest - macos-latest - - windows-latest +# - windows-latest inhouse: - stable - master From d0951abdedd11c8ae4c044905673d270e56dc52e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 21 Mar 2024 12:02:02 +0100 Subject: [PATCH 167/173] cluster_network: allow more solvers that can handle quadratic problems --- doc/release_notes.rst | 2 ++ scripts/cluster_network.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 21094250..2982f0a6 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,8 @@ Release Notes Upcoming Release ================ +* Allow the use of more solvers in clustering (Xpress, COPT, Gurobi, CPLEX, SCIP, MOSEK). + * Enhanced support for choosing different weather years (https://github.com/PyPSA/pypsa-eur/pull/204): diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 270774af..3f6c9ea2 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -267,7 +267,7 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="scip"): m.objective = (clusters * clusters - 2 * clusters * L * n_clusters).sum() if solver_name == "gurobi": logging.getLogger("gurobipy").propagate = False - elif solver_name not in ["scip", "cplex"]: + elif solver_name not in ["scip", "cplex", "xpress", "copt", "mosek"]: logger.info( f"The configured solver `{solver_name}` does not support quadratic objectives. Falling back to `scip`." ) From fd01cb0a756d769c1d638b38c372ae517814d097 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 22 Mar 2024 15:57:37 +0100 Subject: [PATCH 168/173] Fix type error in cluster_network with "m" configuration --- doc/release_notes.rst | 2 ++ scripts/cluster_network.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index b868e5e9..e561cbe0 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -164,6 +164,8 @@ Upcoming Release * Fix duplicated years in `add_land_use_constraint_m`. +* Fix type error with `m` option in `cluster_network`. + PyPSA-Eur 0.10.0 (19th February 2024) ===================================== diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 3f6c9ea2..22fb4176 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -471,7 +471,7 @@ if __name__ == "__main__": conventional_carriers = set(params.conventional_carriers) if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) - aggregate_carriers = params.conventional_carriers & aggregate_carriers + aggregate_carriers = conventional_carriers & aggregate_carriers elif snakemake.wildcards.clusters.endswith("c"): n_clusters = int(snakemake.wildcards.clusters[:-1]) aggregate_carriers = aggregate_carriers - conventional_carriers From 585f9a32a7a18d47d3b35c39169d10019f129f4d Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 25 Mar 2024 14:07:02 +0100 Subject: [PATCH 169/173] cluster_network: ensure correct indexing of weights --- scripts/cluster_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 22fb4176..903cbe9b 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -231,7 +231,7 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="scip"): .pipe(normed) ) - N = n.buses.groupby(["country", "sub_network"]).size() + N = n.buses.groupby(["country", "sub_network"]).size()[L.index] assert ( n_clusters >= len(N) and n_clusters <= N.sum() @@ -454,7 +454,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("cluster_network", simpl="", clusters="5") + snakemake = mock_snakemake("cluster_network", simpl="", clusters="40") configure_logging(snakemake) set_scenario_config(snakemake) From 34f02942640504dbbfc2befead9d5820a783cca2 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 25 Mar 2024 14:08:38 +0100 Subject: [PATCH 170/173] scenario management: fix shared resources --- scripts/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 0fd5ab2b..8e8d67c8 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -106,7 +106,7 @@ def get_run_path(fn, dir, rdir, shared_resources): elif isinstance(shared_resources, str): rdir = shared_resources + "/" elif isinstance(shared_resources, bool): - rdir = "" + rdir = "" if shared_resources else rdir else: raise ValueError( "shared_resources must be a boolean, str, or 'base' for special handling." From 8b2a068b3a4c4411c8fde1ff0e46a498cae7f1f8 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Tue, 26 Mar 2024 11:57:21 +0100 Subject: [PATCH 171/173] Check if scenario file exists before attempting to load Useful when running the `create_scenarios` rule the first time. --- scripts/_helpers.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 8e8d67c8..f063877e 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -43,10 +43,11 @@ def get_scenarios(run): scenario_config = run.get("scenarios", {}) if run["name"] and scenario_config.get("enable"): fn = Path(scenario_config["file"]) - scenarios = yaml.safe_load(fn.read_text()) - if run["name"] == "all": - run["name"] = list(scenarios.keys()) - return scenarios + if fn.exists(): + scenarios = yaml.safe_load(fn.read_text()) + if run["name"] == "all": + run["name"] = list(scenarios.keys()) + return scenarios return {} From be7dc0d0bbf7047002d87828f823d81022d6e474 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Tue, 26 Mar 2024 13:28:06 +0100 Subject: [PATCH 172/173] postprocess: fix typo in benchmark folder --- rules/postprocess.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 2b298610..e7df2e66 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -44,7 +44,7 @@ if config["foresight"] != "perfect": benchmark: ( RESULTS - + "benchmarksplot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + + "benchmarks/plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" From 89dbd53805680427231c96904655526548ee0021 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Tue, 26 Mar 2024 17:20:10 +0100 Subject: [PATCH 173/173] provide Path object as input to ConfigSettings --- doc/release_notes.rst | 2 ++ scripts/_helpers.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index e561cbe0..e08981d3 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,8 @@ Release Notes Upcoming Release ================ +* bugfix: convert Strings to pathlib.Path objects as input to ConfigSettings + * Allow the use of more solvers in clustering (Xpress, COPT, Gurobi, CPLEX, SCIP, MOSEK). * Enhanced support for choosing different weather years diff --git a/scripts/_helpers.py b/scripts/_helpers.py index f063877e..64ccab87 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -427,7 +427,7 @@ def mock_snakemake( configfiles = [configfiles] resource_settings = ResourceSettings() - config_settings = ConfigSettings(configfiles=configfiles) + config_settings = ConfigSettings(configfiles=map(Path, configfiles)) workflow_settings = WorkflowSettings() storage_settings = StorageSettings() dag_settings = DAGSettings(rerun_triggers=[])