From 4fef5616e887c0c9e0cdbee484bf665c763e27bb Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 10 Nov 2020 09:48:13 +0100 Subject: [PATCH 01/31] scripts: change mock snakemake defaults --- scripts/add_brownfield.py | 12 ++++++------ scripts/add_existing_baseyear.py | 14 +++++++------- scripts/prepare_sector_network.py | 24 ++++++++++++------------ scripts/solve_network.py | 8 ++++---- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index a0b8d97b..a3624395 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -86,16 +86,16 @@ if __name__ == "__main__": if 'snakemake' not in globals(): from vresutils.snakemake import MockSnakemake snakemake = MockSnakemake( - wildcards=dict(network='elec', simpl='', clusters='37', lv='1.0', + wildcards=dict(simpl='', clusters='37', lv='1.0', sector_opts='Co2L0-168H-T-H-B-I-solar3-dist1', co2_budget_name='go', planning_horizons='2030'), - input=dict(network='pypsa-eur-sec/results/test/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', - network_p='pypsa-eur-sec/results/test/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_2020.nc', + input=dict(network='pypsa-eur-sec/results/test/prenetworks/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', + network_p='pypsa-eur-sec/results/test/postnetworks/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_2020.nc', costs='pypsa-eur-sec/data/costs/costs_{planning_horizons}.csv', - cop_air_total="pypsa-eur-sec/resources/cop_air_total_{network}_s{simpl}_{clusters}.nc", - cop_soil_total="pypsa-eur-sec/resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc"), - output=['pypsa-eur-sec/results/test/prenetworks_brownfield/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{planning_horizons}.nc'] + cop_air_total="pypsa-eur-sec/resources/cop_air_total_elec_s{simpl}_{clusters}.nc", + cop_soil_total="pypsa-eur-sec/resources/cop_soil_total_elec_s{simpl}_{clusters}.nc"), + output=['pypsa-eur-sec/results/test/prenetworks_brownfield/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{planning_horizons}.nc'] ) import yaml with open('config.yaml', encoding='utf8') as f: diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index cfef2e5e..9eb2ae3a 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -408,18 +408,18 @@ if __name__ == "__main__": if 'snakemake' not in globals(): from vresutils.snakemake import MockSnakemake snakemake = MockSnakemake( - wildcards=dict(network='elec', simpl='', clusters='39', lv='1.0', + wildcards=dict(simpl='', clusters='39', lv='1.0', sector_opts='Co2L0-168H-T-H-B-I-solar3-dist1', co2_budget_name='b30b3', planning_horizons='2020'), - input=dict(network='pypsa-eur-sec/results/test/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', + input=dict(network='pypsa-eur-sec/results/test/prenetworks/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', powerplants='pypsa-eur/resources/powerplants.csv', - busmap_s='pypsa-eur/resources/busmap_{network}_s{simpl}.csv', - busmap='pypsa-eur/resources/busmap_{network}_s{simpl}_{clusters}.csv', + busmap_s='pypsa-eur/resources/busmap_elec_s{simpl}.csv', + busmap='pypsa-eur/resources/busmap_elec_s{simpl}_{clusters}.csv', costs='pypsa-eur-sec/data/costs/costs_{planning_horizons}.csv', - cop_air_total="pypsa-eur-sec/resources/cop_air_total_{network}_s{simpl}_{clusters}.nc", - cop_soil_total="pypsa-eur-sec/resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc"), - output=['pypsa-eur-sec/results/test/prenetworks_brownfield/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{planning_horizons}.nc'], + cop_air_total="pypsa-eur-sec/resources/cop_air_total_elec_s{simpl}_{clusters}.nc", + cop_soil_total="pypsa-eur-sec/resources/cop_soil_total_elec_s{simpl}_{clusters}.nc"), + output=['pypsa-eur-sec/results/test/prenetworks_brownfield/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{planning_horizons}.nc'], ) import yaml with open('config.yaml', encoding='utf8') as f: diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index f6e8a7be..990b6710 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1745,31 +1745,31 @@ if __name__ == "__main__": if 'snakemake' not in globals(): from vresutils.snakemake import MockSnakemake snakemake = MockSnakemake( - wildcards=dict(network='elec', simpl='', clusters='37', lv='1.0', + wildcards=dict(simpl='', clusters='37', lv='1.0', opts='', planning_horizons='2020', co2_budget_name='go', sector_opts='Co2L0-168H-T-H-B-I-solar3-dist1'), - input=dict(network='pypsa-eur/networks/{network}_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc', + input=dict(network='pypsa-eur/networks/elec_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc', timezone_mappings='pypsa-eur-sec/data/timezone_mappings.csv', co2_budget='pypsa-eur-sec/data/co2_budget.csv', - clustered_pop_layout='pypsa-eur-sec/resources/pop_layout_{network}_s{simpl}_{clusters}.csv', + clustered_pop_layout='pypsa-eur-sec/resources/pop_layout_elec_s{simpl}_{clusters}.csv', costs='technology-data/outputs/costs_{planning_horizons}.csv', profile_offwind_ac='pypsa-eur/resources/profile_offwind-ac.nc', profile_offwind_dc='pypsa-eur/resources/profile_offwind-dc.nc', - busmap_s='pypsa-eur/resources/busmap_{network}_s{simpl}.csv', - busmap='pypsa-eur/resources/busmap_{network}_s{simpl}_{clusters}.csv', - cop_air_total='pypsa-eur-sec/resources/cop_air_total_{network}_s{simpl}_{clusters}.nc', - cop_soil_total='pypsa-eur-sec/resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc', - solar_thermal_total='pypsa-eur-sec/resources/solar_thermal_total_{network}_s{simpl}_{clusters}.nc', + busmap_s='pypsa-eur/resources/busmap_elec_s{simpl}.csv', + busmap='pypsa-eur/resources/busmap_elec_s{simpl}_{clusters}.csv', + cop_air_total='pypsa-eur-sec/resources/cop_air_total_elec_s{simpl}_{clusters}.nc', + cop_soil_total='pypsa-eur-sec/resources/cop_soil_total_elec_s{simpl}_{clusters}.nc', + solar_thermal_total='pypsa-eur-sec/resources/solar_thermal_total_elec_s{simpl}_{clusters}.nc', energy_totals_name='pypsa-eur-sec/data/energy_totals.csv', - heat_demand_total='pypsa-eur-sec/resources/heat_demand_total_{network}_s{simpl}_{clusters}.nc', + heat_demand_total='pypsa-eur-sec/resources/heat_demand_total_elec_s{simpl}_{clusters}.nc', heat_profile='pypsa-eur-sec/data/heat_load_profile_BDEW.csv', transport_name='pypsa-eur-sec/data/transport_data.csv', - temp_air_total='pypsa-eur-sec/resources/temp_air_total_{network}_s{simpl}_{clusters}.nc', + temp_air_total='pypsa-eur-sec/resources/temp_air_total_elec_s{simpl}_{clusters}.nc', co2_totals_name='pypsa-eur-sec/data/co2_totals.csv', biomass_potentials='pypsa-eur-sec/data/biomass_potentials.csv', - industrial_demand='pypsa-eur-sec/resources/industrial_demand_{network}_s{simpl}_{clusters}.csv',), - output=['pypsa-eur-sec/results/test/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc'] + industrial_demand='pypsa-eur-sec/resources/industrial_demand_elec_s{simpl}_{clusters}.csv',), + output=['pypsa-eur-sec/results/test/prenetworks/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc'] ) import yaml with open('config.yaml', encoding='utf8') as f: diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a7078433..b2c56d54 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -369,13 +369,13 @@ if __name__ == "__main__": if 'snakemake' not in globals(): from vresutils.snakemake import MockSnakemake, Dict snakemake = MockSnakemake( - wildcards=dict(network='elec', simpl='', clusters='39', lv='1.0', + wildcards=dict(simpl='', clusters='39', lv='1.0', sector_opts='Co2L0-168H-T-H-B-I-solar3-dist1', co2_budget_name='b30b3', planning_horizons='2050'), - input=dict(network="pypsa-eur-sec/results/test/prenetworks_brownfield/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc"), + input=dict(network="pypsa-eur-sec/results/test/prenetworks_brownfield/elec_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc"), output=["results/networks/s{simpl}_{clusters}_lv{lv}_{sector_opts}_{co2_budget_name}_{planning_horizons}-test.nc"], - log=dict(gurobi="logs/{network}_s{simpl}_{clusters}_lv{lv}_{sector_opts}_{co2_budget_name}_{planning_horizons}_gurobi-test.log", - python="logs/{network}_s{simpl}_{clusters}_lv{lv}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python-test.log") + log=dict(gurobi="logs/elec_s{simpl}_{clusters}_lv{lv}_{sector_opts}_{co2_budget_name}_{planning_horizons}_gurobi-test.log", + python="logs/elec_s{simpl}_{clusters}_lv{lv}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python-test.log") ) import yaml with open('config.yaml', encoding='utf8') as f: From 723afc32e9b292f5cc543c04c28bd2302f17176c Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 10 Nov 2020 09:51:02 +0100 Subject: [PATCH 02/31] scripts: step towards support for subset of countries --- config.default.yaml | 3 +++ config.myopic.yaml | 3 +++ scripts/build_industrial_energy_demand_per_country_today.py | 2 ++ scripts/build_industrial_production_per_country.py | 1 + 4 files changed, 9 insertions(+) diff --git a/config.default.yaml b/config.default.yaml index 7aa23901..ead33e2b 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -11,6 +11,7 @@ foresight: 'overnight' #options are overnight, myopic, perfect (perfect is not y scenario: sectors: [E] # ignore this legacy setting + year: [''] # weather year simpl: [''] # only relevant for PyPSA-Eur lv: [1.0,1.5] # allowed transmission line volume expansion, can be any float >= 1.0 (today) or "opt" clusters: [45,50] # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred @@ -34,6 +35,8 @@ snapshots: end: "2014-01-01" closed: 'left' # end is not inclusive +countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] + atlite: cutout_dir: '../pypsa-eur/cutouts' cutout_name: "europe-2013-era5" diff --git a/config.myopic.yaml b/config.myopic.yaml index c7f5ac1e..0dc4cdf3 100644 --- a/config.myopic.yaml +++ b/config.myopic.yaml @@ -11,6 +11,7 @@ foresight: 'myopic' #options are overnight, myopic, perfect (perfect is not yet scenario: sectors: [E] # ignore this legacy setting + year: [''] # weather year simpl: [''] # only relevant for PyPSA-Eur lv: [1.0,1.5] # allowed transmission line volume expansion, can be any float >= 1.0 (today) or "opt" clusters: [45,50] # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred @@ -34,6 +35,8 @@ snapshots: end: "2014-01-01" closed: 'left' # end is not inclusive +countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] + atlite: cutout_dir: '../pypsa-eur/cutouts' cutout_name: "europe-2013-era5" diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 7593477b..00611318 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -61,6 +61,8 @@ eu28 = ['FR', 'DE', 'GB', 'IT', 'ES', 'PL', 'SE', 'NL', 'BE', 'FI', 'DK', 'PT', 'RO', 'AT', 'BG', 'EE', 'GR', 'LV', 'CZ', 'HU', 'IE', 'SK', 'LT', 'HR', 'LU', 'SI', 'CY', 'MT'] +eu28 = list(set(eu28).intersection(snakemake.config["countries"])) + jrc_names = {"GR" : "EL", "GB" : "UK"} diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index c8fa6910..4a993a91 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -39,6 +39,7 @@ eu28 = ['FR', 'DE', 'GB', 'IT', 'ES', 'PL', 'SE', 'NL', 'BE', 'FI', countries = non_EU + eu28 +countries = list(set(countries).intersection(snakemake.config["countries"])) sectors = ['Iron and steel','Chemicals Industry','Non-metallic mineral products', 'Pulp, paper and printing', 'Food, beverages and tobacco', 'Non Ferrous Metals', From 59cb0361222dbd457307baf68b62d58075b46537 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 10 Nov 2020 09:51:32 +0100 Subject: [PATCH 03/31] energy_totals: only fix 'BA' if in list of countries --- scripts/build_energy_totals.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 1682ac40..0cfa709b 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -378,12 +378,12 @@ def build_energy_totals(): clean_df.loc[missing,"total aviation passenger"] = clean_df.loc[missing,["total domestic aviation passenger","total international aviation passenger"]].sum(axis=1) clean_df.loc[missing,"total aviation freight"] = clean_df.loc[missing,["total domestic aviation freight","total international aviation freight"]].sum(axis=1) + if "BA" in clean_df.index: + #fix missing data for BA (services and road energy data) + missing = (clean_df.loc["BA"] == 0.) - #fix missing data for BA (services and road energy data) - missing = (clean_df.loc["BA"] == 0.) - - #add back in proportional to RS with ratio of total residential demand - clean_df.loc["BA",missing] = clean_df.loc["BA","total residential"]/clean_df.loc["RS","total residential"]*clean_df.loc["RS",missing] + #add back in proportional to RS with ratio of total residential demand + clean_df.loc["BA",missing] = clean_df.loc["BA","total residential"]/clean_df.loc["RS","total residential"]*clean_df.loc["RS",missing] clean_df.to_csv(snakemake.output.energy_name) From df94ea1c8bce3f97d696a08ae202585a32f26dee Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 10 Nov 2020 09:52:06 +0100 Subject: [PATCH 04/31] add multiyear support --- Snakefile | 244 +++++++++--------- scripts/build_clustered_population_layouts.py | 5 +- scripts/build_heat_demand.py | 10 +- scripts/build_population_layouts.py | 6 +- scripts/build_solar_thermal_profiles.py | 9 +- scripts/build_temperature_profiles.py | 9 +- 6 files changed, 153 insertions(+), 130 deletions(-) diff --git a/Snakefile b/Snakefile index f003bd5e..2d5564a8 100644 --- a/Snakefile +++ b/Snakefile @@ -2,8 +2,8 @@ configfile: "config.yaml" wildcard_constraints: + year="[0-9]*", lv="[a-z0-9\.]+", - network="[a-zA-Z0-9]*", simpl="[a-zA-Z0-9]*", clusters="[0-9]+m?", sectors="[+a-zA-Z0-9]+", @@ -25,17 +25,17 @@ rule all: rule solve_all_networks: input: - expand(config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", + expand(config['results_dir'] + config['run'] + "/postnetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", **config['scenario']) rule test_script: input: - expand("resources/heat_demand_urban_elec_s_{clusters}.nc", + expand("resources/heat_demand_urban_elec{year}_s_{clusters}.nc", **config['scenario']) rule prepare_sector_networks: input: - expand(config['results_dir'] + config['run'] + "/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", + expand(config['results_dir'] + config['run'] + "/prenetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", **config['scenario']) @@ -44,96 +44,96 @@ rule build_population_layouts: nuts3_shapes=pypsaeur('resources/nuts3_shapes.geojson'), urban_percent="data/urban_percent.csv" output: - pop_layout_total="resources/pop_layout_total.nc", - pop_layout_urban="resources/pop_layout_urban.nc", - pop_layout_rural="resources/pop_layout_rural.nc" + pop_layout_total="resources/pop_layout{year}_total.nc", + pop_layout_urban="resources/pop_layout{year}_urban.nc", + pop_layout_rural="resources/pop_layout{year}_rural.nc" resources: mem_mb=20000 script: "scripts/build_population_layouts.py" rule build_clustered_population_layouts: input: - pop_layout_total="resources/pop_layout_total.nc", - pop_layout_urban="resources/pop_layout_urban.nc", - pop_layout_rural="resources/pop_layout_rural.nc", - regions_onshore=pypsaeur('resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson') + pop_layout_total="resources/pop_layout{year}_total.nc", + pop_layout_urban="resources/pop_layout{year}_urban.nc", + pop_layout_rural="resources/pop_layout{year}_rural.nc", + regions_onshore=pypsaeur('resources/regions_onshore_elec{year}_s{simpl}_{clusters}.geojson') output: - clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv" + clustered_pop_layout="resources/pop_layout_elec{year}_s{simpl}_{clusters}.csv" resources: mem_mb=10000 script: "scripts/build_clustered_population_layouts.py" rule build_simplified_population_layouts: input: - pop_layout_total="resources/pop_layout_total.nc", - pop_layout_urban="resources/pop_layout_urban.nc", - pop_layout_rural="resources/pop_layout_rural.nc", - regions_onshore=pypsaeur('resources/regions_onshore_{network}_s{simpl}.geojson') + pop_layout_total="resources/pop_layout{year}_total.nc", + pop_layout_urban="resources/pop_layout{year}_urban.nc", + pop_layout_rural="resources/pop_layout{year}_rural.nc", + regions_onshore=pypsaeur('resources/regions_onshore_elec{year}_s{simpl}.geojson') output: - clustered_pop_layout="resources/pop_layout_{network}_s{simpl}.csv" + clustered_pop_layout="resources/pop_layout_elec{year}_s{simpl}.csv" resources: mem_mb=10000 script: "scripts/build_clustered_population_layouts.py" rule build_heat_demands: input: - pop_layout_total="resources/pop_layout_total.nc", - pop_layout_urban="resources/pop_layout_urban.nc", - pop_layout_rural="resources/pop_layout_rural.nc", - regions_onshore=pypsaeur("resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson") + pop_layout_total="resources/pop_layout{year}_total.nc", + pop_layout_urban="resources/pop_layout{year}_urban.nc", + pop_layout_rural="resources/pop_layout{year}_rural.nc", + regions_onshore=pypsaeur("resources/regions_onshore_elec{year}_s{simpl}_{clusters}.geojson") output: - heat_demand_urban="resources/heat_demand_urban_{network}_s{simpl}_{clusters}.nc", - heat_demand_rural="resources/heat_demand_rural_{network}_s{simpl}_{clusters}.nc", - heat_demand_total="resources/heat_demand_total_{network}_s{simpl}_{clusters}.nc" + heat_demand_urban="resources/heat_demand_urban_elec{year}_s{simpl}_{clusters}.nc", + heat_demand_rural="resources/heat_demand_rural_elec{year}_s{simpl}_{clusters}.nc", + heat_demand_total="resources/heat_demand_total_elec{year}_s{simpl}_{clusters}.nc" resources: mem_mb=20000 script: "scripts/build_heat_demand.py" rule build_temperature_profiles: input: - pop_layout_total="resources/pop_layout_total.nc", - pop_layout_urban="resources/pop_layout_urban.nc", - pop_layout_rural="resources/pop_layout_rural.nc", - regions_onshore=pypsaeur("resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson") + pop_layout_total="resources/pop_layout{year}_total.nc", + pop_layout_urban="resources/pop_layout{year}_urban.nc", + pop_layout_rural="resources/pop_layout{year}_rural.nc", + regions_onshore=pypsaeur("resources/regions_onshore_elec{year}_s{simpl}_{clusters}.geojson") output: - temp_soil_total="resources/temp_soil_total_{network}_s{simpl}_{clusters}.nc", - temp_soil_rural="resources/temp_soil_rural_{network}_s{simpl}_{clusters}.nc", - temp_soil_urban="resources/temp_soil_urban_{network}_s{simpl}_{clusters}.nc", - temp_air_total="resources/temp_air_total_{network}_s{simpl}_{clusters}.nc", - temp_air_rural="resources/temp_air_rural_{network}_s{simpl}_{clusters}.nc", - temp_air_urban="resources/temp_air_urban_{network}_s{simpl}_{clusters}.nc" + temp_soil_total="resources/temp_soil_total_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_rural="resources/temp_soil_rural_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_urban="resources/temp_soil_urban_elec{year}_s{simpl}_{clusters}.nc", + temp_air_total="resources/temp_air_total_elec{year}_s{simpl}_{clusters}.nc", + temp_air_rural="resources/temp_air_rural_elec{year}_s{simpl}_{clusters}.nc", + temp_air_urban="resources/temp_air_urban_elec{year}_s{simpl}_{clusters}.nc" resources: mem_mb=20000 script: "scripts/build_temperature_profiles.py" rule build_cop_profiles: input: - temp_soil_total="resources/temp_soil_total_{network}_s{simpl}_{clusters}.nc", - temp_soil_rural="resources/temp_soil_rural_{network}_s{simpl}_{clusters}.nc", - temp_soil_urban="resources/temp_soil_urban_{network}_s{simpl}_{clusters}.nc", - temp_air_total="resources/temp_air_total_{network}_s{simpl}_{clusters}.nc", - temp_air_rural="resources/temp_air_rural_{network}_s{simpl}_{clusters}.nc", - temp_air_urban="resources/temp_air_urban_{network}_s{simpl}_{clusters}.nc" + temp_soil_total="resources/temp_soil_total_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_rural="resources/temp_soil_rural_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_urban="resources/temp_soil_urban_elec{year}_s{simpl}_{clusters}.nc", + temp_air_total="resources/temp_air_total_elec{year}_s{simpl}_{clusters}.nc", + temp_air_rural="resources/temp_air_rural_elec{year}_s{simpl}_{clusters}.nc", + temp_air_urban="resources/temp_air_urban_elec{year}_s{simpl}_{clusters}.nc" output: - cop_soil_total="resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc", - cop_soil_rural="resources/cop_soil_rural_{network}_s{simpl}_{clusters}.nc", - cop_soil_urban="resources/cop_soil_urban_{network}_s{simpl}_{clusters}.nc", - cop_air_total="resources/cop_air_total_{network}_s{simpl}_{clusters}.nc", - cop_air_rural="resources/cop_air_rural_{network}_s{simpl}_{clusters}.nc", - cop_air_urban="resources/cop_air_urban_{network}_s{simpl}_{clusters}.nc" + cop_soil_total="resources/cop_soil_total_elec{year}_s{simpl}_{clusters}.nc", + cop_soil_rural="resources/cop_soil_rural_elec{year}_s{simpl}_{clusters}.nc", + cop_soil_urban="resources/cop_soil_urban_elec{year}_s{simpl}_{clusters}.nc", + cop_air_total="resources/cop_air_total_elec{year}_s{simpl}_{clusters}.nc", + cop_air_rural="resources/cop_air_rural_elec{year}_s{simpl}_{clusters}.nc", + cop_air_urban="resources/cop_air_urban_elec{year}_s{simpl}_{clusters}.nc" resources: mem_mb=20000 script: "scripts/build_cop_profiles.py" rule build_solar_thermal_profiles: input: - pop_layout_total="resources/pop_layout_total.nc", - pop_layout_urban="resources/pop_layout_urban.nc", - pop_layout_rural="resources/pop_layout_rural.nc", - regions_onshore=pypsaeur("resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson") + pop_layout_total="resources/pop_layout{year}_total.nc", + pop_layout_urban="resources/pop_layout{year}_urban.nc", + pop_layout_rural="resources/pop_layout{year}_rural.nc", + regions_onshore=pypsaeur("resources/regions_onshore_elec{year}_s{simpl}_{clusters}.geojson") output: - solar_thermal_total="resources/solar_thermal_total_{network}_s{simpl}_{clusters}.nc", - solar_thermal_urban="resources/solar_thermal_urban_{network}_s{simpl}_{clusters}.nc", - solar_thermal_rural="resources/solar_thermal_rural_{network}_s{simpl}_{clusters}.nc" + solar_thermal_total="resources/solar_thermal_total_elec{year}_s{simpl}_{clusters}.nc", + solar_thermal_urban="resources/solar_thermal_urban_elec{year}_s{simpl}_{clusters}.nc", + solar_thermal_rural="resources/solar_thermal_rural_elec{year}_s{simpl}_{clusters}.nc" resources: mem_mb=20000 script: "scripts/build_solar_thermal_profiles.py" @@ -204,12 +204,12 @@ rule build_industrial_production_per_country_tomorrow: rule build_industrial_distribution_key: input: - clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv", + clustered_pop_layout="resources/pop_layout_elec{year}_s{simpl}_{clusters}.csv", europe_shape=pypsaeur('resources/europe_shape.geojson'), hotmaps_industrial_database="data/Industrial_Database.csv", - network=pypsaeur('networks/{network}_s{simpl}_{clusters}.nc') + network=pypsaeur('networks/elec{year}_s{simpl}_{clusters}.nc') output: - industrial_distribution_key="resources/industrial_distribution_key_{network}_s{simpl}_{clusters}.csv" + industrial_distribution_key="resources/industrial_distribution_key_elec{year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=1000 script: 'scripts/build_industrial_distribution_key.py' @@ -218,10 +218,10 @@ rule build_industrial_distribution_key: rule build_industrial_production_per_node: input: - industrial_distribution_key="resources/industrial_distribution_key_{network}_s{simpl}_{clusters}.csv", + industrial_distribution_key="resources/industrial_distribution_key_elec{year}_s{simpl}_{clusters}.csv", industrial_production_per_country_tomorrow="resources/industrial_production_per_country_tomorrow.csv" output: - industrial_production_per_node="resources/industrial_production_{network}_s{simpl}_{clusters}.csv" + industrial_production_per_node="resources/industrial_production_elec{year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=1000 script: 'scripts/build_industrial_production_per_node.py' @@ -230,10 +230,10 @@ rule build_industrial_production_per_node: rule build_industrial_energy_demand_per_node: input: industry_sector_ratios="resources/industry_sector_ratios.csv", - industrial_production_per_node="resources/industrial_production_{network}_s{simpl}_{clusters}.csv", - industrial_energy_demand_per_node_today="resources/industrial_energy_demand_today_{network}_s{simpl}_{clusters}.csv" + industrial_production_per_node="resources/industrial_production_elec{year}_s{simpl}_{clusters}.csv", + industrial_energy_demand_per_node_today="resources/industrial_energy_demand_today_elec{year}_s{simpl}_{clusters}.csv" output: - industrial_energy_demand_per_node="resources/industrial_energy_demand_{network}_s{simpl}_{clusters}.csv" + industrial_energy_demand_per_node="resources/industrial_energy_demand_elec{year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=1000 script: 'scripts/build_industrial_energy_demand_per_node.py' @@ -252,10 +252,10 @@ rule build_industrial_energy_demand_per_country_today: rule build_industrial_energy_demand_per_node_today: input: - industrial_distribution_key="resources/industrial_distribution_key_{network}_s{simpl}_{clusters}.csv", + industrial_distribution_key="resources/industrial_distribution_key_elec{year}_s{simpl}_{clusters}.csv", industrial_energy_demand_per_country_today="resources/industrial_energy_demand_per_country_today.csv" output: - industrial_energy_demand_per_node_today="resources/industrial_energy_demand_today_{network}_s{simpl}_{clusters}.csv" + industrial_energy_demand_per_node_today="resources/industrial_energy_demand_today_elec{year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=1000 script: 'scripts/build_industrial_energy_demand_per_node_today.py' @@ -275,10 +275,10 @@ rule build_industrial_energy_demand_per_country: rule build_industrial_demand: input: - clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv", + clustered_pop_layout="resources/pop_layout_elec{year}_s{simpl}_{clusters}.csv", industrial_demand_per_country="resources/industrial_energy_demand_per_country.csv" output: - industrial_demand="resources/industrial_demand_{network}_s{simpl}_{clusters}.csv" + industrial_demand="resources/industrial_demand_elec{year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=1000 script: 'scripts/build_industrial_demand.py' @@ -286,7 +286,7 @@ rule build_industrial_demand: rule prepare_sector_network: input: - network=pypsaeur('networks/{network}_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc'), + network=pypsaeur('networks/elec{year}_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc'), energy_totals_name='resources/energy_totals.csv', co2_totals_name='resources/co2_totals.csv', transport_name='resources/transport_data.csv', @@ -296,45 +296,45 @@ rule prepare_sector_network: costs=config['costs_dir'] + "costs_{planning_horizons}.csv", h2_cavern = "data/hydrogen_salt_cavern_potentials.csv", co2_budget="data/co2_budget.csv", - profile_offwind_ac=pypsaeur("resources/profile_offwind-ac.nc"), - profile_offwind_dc=pypsaeur("resources/profile_offwind-dc.nc"), - busmap_s=pypsaeur("resources/busmap_{network}_s{simpl}.csv"), - busmap=pypsaeur("resources/busmap_{network}_s{simpl}_{clusters}.csv"), - clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv", - simplified_pop_layout="resources/pop_layout_{network}_s{simpl}.csv", - industrial_demand="resources/industrial_energy_demand_{network}_s{simpl}_{clusters}.csv", - heat_demand_urban="resources/heat_demand_urban_{network}_s{simpl}_{clusters}.nc", - heat_demand_rural="resources/heat_demand_rural_{network}_s{simpl}_{clusters}.nc", - heat_demand_total="resources/heat_demand_total_{network}_s{simpl}_{clusters}.nc", - temp_soil_total="resources/temp_soil_total_{network}_s{simpl}_{clusters}.nc", - temp_soil_rural="resources/temp_soil_rural_{network}_s{simpl}_{clusters}.nc", - temp_soil_urban="resources/temp_soil_urban_{network}_s{simpl}_{clusters}.nc", - temp_air_total="resources/temp_air_total_{network}_s{simpl}_{clusters}.nc", - temp_air_rural="resources/temp_air_rural_{network}_s{simpl}_{clusters}.nc", - temp_air_urban="resources/temp_air_urban_{network}_s{simpl}_{clusters}.nc", - cop_soil_total="resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc", - cop_soil_rural="resources/cop_soil_rural_{network}_s{simpl}_{clusters}.nc", - cop_soil_urban="resources/cop_soil_urban_{network}_s{simpl}_{clusters}.nc", - cop_air_total="resources/cop_air_total_{network}_s{simpl}_{clusters}.nc", - cop_air_rural="resources/cop_air_rural_{network}_s{simpl}_{clusters}.nc", - cop_air_urban="resources/cop_air_urban_{network}_s{simpl}_{clusters}.nc", - solar_thermal_total="resources/solar_thermal_total_{network}_s{simpl}_{clusters}.nc", - solar_thermal_urban="resources/solar_thermal_urban_{network}_s{simpl}_{clusters}.nc", - solar_thermal_rural="resources/solar_thermal_rural_{network}_s{simpl}_{clusters}.nc" - output: config['results_dir'] + config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc' + profile_offwind_ac=pypsaeur("resources/profile{year}_offwind-ac.nc"), + profile_offwind_dc=pypsaeur("resources/profile{year}_offwind-dc.nc"), + busmap_s=pypsaeur("resources/busmap_elec{year}_s{simpl}.csv"), + busmap=pypsaeur("resources/busmap_elec{year}_s{simpl}_{clusters}.csv"), + clustered_pop_layout="resources/pop_layout_elec{year}_s{simpl}_{clusters}.csv", + simplified_pop_layout="resources/pop_layout_elec{year}_s{simpl}.csv", + industrial_demand="resources/industrial_energy_demand_elec{year}_s{simpl}_{clusters}.csv", + heat_demand_urban="resources/heat_demand_urban_elec{year}_s{simpl}_{clusters}.nc", + heat_demand_rural="resources/heat_demand_rural_elec{year}_s{simpl}_{clusters}.nc", + heat_demand_total="resources/heat_demand_total_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_total="resources/temp_soil_total_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_rural="resources/temp_soil_rural_elec{year}_s{simpl}_{clusters}.nc", + temp_soil_urban="resources/temp_soil_urban_elec{year}_s{simpl}_{clusters}.nc", + temp_air_total="resources/temp_air_total_elec{year}_s{simpl}_{clusters}.nc", + temp_air_rural="resources/temp_air_rural_elec{year}_s{simpl}_{clusters}.nc", + temp_air_urban="resources/temp_air_urban_elec{year}_s{simpl}_{clusters}.nc", + cop_soil_total="resources/cop_soil_total_elec{year}_s{simpl}_{clusters}.nc", + cop_soil_rural="resources/cop_soil_rural_elec{year}_s{simpl}_{clusters}.nc", + cop_soil_urban="resources/cop_soil_urban_elec{year}_s{simpl}_{clusters}.nc", + cop_air_total="resources/cop_air_total_elec{year}_s{simpl}_{clusters}.nc", + cop_air_rural="resources/cop_air_rural_elec{year}_s{simpl}_{clusters}.nc", + cop_air_urban="resources/cop_air_urban_elec{year}_s{simpl}_{clusters}.nc", + solar_thermal_total="resources/solar_thermal_total_elec{year}_s{simpl}_{clusters}.nc", + solar_thermal_urban="resources/solar_thermal_urban_elec{year}_s{simpl}_{clusters}.nc", + solar_thermal_rural="resources/solar_thermal_rural_elec{year}_s{simpl}_{clusters}.nc" + output: config['results_dir'] + config['run'] + '/prenetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc' threads: 1 resources: mem_mb=2000 - benchmark: config['results_dir'] + config['run'] + "/benchmarks/prepare_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}" + benchmark: config['results_dir'] + config['run'] + "/benchmarks/prepare_network/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}" script: "scripts/prepare_sector_network.py" rule plot_network: input: - network=config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" + network=config['results_dir'] + config['run'] + "/postnetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" output: - map=config['results_dir'] + config['run'] + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{co2_budget_name}_{planning_horizons}.pdf", - today=config['results_dir'] + config['run'] + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}-today.pdf" + map=config['results_dir'] + config['run'] + "/maps/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{co2_budget_name}_{planning_horizons}.pdf", + today=config['results_dir'] + config['run'] + "/maps/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}-today.pdf" threads: 2 resources: mem_mb=10000 script: "scripts/plot_network.py" @@ -351,10 +351,10 @@ rule copy_config: rule make_summary: input: - networks=expand(config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", + networks=expand(config['results_dir'] + config['run'] + "/postnetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", **config['scenario']), costs=config['costs_dir'] + "costs_{}.csv".format(config['scenario']['planning_horizons'][0]), - plots=expand(config['results_dir'] + config['run'] + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{co2_budget_name}_{planning_horizons}.pdf", + plots=expand(config['results_dir'] + config['run'] + "/maps/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{co2_budget_name}_{planning_horizons}.pdf", **config['scenario']) #heat_demand_name='data/heating/daily_heat_demand.h5' output: @@ -397,16 +397,16 @@ if config["foresight"] == "overnight": rule solve_network: input: - network=config['results_dir'] + config['run'] + "/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", + network=config['results_dir'] + config['run'] + "/prenetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", costs=config['costs_dir'] + "costs_{planning_horizons}.csv", config=config['summary_dir'] + '/' + config['run'] + '/configs/config.yaml' - output: config['results_dir'] + config['run'] + "/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" + output: config['results_dir'] + config['run'] + "/postnetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" shadow: "shallow" log: - solver=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_solver.log", - python=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python.log", - memory=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_memory.log" - benchmark: config['results_dir'] + config['run'] + "/benchmarks/solve_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}" + solver=config['results_dir'] + config['run'] + "/logs/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_solver.log", + python=config['results_dir'] + config['run'] + "/logs/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python.log", + memory=config['results_dir'] + config['run'] + "/logs/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_memory.log" + benchmark: config['results_dir'] + config['run'] + "/benchmarks/solve_network/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}" threads: 4 resources: mem_mb=config['solving']['mem'] # group: "solve" # with group, threads is ignored https://bitbucket.org/snakemake/snakemake/issues/971/group-job-description-does-not-contain @@ -417,15 +417,15 @@ if config["foresight"] == "myopic": rule add_existing_baseyear: input: - network=config['results_dir'] + config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', + network=config['results_dir'] + config['run'] + '/prenetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', powerplants=pypsaeur('resources/powerplants.csv'), - busmap_s=pypsaeur("resources/busmap_{network}_s{simpl}.csv"), - busmap=pypsaeur("resources/busmap_{network}_s{simpl}_{clusters}.csv"), - clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv", + busmap_s=pypsaeur("resources/busmap_elec{year}_s{simpl}.csv"), + busmap=pypsaeur("resources/busmap_elec{year}_s{simpl}_{clusters}.csv"), + clustered_pop_layout="resources/pop_layout_elec{year}_s{simpl}_{clusters}.csv", costs=config['costs_dir'] + "costs_{}.csv".format(config['scenario']['planning_horizons'][0]), - cop_soil_total="resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc", - cop_air_total="resources/cop_air_total_{network}_s{simpl}_{clusters}.nc" - output: config['results_dir'] + config['run'] + '/prenetworks-brownfield/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc' + cop_soil_total="resources/cop_soil_total_elec{year}_s{simpl}_{clusters}.nc", + cop_air_total="resources/cop_air_total_elec{year}_s{simpl}_{clusters}.nc" + output: config['results_dir'] + config['run'] + '/prenetworks-brownfield/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc' wildcard_constraints: planning_horizons=config['scenario']['planning_horizons'][0] #only applies to baseyear threads: 1 @@ -434,18 +434,18 @@ if config["foresight"] == "myopic": def process_input(wildcards): i = config["scenario"]["planning_horizons"].index(int(wildcards.planning_horizons)) - return config['results_dir'] + config['run'] + "/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_" + str(config["scenario"]["planning_horizons"][i-1]) + ".nc" + return config['results_dir'] + config['run'] + "/postnetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_" + str(config["scenario"]["planning_horizons"][i-1]) + ".nc" rule add_brownfield: input: - network=config['results_dir'] + config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', + network=config['results_dir'] + config['run'] + '/prenetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc', network_p=process_input, #solved network at previous time step costs=config['costs_dir'] + "costs_{planning_horizons}.csv", - cop_soil_total="resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc", - cop_air_total="resources/cop_air_total_{network}_s{simpl}_{clusters}.nc" + cop_soil_total="resources/cop_soil_total_elec{year}_s{simpl}_{clusters}.nc", + cop_air_total="resources/cop_air_total_elec{year}_s{simpl}_{clusters}.nc" - output: config['results_dir'] + config['run'] + "/prenetworks-brownfield/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" + output: config['results_dir'] + config['run'] + "/prenetworks-brownfield/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" threads: 4 resources: mem_mb=2000 script: "scripts/add_brownfield.py" @@ -454,16 +454,16 @@ if config["foresight"] == "myopic": rule solve_network_myopic: input: - network=config['results_dir'] + config['run'] + "/prenetworks-brownfield/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", + network=config['results_dir'] + config['run'] + "/prenetworks-brownfield/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc", costs=config['costs_dir'] + "costs_{planning_horizons}.csv", config=config['summary_dir'] + '/' + config['run'] + '/configs/config.yaml' - output: config['results_dir'] + config['run'] + "/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" + output: config['results_dir'] + config['run'] + "/postnetworks/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc" shadow: "shallow" log: - solver=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_solver.log", - python=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python.log", - memory=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_memory.log" - benchmark: config['results_dir'] + config['run'] + "/benchmarks/solve_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}" + solver=config['results_dir'] + config['run'] + "/logs/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_solver.log", + python=config['results_dir'] + config['run'] + "/logs/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python.log", + memory=config['results_dir'] + config['run'] + "/logs/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_memory.log" + benchmark: config['results_dir'] + config['run'] + "/benchmarks/solve_network/elec{year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}" threads: 4 resources: mem_mb=config['solving']['mem'] script: "scripts/solve_network.py" diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index 94cbd133..85543afc 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -5,8 +5,11 @@ import pandas as pd import atlite import helper +year = snakemake.wildcards.year +cutout_name = snakemake.config['atlite']['cutout_name'] +if year: cutout_name = cutout_name.format(year=year) -cutout = atlite.Cutout(snakemake.config['atlite']['cutout_name'], +cutout = atlite.Cutout(cutout_name, cutout_dir=snakemake.config['atlite']['cutout_dir']) diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 865f12bd..0e6cdbd7 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -15,10 +15,16 @@ if 'snakemake' not in globals(): snakemake.input = Dict() snakemake.output = Dict() -time = pd.date_range(freq='m', **snakemake.config['snapshots']) +year = snakemake.wildcards.year + +snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] +time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) -cutout = atlite.Cutout(snakemake.config['atlite']['cutout_name'], +cutout_name = snakemake.config['atlite']['cutout_name'] +if year: cutout_name = cutout_name.format(year=year) + +cutout = atlite.Cutout(cutout_name, cutout_dir=snakemake.config['atlite']['cutout_dir'], **params) diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index 5093e1b4..15804576 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -21,7 +21,11 @@ if 'snakemake' not in globals(): snakemake.input["urban_percent"] = "data/urban_percent.csv" -cutout = atlite.Cutout(snakemake.config['atlite']['cutout_name'], +year = snakemake.wildcards.year +cutout_name = snakemake.config['atlite']['cutout_name'] +if year: cutout_name = cutout_name.format(year=year) + +cutout = atlite.Cutout(cutout_name, cutout_dir=snakemake.config['atlite']['cutout_dir']) grid_cells = cutout.grid_cells() diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index c26266aa..7d4199d5 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -15,12 +15,17 @@ if 'snakemake' not in globals(): snakemake.input = Dict() snakemake.output = Dict() -time = pd.date_range(freq='m', **snakemake.config['snapshots']) +year = snakemake.wildcards.year + +snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] +time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) +cutout_name = snakemake.config['atlite']['cutout_name'] +if year: cutout_name = cutout_name.format(year=year) -cutout = atlite.Cutout(snakemake.config['atlite']['cutout_name'], +cutout = atlite.Cutout(cutout_name, cutout_dir=snakemake.config['atlite']['cutout_dir'], **params) diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index a55bd606..2cad501c 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -15,11 +15,16 @@ if 'snakemake' not in globals(): snakemake.input = Dict() snakemake.output = Dict() -time = pd.date_range(freq='m', **snakemake.config['snapshots']) +year = snakemake.wildcards.year + +snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] +time = pd.date_range(freq='m', **snapshots) params = dict(years=slice(*time.year[[0, -1]]), months=slice(*time.month[[0, -1]])) +cutout_name = snakemake.config['atlite']['cutout_name'] +if year: cutout_name = cutout_name.format(year=year) -cutout = atlite.Cutout(snakemake.config['atlite']['cutout_name'], +cutout = atlite.Cutout(cutout_name, cutout_dir=snakemake.config['atlite']['cutout_dir'], **params) From 8d1047380a406d0724e964cc8cdec7f0030e89e6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 18 Feb 2022 11:27:07 +0100 Subject: [PATCH 05/31] remove myopic config file --- config.myopic.yaml | 346 --------------------------------------------- 1 file changed, 346 deletions(-) delete mode 100644 config.myopic.yaml diff --git a/config.myopic.yaml b/config.myopic.yaml deleted file mode 100644 index 0dc4cdf3..00000000 --- a/config.myopic.yaml +++ /dev/null @@ -1,346 +0,0 @@ -version: 0.3.0 - -logging_level: INFO - -results_dir: 'results/' -summary_dir: results -costs_dir: '../technology-data/outputs/' -run: 'your-run-name' # use this to keep track of runs with different settings -foresight: 'myopic' #options are overnight, myopic, perfect (perfect is not yet implemented) - - -scenario: - sectors: [E] # ignore this legacy setting - year: [''] # weather year - simpl: [''] # only relevant for PyPSA-Eur - lv: [1.0,1.5] # allowed transmission line volume expansion, can be any float >= 1.0 (today) or "opt" - clusters: [45,50] # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred - opts: [''] # only relevant for PyPSA-Eur - sector_opts: [Co2L0-3H-H-B-solar3-dist1] # this is where the main scenario settings are - # to really understand the options here, look in scripts/prepare_sector_network.py - # Co2Lx specifies the CO2 target in x% of the 1990 values; default will give default (5%); - # Co2L0p25 will give 25% CO2 emissions; Co2Lm0p05 will give 5% negative emissions - # xH is the temporal resolution; 3H is 3-hourly, i.e. one snapshot every 3 hours - # single letters are sectors: T for land transport, H for building heating, - # B for biomass supply, I for industry, shipping and aviation - # solarx or onwindx changes the available installable potential by factor x - # dist{n} includes distribution grids with investment cost of n times cost in data/costs.csv - planning_horizons : [2020, 2030, 2040, 2050] #investment years for myopic and perfect; or costs year for overnight - co2_budget_name: ['go'] #gives shape of CO2 budgets over planning horizon - -# snapshots are originally set in PyPSA-Eur/config.yaml but used again by PyPSA-Eur-Sec -snapshots: - # arguments to pd.date_range - start: "2013-01-01" - end: "2014-01-01" - closed: 'left' # end is not inclusive - -countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] - -atlite: - cutout_dir: '../pypsa-eur/cutouts' - cutout_name: "europe-2013-era5" - -# this information is NOT used but needed as an argument for -# pypsa-eur/scripts/add_electricity.py/load_costs in make_summary.py -electricity: - max_hours: - battery: 6 - H2: 168 - -biomass: - year: 2030 - scenario: "Med" - classes: - solid biomass: ['Primary agricultural residues', 'Forestry energy residue', 'Secondary forestry residues', 'Secondary Forestry residues – sawdust', 'Forestry residues from landscape care biomass', 'Municipal waste'] - not included: ['Bioethanol sugar beet biomass', 'Rapeseeds for biodiesel', 'sunflower and soya for Biodiesel', 'Starchy crops biomass', 'Grassy crops biomass', 'Willow biomass', 'Poplar biomass potential', 'Roundwood fuelwood', 'Roundwood Chips & Pellets'] - biogas: ['Manure biomass potential', 'Sludge biomass'] - -# only relevant for foresight = myopic or perfect -existing_capacities: - grouping_years: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] - threshold_capacity: 10 - conventional_carriers: ['lignite', 'coal', 'oil', 'uranium'] - -sector: - 'central' : True - 'central_fraction' : 0.6 - 'dsm_restriction_value' : 0.75 #Set to 0 for no restriction on BEV DSM - 'dsm_restriction_time' : 7 #Time at which SOC of BEV has to be dsm_restriction_value - 'transport_heating_deadband_upper' : 20. - 'transport_heating_deadband_lower' : 15. - 'ICE_lower_degree_factor' : 0.375 #in per cent increase in fuel consumption per degree above deadband - 'ICE_upper_degree_factor' : 1.6 - 'EV_lower_degree_factor' : 0.98 - 'EV_upper_degree_factor' : 0.63 - 'district_heating_loss' : 0.15 - 'bev' : True #turns on EV battery - 'bev_availability' : 0.5 #How many cars do smart charging - 'v2g' : True #allows feed-in to grid from EV battery - 'transport_fuel_cell_share' : 0. #0 means all EVs, 1 means all FCs - 'shipping_average_efficiency' : 0.4 #For conversion of fuel oil to propulsion in 2011 - 'time_dep_hp_cop' : True - 'space_heating_fraction' : 1.0 #fraction of space heating active - 'retrofitting' : False - 'retroI-fraction' : 0.25 - 'retroII-fraction' : 0.55 - 'retrofitting-cost_factor' : 1.0 - 'tes' : True - 'tes_tau' : 3. - 'boilers' : True - 'oil_boilers': False - 'chp' : True - 'micro_chp' : False - 'solar_thermal' : True - 'solar_cf_correction': 0.788457 # = >>> 1/1.2683 - 'marginal_cost_storage' : 0. #1e-4 - 'methanation' : True - 'helmeth' : True - 'dac' : True - 'co2_vent' : True - 'SMR' : True - 'ccs_fraction' : 0.9 - 'hydrogen_underground_storage' : True - 'use_fischer_tropsch_waste_heat' : True - 'use_fuel_cell_waste_heat' : True - 'electricity_distribution_grid' : False - 'electricity_distribution_grid_cost_factor' : 1.0 #multiplies cost in data/costs.csv - 'electricity_grid_connection' : True # only applies to onshore wind and utility PV - 'gas_distribution_grid' : True - 'gas_distribution_grid_cost_factor' : 1.0 #multiplies cost in data/costs.csv - -costs: - year: 2030 - lifetime: 25 #default lifetime - # From a Lion Hirth paper, also reflects average of Noothout et al 2016 - discountrate: 0.07 - # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html # noqa: E501 - USD2013_to_EUR2013: 0.7532 - - # Marginal and capital costs can be overwritten - # capital_cost: - # Wind: Bla - marginal_cost: # - solar: 0.01 - onwind: 0.015 - offwind: 0.015 - hydro: 0. - H2: 0. - battery: 0. - - emission_prices: # only used with the option Ep (emission prices) - co2: 0. - - lines: - length_factor: 1.25 #to estimate offwind connection costs - - -solving: - #tmpdir: "path/to/tmp" - options: - formulation: kirchhoff - clip_p_max_pu: 1.e-2 - load_shedding: false - noisy_costs: true - - min_iterations: 1 - max_iterations: 1 - # nhours: 1 - - solver: - name: gurobi - threads: 4 - method: 2 # barrier - crossover: 0 - BarConvTol: 1.e-5 - Seed: 123 - AggFill: 0 - PreDual: 0 - GURO_PAR_BARDENSETHRESH: 200 - #FeasibilityTol: 1.e-6 - - #name: cplex - #threads: 4 - #lpmethod: 4 # barrier - #solutiontype: 2 # non basic solution, ie no crossover - #barrier_convergetol: 1.e-5 - #feasopt_tolerance: 1.e-6 - mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 - -industry: - 'St_primary_fraction' : 0.3 # fraction of steel produced via primary route (DRI + EAF) versus secondary route (EAF); today fraction is 0.6 - 'H2_DRI' : 1.7 #H2 consumption in Direct Reduced Iron (DRI), MWh_H2,LHV/ton_Steel from Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279 - 'elec_DRI' : 0.322 #electricity consumption in Direct Reduced Iron (DRI) shaft, MWh/tSt HYBRIT brochure https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf - 'Al_primary_fraction' : 0.2 # fraction of aluminium produced via the primary route versus scrap; today fraction is 0.4 - 'MWh_CH4_per_tNH3_SMR' : 10.8 # 2012's demand from https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf - 'MWh_elec_per_tNH3_SMR' : 0.7 # same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3 - 'MWh_H2_per_tNH3_electrolysis' : 6.5 # from https://doi.org/10.1016/j.joule.2018.04.017, around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy) - 'MWh_elec_per_tNH3_electrolysis' : 1.17 # from https://doi.org/10.1016/j.joule.2018.04.017 Table 13 (air separation and HB) - 'NH3_process_emissions' : 24.5 # in MtCO2/a from SMR for H2 production for NH3 from UNFCCC for 2015 for EU28 - 'petrochemical_process_emissions' : 25.5 # in MtCO2/a for petrochemical and other from UNFCCC for 2015 for EU28 - 'HVC_primary_fraction' : 1.0 #fraction of current non-ammonia basic chemicals produced via primary route - -plotting: - map: - figsize: [7, 7] - boundaries: [-10.2, 29, 35, 72] - p_nom: - bus_size_factor: 5.e+4 - linewidth_factor: 3.e+3 # 1.e+3 #3.e+3 - - costs_max: 1200 - costs_threshold: 1 - - - energy_max: 20000. - energy_min: -15000. - energy_threshold: 50. - - - vre_techs: ["onwind", "offwind-ac", "offwind-dc", "solar", "ror"] - renewable_storage_techs: ["PHS","hydro"] - conv_techs: ["OCGT", "CCGT", "Nuclear", "Coal"] - storage_techs: ["hydro+PHS", "battery", "H2"] - # store_techs: ["Li ion", "water tanks"] - load_carriers: ["AC load"] #, "heat load", "Li ion load"] - AC_carriers: ["AC line", "AC transformer"] - link_carriers: ["DC line", "Converter AC-DC"] - heat_links: ["heat pump", "resistive heater", "CHP heat", "CHP electric", - "gas boiler", "central heat pump", "central resistive heater", "central CHP heat", - "central CHP electric", "central gas boiler"] - heat_generators: ["gas boiler", "central gas boiler", "solar thermal collector", "central solar thermal collector"] - tech_colors: - "onwind" : "b" - "onshore wind" : "b" - 'offwind' : "c" - 'offshore wind' : "c" - 'offwind-ac' : "c" - 'offshore wind (AC)' : "c" - 'offwind-dc' : "#009999" - 'offshore wind (DC)' : "#009999" - 'wave' : "#004444" - "hydro" : "#3B5323" - "hydro reservoir" : "#3B5323" - "ror" : "#78AB46" - "run of river" : "#78AB46" - 'hydroelectricity' : '#006400' - 'solar' : "y" - 'solar PV' : "y" - 'solar thermal' : 'coral' - 'solar rooftop' : '#e6b800' - "OCGT" : "wheat" - "OCGT marginal" : "sandybrown" - "OCGT-heat" : "orange" - "gas boiler" : "orange" - "gas boilers" : "orange" - "gas boiler marginal" : "orange" - "gas-to-power/heat" : "orange" - "gas" : "brown" - "natural gas" : "brown" - "SMR" : "#4F4F2F" - "oil" : "#B5A642" - "oil boiler" : "#B5A677" - "lines" : "k" - "transmission lines" : "k" - "H2" : "m" - "hydrogen storage" : "m" - "battery" : "slategray" - "battery storage" : "slategray" - "home battery" : "#614700" - "home battery storage" : "#614700" - "Nuclear" : "r" - "Nuclear marginal" : "r" - "nuclear" : "r" - "uranium" : "r" - "Coal" : "k" - "coal" : "k" - "Coal marginal" : "k" - "Lignite" : "grey" - "lignite" : "grey" - "Lignite marginal" : "grey" - "CCGT" : "orange" - "CCGT marginal" : "orange" - "heat pumps" : "#76EE00" - "heat pump" : "#76EE00" - "air heat pump" : "#76EE00" - "ground heat pump" : "#40AA00" - "power-to-heat" : "#40AA00" - "resistive heater" : "pink" - "Sabatier" : "#FF1493" - "methanation" : "#FF1493" - "power-to-gas" : "#FF1493" - "power-to-liquid" : "#FFAAE9" - "helmeth" : "#7D0552" - "helmeth" : "#7D0552" - "DAC" : "#E74C3C" - "co2 stored" : "#123456" - "CO2 sequestration" : "#123456" - "CCS" : "k" - "co2" : "#123456" - "co2 vent" : "#654321" - "solid biomass for industry co2 from atmosphere" : "#654321" - "solid biomass for industry co2 to stored": "#654321" - "gas for industry co2 to atmosphere": "#654321" - "gas for industry co2 to stored": "#654321" - "Fischer-Tropsch" : "#44DD33" - "kerosene for aviation": "#44BB11" - "naphtha for industry" : "#44FF55" - "water tanks" : "#BBBBBB" - "hot water storage" : "#BBBBBB" - "hot water charging" : "#BBBBBB" - "hot water discharging" : "#999999" - "CHP" : "r" - "CHP heat" : "r" - "CHP electric" : "r" - "PHS" : "g" - "Ambient" : "k" - "Electric load" : "b" - "Heat load" : "r" - "Transport load" : "grey" - "heat" : "darkred" - "rural heat" : "#880000" - "central heat" : "#b22222" - "decentral heat" : "#800000" - "low-temperature heat for industry" : "#991111" - "process heat" : "#FF3333" - "heat demand" : "darkred" - "electric demand" : "k" - "Li ion" : "grey" - "district heating" : "#CC4E5C" - "retrofitting" : "purple" - "building retrofitting" : "purple" - "BEV charger" : "grey" - "V2G" : "grey" - "transport" : "grey" - "electricity" : "k" - "gas for industry" : "#333333" - "solid biomass for industry" : "#555555" - "industry electricity" : "#222222" - "industry new electricity" : "#222222" - "process emissions to stored" : "#444444" - "process emissions to atmosphere" : "#888888" - "process emissions" : "#222222" - "transport fuel cell" : "#AAAAAA" - "biogas" : "#800000" - "solid biomass" : "#DAA520" - "today" : "#D2691E" - "shipping" : "#6495ED" - "electricity distribution grid" : "#333333" - nice_names: - # OCGT: "Gas" - # OCGT marginal: "Gas (marginal)" - offwind: "offshore wind" - onwind: "onshore wind" - battery: "Battery storage" - lines: "Transmission lines" - AC line: "AC lines" - AC-AC: "DC lines" - ror: "Run of river" - nice_names_n: - offwind: "offshore\nwind" - onwind: "onshore\nwind" - # OCGT: "Gas" - H2: "Hydrogen\nstorage" - # OCGT marginal: "Gas (marginal)" - lines: "transmission\nlines" - ror: "run of river" From 4397f1fd0c52cc4454b231720d92cfdfeb29c1f3 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 18 Feb 2022 11:32:06 +0100 Subject: [PATCH 06/31] minor fixes --- Snakefile | 2 +- config.default.yaml | 2 -- scripts/build_population_layouts.py | 5 ++++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Snakefile b/Snakefile index c0c1c7fa..4efb15f1 100644 --- a/Snakefile +++ b/Snakefile @@ -12,7 +12,7 @@ configfile: "config.yaml" wildcard_constraints: - year="[0-9]*", + weather_year="[0-9]*", lv="[a-z0-9\.]+", simpl="[a-zA-Z0-9]*", clusters="[0-9]+m?", diff --git a/config.default.yaml b/config.default.yaml index 010c31d3..63dcab51 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -65,8 +65,6 @@ snapshots: end: "2014-01-01" closed: left # end is not inclusive -countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] - atlite: cutout: ../pypsa-eur/cutouts/europe-2013-era5.nc diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index e83dd5f3..a5ae2ebf 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -12,7 +12,10 @@ from vresutils import shapes as vshapes if __name__ == '__main__': if 'snakemake' not in globals(): from helper import mock_snakemake - snakemake = mock_snakemake('build_population_layouts', year='') + snakemake = mock_snakemake( + 'build_population_layouts', + weather_year='', + ) year = snakemake.wildcards.weather_year cutout_config = snakemake.config['atlite']['cutout'] From cd2078849a45aca70eb43257df57e1fbda99709d Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 18 Feb 2022 11:33:30 +0100 Subject: [PATCH 07/31] remove misplaced code snippet --- scripts/build_clustered_population_layouts.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index e9ef7398..7a8de88b 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -5,10 +5,6 @@ import xarray as xr import pandas as pd import atlite -year = snakemake.wildcards.weather_year -cutout_name = snakemake.config['atlite']['cutout_name'] -if year: cutout_name = cutout_name.format(year=year) - if __name__ == '__main__': if 'snakemake' not in globals(): from helper import mock_snakemake From 8055e45b6807160f957b41cb212619e672e378b6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 14:56:46 +0200 Subject: [PATCH 08/31] combine time selection with weather year --- Snakefile | 7 ++++- config.default.yaml | 3 ++- scripts/build_clustered_population_layouts.py | 6 ++--- scripts/build_heat_demand.py | 27 +++++++++---------- scripts/build_population_layouts.py | 6 ++--- scripts/build_solar_thermal_profiles.py | 27 +++++++++---------- scripts/build_temperature_profiles.py | 18 +++++++++---- 7 files changed, 53 insertions(+), 41 deletions(-) diff --git a/Snakefile b/Snakefile index e2c1a881..0ca4e7c8 100644 --- a/Snakefile +++ b/Snakefile @@ -12,7 +12,7 @@ configfile: "config.yaml" wildcard_constraints: - weather_year="[0-9]*", + weather_year="[0-9]{4}|", lv="[a-z0-9\.]+", simpl="[a-zA-Z0-9]*", clusters="[0-9]+m?", @@ -68,6 +68,7 @@ if config.get('retrieve_sector_databundle', True): rule build_population_layouts: input: + cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), nuts3_shapes=pypsaeur('resources/nuts3_shapes.geojson'), urban_percent="data/urban_percent.csv" output: @@ -82,6 +83,7 @@ rule build_population_layouts: rule build_clustered_population_layouts: input: + cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -163,6 +165,7 @@ else: rule build_heat_demands: input: + cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -178,6 +181,7 @@ rule build_heat_demands: rule build_temperature_profiles: input: + cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -216,6 +220,7 @@ rule build_cop_profiles: rule build_solar_thermal_profiles: input: + cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", diff --git a/config.default.yaml b/config.default.yaml index 3ed14371..7f137515 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -66,7 +66,8 @@ snapshots: closed: left # end is not inclusive atlite: - cutout: ../pypsa-eur/cutouts/europe-2013-era5.nc + cutout: europe-era5-{weather_year} + drop_leap_day: false # this information is NOT used but needed as an argument for # pypsa-eur/scripts/add_electricity.py/load_costs in make_summary.py diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index 7a8de88b..ca50db51 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -15,10 +15,10 @@ if __name__ == '__main__': clusters=48, ) + cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - cutout_config = snakemake.config['atlite']['cutout'] - if year: cutout_name = cutout_config.format(weather_year=year) - cutout = atlite.Cutout(cutout_config) + if year: cutout_name = cutout_name.format(weather_year=year) + cutout = atlite.Cutout(cutout_name) clustered_regions = gpd.read_file( snakemake.input.regions_onshore).set_index('name').buffer(0).squeeze() diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 1640379c..23eeae27 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -16,22 +16,21 @@ if __name__ == '__main__': clusters=48, ) - if 'snakemake' not in globals(): - from vresutils import Dict - import yaml - snakemake = Dict() - with open('config.yaml') as f: - snakemake.config = yaml.safe_load(f) - snakemake.input = Dict() - snakemake.output = Dict() - + cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] - time = pd.date_range(freq='m', **snapshots) + drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) - cutout_config = snakemake.config['atlite']['cutout'] - if year: cutout_name = cutout_config.format(weather_year=year) - cutout = atlite.Cutout(cutout_config).sel(time=time) + if year: + snapshots = dict(start=year, end=str(int(year)+1), closed="left") + cutout_name = cutout_name.format(weather_year=year) + else: + snapshots = snakemake.config['snapshots'] + + time = pd.date_range(freq='m', **snapshots) + if drop_leap_day: + time = time[~((time.month == 2) & (time.day == 29))] + + cutout = atlite.Cutout(cutout_name).sel(time=time) clustered_regions = gpd.read_file( snakemake.input.regions_onshore).set_index('name').buffer(0).squeeze() diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index e6dc767d..01727777 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -17,10 +17,10 @@ if __name__ == '__main__': weather_year='', ) + cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - cutout_config = snakemake.config['atlite']['cutout'] - if year: cutout_name = cutout_config.format(weather_year=year) - cutout = atlite.Cutout(cutout_config) + if year: cutout_name = cutout_name.format(weather_year=year) + cutout = atlite.Cutout(cutout_name) grid_cells = cutout.grid_cells() diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index 6ae522eb..e87c9a1e 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -16,24 +16,23 @@ if __name__ == '__main__': clusters=48, ) - if 'snakemake' not in globals(): - from vresutils import Dict - import yaml - snakemake = Dict() - with open('config.yaml') as f: - snakemake.config = yaml.safe_load(f) - snakemake.input = Dict() - snakemake.output = Dict() - config = snakemake.config['solar_thermal'] + cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] - time = pd.date_range(freq='m', **snapshots) + drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) - cutout_config = snakemake.config['atlite']['cutout'] - if year: cutout_name = cutout_config.format(weather_year=year) - cutout = atlite.Cutout(cutout_config).sel(time=time) + if year: + snapshots = dict(start=year, end=str(int(year)+1), closed="left") + cutout_name = cutout_name.format(weather_year=year) + else: + snapshots = snakemake.config['snapshots'] + + time = pd.date_range(freq='m', **snapshots) + if drop_leap_day: + time = time[~((time.month == 2) & (time.day == 29))] + + cutout = atlite.Cutout(cutout_name).sel(time=time) clustered_regions = gpd.read_file( snakemake.input.regions_onshore).set_index('name').buffer(0).squeeze() diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index 75fc9ca1..02cae24c 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -16,13 +16,21 @@ if __name__ == '__main__': clusters=48, ) + cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] - time = pd.date_range(freq='m', **snapshots) + drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) - cutout_config = snakemake.config['atlite']['cutout'] - if year: cutout_name = cutout_config.format(weather_year=year) - cutout = atlite.Cutout(cutout_config).sel(time=time) + if year: + snapshots = dict(start=year, end=str(int(year)+1), closed="left") + cutout_name = cutout_name.format(weather_year=year) + else: + snapshots = snakemake.config['snapshots'] + + time = pd.date_range(freq='m', **snapshots) + if drop_leap_day: + time = time[~((time.month == 2) & (time.day == 29))] + + cutout = atlite.Cutout(cutout_name).sel(time=time) clustered_regions = gpd.read_file( snakemake.input.regions_onshore).set_index('name').buffer(0).squeeze() From 9f9e65afb1433d35809bb9e431ef33ab10dbe5e3 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 15:18:07 +0200 Subject: [PATCH 09/31] add missing handling of weather_year wildcard --- Snakefile | 28 ++++++++++++------------- scripts/build_heat_demand.py | 3 +-- scripts/build_solar_thermal_profiles.py | 3 +-- scripts/build_temperature_profiles.py | 3 +-- scripts/build_transport_demand.py | 8 ++++++- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Snakefile b/Snakefile index 0ca4e7c8..fa78641b 100644 --- a/Snakefile +++ b/Snakefile @@ -440,8 +440,8 @@ else: rule build_population_weighted_energy_totals: input: energy_totals='resources/energy_totals.csv', - clustered_pop_layout="resources/pop_layout_elec_s{simpl}_{clusters}.csv" - output: "resources/pop_weighted_energy_totals_s{simpl}_{clusters}.csv" + clustered_pop_layout="resources/pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" + output: "resources/pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=2000 script: "scripts/build_population_weighted_energy_totals.py" @@ -449,17 +449,17 @@ rule build_population_weighted_energy_totals: rule build_transport_demand: input: - clustered_pop_layout="resources/pop_layout_elec_s{simpl}_{clusters}.csv", - pop_weighted_energy_totals="resources/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", + clustered_pop_layout="resources/pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv", + pop_weighted_energy_totals="resources/pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv", transport_data='resources/transport_data.csv', traffic_data_KFZ="data/emobility/KFZ__count", traffic_data_Pkw="data/emobility/Pkw__count", - temp_air_total="resources/temp_air_total_elec_s{simpl}_{clusters}.nc", + temp_air_total="resources/temp_air_total_elec{weather_year}_s{simpl}_{clusters}.nc", output: - transport_demand="resources/transport_demand_s{simpl}_{clusters}.csv", - transport_data="resources/transport_data_s{simpl}_{clusters}.csv", - avail_profile="resources/avail_profile_s{simpl}_{clusters}.csv", - dsm_profile="resources/dsm_profile_s{simpl}_{clusters}.csv" + transport_demand="resources/transport_demand{weather_year}_s{simpl}_{clusters}.csv", + transport_data="resources/transport_data{weather_year}_s{simpl}_{clusters}.csv", + avail_profile="resources/avail_profile{weather_year}_s{simpl}_{clusters}.csv", + dsm_profile="resources/dsm_profile{weather_year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=2000 script: "scripts/build_transport_demand.py" @@ -470,11 +470,11 @@ rule prepare_sector_network: overrides="data/override_component_attrs", network=pypsaeur('networks/elec{weather_year}_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc'), energy_totals_name='resources/energy_totals.csv', - pop_weighted_energy_totals="resources/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", - transport_demand="resources/transport_demand_s{simpl}_{clusters}.csv", - transport_data="resources/transport_data_s{simpl}_{clusters}.csv", - avail_profile="resources/avail_profile_s{simpl}_{clusters}.csv", - dsm_profile="resources/dsm_profile_s{simpl}_{clusters}.csv", + pop_weighted_energy_totals="resources/pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv", + transport_demand="resources/transport_demand{weather_year}_s{simpl}_{clusters}.csv", + transport_data="resources/transport_data{weather_year}_s{simpl}_{clusters}.csv", + avail_profile="resources/avail_profile{weather_year}_s{simpl}_{clusters}.csv", + dsm_profile="resources/dsm_profile{weather_year}_s{simpl}_{clusters}.csv", co2_totals_name='resources/co2_totals.csv', biomass_potentials='resources/biomass_potentials_s{simpl}_{clusters}.csv', heat_profile="data/heat_load_profile_BDEW.csv", diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 23eeae27..fe44de16 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -18,7 +18,6 @@ if __name__ == '__main__': cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) if year: snapshots = dict(start=year, end=str(int(year)+1), closed="left") @@ -27,7 +26,7 @@ if __name__ == '__main__': snapshots = snakemake.config['snapshots'] time = pd.date_range(freq='m', **snapshots) - if drop_leap_day: + if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] cutout = atlite.Cutout(cutout_name).sel(time=time) diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index e87c9a1e..8bdc07c5 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -20,7 +20,6 @@ if __name__ == '__main__': cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) if year: snapshots = dict(start=year, end=str(int(year)+1), closed="left") @@ -29,7 +28,7 @@ if __name__ == '__main__': snapshots = snakemake.config['snapshots'] time = pd.date_range(freq='m', **snapshots) - if drop_leap_day: + if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] cutout = atlite.Cutout(cutout_name).sel(time=time) diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index 02cae24c..ab006409 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -18,7 +18,6 @@ if __name__ == '__main__': cutout_name = snakemake.input.cutout year = snakemake.wildcards.weather_year - drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) if year: snapshots = dict(start=year, end=str(int(year)+1), closed="left") @@ -27,7 +26,7 @@ if __name__ == '__main__': snapshots = snakemake.config['snapshots'] time = pd.date_range(freq='m', **snapshots) - if drop_leap_day: + if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] cutout = atlite.Cutout(cutout_name).sel(time=time) diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index a5aabc31..a9d6e440 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -159,6 +159,7 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_transport_demand", + weather_year='', simpl="", clusters=48, ) @@ -173,7 +174,12 @@ if __name__ == "__main__": options = snakemake.config["sector"] - snapshots = pd.date_range(freq='h', **snakemake.config["snapshots"], tz="UTC") + year = snakemake.wildcards.weather_year + snapshots = dict(start=year, end=str(int(year)+1), closed="left") if year else snakemake.config['snapshots'] + snapshots = pd.date_range(freq='h', **snapshots, tz="UTC") + if snakemake.config["atlite"].get("drop_leap_day", False): + leap_day = (snapshots.month == 2) & (snapshots.day == 29) + snapshots = snapshots[~leap_day] Nyears = 1 From a2e26004d7b2026a61cc17683323b2673ad40649 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 15 Jun 2022 15:36:37 +0200 Subject: [PATCH 10/31] add further missing handling of weather_year wildcard --- Snakefile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Snakefile b/Snakefile index fa78641b..852e924f 100644 --- a/Snakefile +++ b/Snakefile @@ -68,7 +68,7 @@ if config.get('retrieve_sector_databundle', True): rule build_population_layouts: input: - cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), + cutout=pypsaeur("cutouts/" + config['atlite']['cutout'] + ".nc"), nuts3_shapes=pypsaeur('resources/nuts3_shapes.geojson'), urban_percent="data/urban_percent.csv" output: @@ -83,7 +83,7 @@ rule build_population_layouts: rule build_clustered_population_layouts: input: - cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), + cutout=pypsaeur("cutouts/" + config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -138,8 +138,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson", production="data/gas_network/scigrid-gas/data/IGGIELGN_Productions.geojson", planned_lng="data/gas_network/planned_LNGs.csv", - regions_onshore=pypsaeur("resources/regions_onshore_elec_s{simpl}_{clusters}.geojson"), - regions_offshore=pypsaeur('resources/regions_offshore_elec_s{simpl}_{clusters}.geojson') + regions_onshore=pypsaeur("resources/regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_offshore=pypsaeur('resources/regions_offshore{weather_year}_elec_s{simpl}_{clusters}.geojson') output: gas_input_nodes="resources/gas_input_locations_s{simpl}_{clusters}.geojson", gas_input_nodes_simplified="resources/gas_input_locations_s{simpl}_{clusters}_simplified.csv" @@ -150,8 +150,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule cluster_gas_network: input: cleaned_gas_network="resources/gas_network.csv", - regions_onshore=pypsaeur("resources/regions_onshore_elec_s{simpl}_{clusters}.geojson"), - regions_offshore=pypsaeur("resources/regions_offshore_elec_s{simpl}_{clusters}.geojson") + regions_onshore=pypsaeur("resources/regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"), + regions_offshore=pypsaeur("resources/regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson") output: clustered_gas_network="resources/gas_network_elec_s{simpl}_{clusters}.csv" resources: mem_mb=4000 @@ -165,7 +165,7 @@ else: rule build_heat_demands: input: - cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), + cutout=pypsaeur("cutouts/" + config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -181,7 +181,7 @@ rule build_heat_demands: rule build_temperature_profiles: input: - cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), + cutout=pypsaeur("cutouts/" + config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -220,7 +220,7 @@ rule build_cop_profiles: rule build_solar_thermal_profiles: input: - cutout=pypsaeur("cutouts/" + snakemake.config['atlite']['cutout'] + ".nc"), + cutout=pypsaeur("cutouts/" + config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", @@ -476,7 +476,7 @@ rule prepare_sector_network: avail_profile="resources/avail_profile{weather_year}_s{simpl}_{clusters}.csv", dsm_profile="resources/dsm_profile{weather_year}_s{simpl}_{clusters}.csv", co2_totals_name='resources/co2_totals.csv', - biomass_potentials='resources/biomass_potentials_s{simpl}_{clusters}.csv', + biomass_potentials='resources/biomass_potentials{weather_year}_s{simpl}_{clusters}.csv', heat_profile="data/heat_load_profile_BDEW.csv", costs=CDIR + "costs_{planning_horizons}.csv", profile_offwind_ac=pypsaeur("resources/profile{weather_year}_offwind-ac.nc"), From 16bd4b6c2cbe4f2f3b315b19ed27b4f8c804ca27 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 22 Jul 2022 19:14:00 +0200 Subject: [PATCH 11/31] fix snapshot creation based on weather year --- scripts/build_heat_demand.py | 6 +++--- scripts/build_solar_thermal_profiles.py | 2 +- scripts/build_temperature_profiles.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index fe44de16..1e165c69 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -11,9 +11,9 @@ if __name__ == '__main__': from helper import mock_snakemake snakemake = mock_snakemake( 'build_heat_demands', - weather_year='', + weather_year='1969', simpl='', - clusters=48, + clusters=37, ) cutout_name = snakemake.input.cutout @@ -25,7 +25,7 @@ if __name__ == '__main__': else: snapshots = snakemake.config['snapshots'] - time = pd.date_range(freq='m', **snapshots) + time = pd.date_range(freq='h', **snapshots) if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index 8bdc07c5..ec9e59c4 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -27,7 +27,7 @@ if __name__ == '__main__': else: snapshots = snakemake.config['snapshots'] - time = pd.date_range(freq='m', **snapshots) + time = pd.date_range(freq='h', **snapshots) if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index ab006409..3e690cca 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -25,7 +25,7 @@ if __name__ == '__main__': else: snapshots = snakemake.config['snapshots'] - time = pd.date_range(freq='m', **snapshots) + time = pd.date_range(freq='h', **snapshots) if snakemake.config["atlite"].get("drop_leap_day", False): time = time[~((time.month == 2) & (time.day == 29))] From 6e47c711952ab94ac098f62d922420e27a27c3e0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 22 Jul 2022 20:48:24 +0200 Subject: [PATCH 12/31] clustered_population: add missing cutout input --- Snakefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Snakefile b/Snakefile index 68d32443..54829b6a 100644 --- a/Snakefile +++ b/Snakefile @@ -97,6 +97,7 @@ rule build_clustered_population_layouts: rule build_simplified_population_layouts: input: + cutout=pypsaeur("cutouts/" + config['atlite']['cutout'] + ".nc"), pop_layout_total="resources/pop_layout_total{weather_year}.nc", pop_layout_urban="resources/pop_layout_urban{weather_year}.nc", pop_layout_rural="resources/pop_layout_rural{weather_year}.nc", From d47a9aac72309420b8e941132ccf1c30169343c5 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 23 Jul 2022 10:37:32 +0200 Subject: [PATCH 13/31] energy_totals: update eurostat to 2021 --- Snakefile | 10 +-- scripts/build_energy_totals.py | 153 ++++++++++++++------------------- 2 files changed, 68 insertions(+), 95 deletions(-) diff --git a/Snakefile b/Snakefile index 54829b6a..fa2323f8 100644 --- a/Snakefile +++ b/Snakefile @@ -54,8 +54,7 @@ datafiles = [ "data/emobility/KFZ__count", "data/emobility/Pkw__count", "data/h2_salt_caverns_GWh_per_sqkm.geojson", - directory("data/eurostat-energy_balances-june_2016_edition"), - directory("data/eurostat-energy_balances-may_2018_edition"), + directory("data/eurostat-energy_balances-april_2022_edition"), directory("data/jrc-idees-2015"), ] @@ -235,11 +234,6 @@ rule build_solar_thermal_profiles: script: "scripts/build_solar_thermal_profiles.py" -def input_eurostat(w): - # 2016 includes BA, 2017 does not - report_year = config["energy"]["eurostat_report_year"] - return f"data/eurostat-energy_balances-june_{report_year}_edition" - rule build_energy_totals: input: nuts3_shapes=pypsaeur('resources/nuts3_shapes.geojson'), @@ -247,7 +241,7 @@ rule build_energy_totals: swiss="data/switzerland-sfoe/switzerland-new_format.csv", idees="data/jrc-idees-2015", district_heat_share='data/district_heat_share.csv', - eurostat=input_eurostat + eurostat=directory("data/eurostat-energy_balances-june_2021_edition"), output: energy_name='resources/energy_totals.csv', co2_name='resources/co2_totals.csv', diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 3f376b0c..f2e565e1 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -17,55 +17,6 @@ def reverse(dictionary): """reverses a keys and values of a dictionary""" return {v: k for k, v in dictionary.items()} - -# translations for Eurostat -eurostat_country_to_alpha2 = { - "EU28": "EU", - "EA19": "EA", - "Belgium": "BE", - "Bulgaria": "BG", - "Czech Republic": "CZ", - "Denmark": "DK", - "Germany": "DE", - "Estonia": "EE", - "Ireland": "IE", - "Greece": "GR", - "Spain": "ES", - "France": "FR", - "Croatia": "HR", - "Italy": "IT", - "Cyprus": "CY", - "Latvia": "LV", - "Lithuania": "LT", - "Luxembourg": "LU", - "Hungary": "HU", - "Malta": "MA", - "Netherlands": "NL", - "Austria": "AT", - "Poland": "PL", - "Portugal": "PT", - "Romania": "RO", - "Slovenia": "SI", - "Slovakia": "SK", - "Finland": "FI", - "Sweden": "SE", - "United Kingdom": "GB", - "Iceland": "IS", - "Norway": "NO", - "Montenegro": "ME", - "FYR of Macedonia": "MK", - "Albania": "AL", - "Serbia": "RS", - "Turkey": "TU", - "Bosnia and Herzegovina": "BA", - "Kosovo\n(UNSCR 1244/99)": "KO", # 2017 version - # 2016 version - "Kosovo\n(under United Nations Security Council Resolution 1244/99)": "KO", - "Moldova": "MO", - "Ukraine": "UK", - "Switzerland": "CH", -} - non_EU = ["NO", "CH", "ME", "MK", "RS", "BA", "AL"] idees_rename = {"GR": "EL", "GB": "UK"} @@ -127,47 +78,68 @@ to_ipcc = { } -def build_eurostat(countries, year): - """Return multi-index for all countries' energy data in TWh/a.""" +def eurostat_per_country(country): + + country_fn = idees_rename.get(country, country) + fn = snakemake.input.eurostat + f"/{country_fn}-Energy-balance-sheets-June-2021-edition.xlsb" - report_year = snakemake.config["energy"]["eurostat_report_year"] - filenames = { - 2016: f"/{year}-Energy-Balances-June2016edition.xlsx", - 2017: f"/{year}-ENERGY-BALANCES-June2017edition.xlsx" - } - - dfs = pd.read_excel( - snakemake.input.eurostat + filenames[report_year], + df = pd.read_excel( + fn, sheet_name=None, - skiprows=1, - index_col=list(range(4)), + skiprows=4, + index_col=list(range(3)), + na_values=["+", "-", "=", "Z", ":"], ) - # sorted_index necessary for slicing - lookup = eurostat_country_to_alpha2 - labelled_dfs = {lookup[df.columns[0]]: df - for df in dfs.values() - if lookup[df.columns[0]] in countries} - df = pd.concat(labelled_dfs, sort=True).sort_index() + df.pop("Cover") - # drop non-numeric and country columns - non_numeric_cols = df.columns[df.dtypes != float] - country_cols = df.columns.intersection(lookup.keys()) - to_drop = non_numeric_cols.union(country_cols) - df.drop(to_drop, axis=1, inplace=True) + return pd.concat(df) + + +def build_eurostat(countries, year=None): + """Return multi-index for all countries' energy data in TWh/a.""" + + nprocesses = snakemake.threads + tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries), + desc='Build from eurostat database') + with mp.Pool(processes=nprocesses) as pool: + dfs = list(tqdm(pool.imap(eurostat_per_country, countries), **tqdm_kwargs)) + + index_names = ['country', 'year', 'lvl1', 'lvl2', 'lvl3'] + df = pd.concat(dfs, keys=countries, names=index_names) + + df.dropna(how='all', axis=0, inplace=True) + df.dropna(how='all', axis=1, inplace=True) + df = df[df.index.get_level_values('lvl1') != 'ktoe'] + + i = df.index.to_frame(index=False) + i.loc[i.lvl2 == 'Primary production', ['lvl1', 'lvl3']] = 'Main' + i.loc[i.lvl2 == 'Gross electricity production', 'lvl1'] = "Gross production" + i.ffill(inplace=True) + df.index = pd.MultiIndex.from_frame(i) + + df.drop(list(range(1990, 2020)), axis=1, inplace=True) + df.drop("Unnamed: 7", axis=1, inplace=True) + df.fillna(0., inplace=True) # convert ktoe/a to TWh/a df *= 11.63 / 1e3 + if year: + df = df.xs(str(year), level='year') + return df -def build_swiss(year): +def build_swiss(year=None): """Return a pd.Series of Swiss energy data in TWh/a""" fn = snakemake.input.swiss - df = pd.read_csv(fn, index_col=[0,1]).loc["CH", str(year)] + df = pd.read_csv(fn, index_col=[0,1]).loc["CH"] + + if year: + df = df[str(year)] # convert PJ/a to TWh/a df /= 3.6 @@ -406,8 +378,17 @@ def build_idees(countries, year): def build_energy_totals(countries, eurostat, swiss, idees): - eurostat_fuels = {"electricity": "Electricity", - "total": "Total all products"} + eurostat_fuels = dict( + electricity="Electricity", + total="Total" + ) + + eurostat_sectors = dict( + residential="Households", + services="Commercial & public services", + road="Road", + rail="Rail" + ) to_drop = ["passenger cars", "passenger car efficiency"] df = idees.reindex(countries).drop(to_drop, axis=1) @@ -417,8 +398,8 @@ def build_energy_totals(countries, eurostat, swiss, idees): # add international navigation - slicer = idx[in_eurostat, :, "Bunkers", :] - fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() + slicer = idx[in_eurostat, :, "International maritime bunkers", :] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() df.loc[in_eurostat, "total international navigation"] = fill_values # add swiss energy data @@ -434,12 +415,10 @@ def build_energy_totals(countries, eurostat, swiss, idees): for sector in ["residential", "services", "road", "rail"]: - eurostat_sector = sector.capitalize() - # fuel use for fuel in ["electricity", "total"]: - slicer = idx[to_fill, :, :, eurostat_sector] + slicer = idx[to_fill, :, :, eurostat_sectors[sector]] fill_values = eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=0).sum() df.loc[to_fill, f"{fuel} {sector}"] = fill_values @@ -489,17 +468,17 @@ def build_energy_totals(countries, eurostat, swiss, idees): # Missing aviation slicer = idx[to_fill, :, :, "Domestic aviation"] - fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() + fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() df.loc[to_fill, "total domestic aviation"] = fill_values - slicer = idx[to_fill, :, :, "International aviation"] - fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() + slicer = idx[to_fill, :, "International aviation", :] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() df.loc[to_fill, "total international aviation"] = fill_values # missing domestic navigation - slicer = idx[to_fill, :, :, "Domestic Navigation"] - fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=0).sum() + slicer = idx[to_fill, :, :, "Domestic navigation"] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() df.loc[to_fill, "total domestic navigation"] = fill_values # split road traffic for non-IDEES @@ -702,7 +681,7 @@ if __name__ == "__main__": idees_countries = countries.intersection(eu28) data_year = config["energy_totals_year"] - eurostat = build_eurostat(countries, data_year) + eurostat = build_eurostat(countries.difference(['CH']), data_year) swiss = build_swiss(data_year) idees = build_idees(idees_countries, data_year) From eee6651d12ff8ae87e4bf3a30119541c870decc1 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 23 Jul 2022 11:19:37 +0200 Subject: [PATCH 14/31] energy_totals: fetch all years 2000-2015 from IDEES --- scripts/build_energy_totals.py | 158 ++++++++++++++++----------------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index f2e565e1..1845c93c 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -147,90 +147,89 @@ def build_swiss(year=None): return df -def idees_per_country(ct, year): +def idees_per_country(country): base_dir = snakemake.input.idees ct_totals = {} - ct_idees = idees_rename.get(ct, ct) + ct_idees = idees_rename.get(country, country) fn_residential = f"{base_dir}/JRC-IDEES-2015_Residential_{ct_idees}.xlsx" fn_tertiary = f"{base_dir}/JRC-IDEES-2015_Tertiary_{ct_idees}.xlsx" fn_transport = f"{base_dir}/JRC-IDEES-2015_Transport_{ct_idees}.xlsx" # residential - df = pd.read_excel(fn_residential, "RES_hh_fec", index_col=0)[year] + df = pd.read_excel(fn_residential, "RES_hh_fec", index_col=0) - ct_totals["total residential space"] = df["Space heating"] + ct_totals["total residential space"] = df.loc["Space heating"] rows = ["Advanced electric heating", "Conventional electric heating"] - ct_totals["electricity residential space"] = df[rows].sum() + ct_totals["electricity residential space"] = df.loc[rows].sum() - ct_totals["total residential water"] = df.at["Water heating"] + ct_totals["total residential water"] = df.loc["Water heating"] assert df.index[23] == "Electricity" - ct_totals["electricity residential water"] = df[23] + ct_totals["electricity residential water"] = df.iloc[23] - ct_totals["total residential cooking"] = df["Cooking"] + ct_totals["total residential cooking"] = df.loc["Cooking"] assert df.index[30] == "Electricity" - ct_totals["electricity residential cooking"] = df[30] + ct_totals["electricity residential cooking"] = df.iloc[30] - df = pd.read_excel(fn_residential, "RES_summary", index_col=0)[year] + df = pd.read_excel(fn_residential, "RES_summary", index_col=0) row = "Energy consumption by fuel - Eurostat structure (ktoe)" - ct_totals["total residential"] = df[row] + ct_totals["total residential"] = df.loc[row] assert df.index[47] == "Electricity" - ct_totals["electricity residential"] = df[47] + ct_totals["electricity residential"] = df.iloc[47] assert df.index[46] == "Derived heat" - ct_totals["derived heat residential"] = df[46] + ct_totals["derived heat residential"] = df.iloc[46] assert df.index[50] == 'Thermal uses' - ct_totals["thermal uses residential"] = df[50] + ct_totals["thermal uses residential"] = df.iloc[50] # services - df = pd.read_excel(fn_tertiary, "SER_hh_fec", index_col=0)[year] + df = pd.read_excel(fn_tertiary, "SER_hh_fec", index_col=0) - ct_totals["total services space"] = df["Space heating"] + ct_totals["total services space"] = df.loc["Space heating"] rows = ["Advanced electric heating", "Conventional electric heating"] - ct_totals["electricity services space"] = df[rows].sum() + ct_totals["electricity services space"] = df.loc[rows].sum() - ct_totals["total services water"] = df["Hot water"] + ct_totals["total services water"] = df.loc["Hot water"] assert df.index[24] == "Electricity" - ct_totals["electricity services water"] = df[24] + ct_totals["electricity services water"] = df.iloc[24] - ct_totals["total services cooking"] = df["Catering"] + ct_totals["total services cooking"] = df.loc["Catering"] assert df.index[31] == "Electricity" - ct_totals["electricity services cooking"] = df[31] + ct_totals["electricity services cooking"] = df.iloc[31] - df = pd.read_excel(fn_tertiary, "SER_summary", index_col=0)[year] + df = pd.read_excel(fn_tertiary, "SER_summary", index_col=0) row = "Energy consumption by fuel - Eurostat structure (ktoe)" - ct_totals["total services"] = df[row] + ct_totals["total services"] = df.loc[row] assert df.index[50] == "Electricity" - ct_totals["electricity services"] = df[50] + ct_totals["electricity services"] = df.iloc[50] assert df.index[49] == "Derived heat" - ct_totals["derived heat services"] = df[49] + ct_totals["derived heat services"] = df.iloc[49] assert df.index[53] == 'Thermal uses' - ct_totals["thermal uses services"] = df[53] - + ct_totals["thermal uses services"] = df.iloc[53] # agriculture, forestry and fishing start = "Detailed split of energy consumption (ktoe)" end = "Market shares of energy uses (%)" - df = pd.read_excel(fn_tertiary, "AGR_fec", index_col=0).loc[start:end, year] + df = pd.read_excel(fn_tertiary, "AGR_fec", index_col=0).loc[start:end] rows = [ "Lighting", @@ -238,142 +237,143 @@ def idees_per_country(ct, year): "Specific electricity uses", "Pumping devices (electric)" ] - ct_totals["total agriculture electricity"] = df[rows].sum() + ct_totals["total agriculture electricity"] = df.loc[rows].sum() rows = ["Specific heat uses", "Low enthalpy heat"] - ct_totals["total agriculture heat"] = df[rows].sum() + ct_totals["total agriculture heat"] = df.loc[rows].sum() rows = [ "Motor drives", "Farming machine drives (diesel oil incl. biofuels)", "Pumping devices (diesel oil incl. biofuels)", ] - ct_totals["total agriculture machinery"] = df[rows].sum() + ct_totals["total agriculture machinery"] = df.loc[rows].sum() row = "Agriculture, forestry and fishing" - ct_totals["total agriculture"] = df[row] + ct_totals["total agriculture"] = df.loc[row] # transport - df = pd.read_excel(fn_transport, "TrRoad_ene", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrRoad_ene", index_col=0) - ct_totals["total road"] = df["by fuel (EUROSTAT DATA)"] + ct_totals["total road"] = df.loc["by fuel (EUROSTAT DATA)"] - ct_totals["electricity road"] = df["Electricity"] + ct_totals["electricity road"] = df.loc["Electricity"] - ct_totals["total two-wheel"] = df["Powered 2-wheelers (Gasoline)"] + ct_totals["total two-wheel"] = df.loc["Powered 2-wheelers (Gasoline)"] assert df.index[19] == "Passenger cars" - ct_totals["total passenger cars"] = df[19] + ct_totals["total passenger cars"] = df.iloc[19] assert df.index[30] == "Battery electric vehicles" - ct_totals["electricity passenger cars"] = df[30] + ct_totals["electricity passenger cars"] = df.iloc[30] assert df.index[31] == "Motor coaches, buses and trolley buses" - ct_totals["total other road passenger"] = df[31] + ct_totals["total other road passenger"] = df.iloc[31] assert df.index[39] == "Battery electric vehicles" - ct_totals["electricity other road passenger"] = df[39] + ct_totals["electricity other road passenger"] = df.iloc[39] assert df.index[41] == "Light duty vehicles" - ct_totals["total light duty road freight"] = df[41] + ct_totals["total light duty road freight"] = df.iloc[41] assert df.index[49] == "Battery electric vehicles" - ct_totals["electricity light duty road freight"] = df[49] + ct_totals["electricity light duty road freight"] = df.iloc[49] row = "Heavy duty vehicles (Diesel oil incl. biofuels)" - ct_totals["total heavy duty road freight"] = df[row] + ct_totals["total heavy duty road freight"] = df.loc[row] assert df.index[61] == "Passenger cars" - ct_totals["passenger car efficiency"] = df[61] + ct_totals["passenger car efficiency"] = df.iloc[61] - df = pd.read_excel(fn_transport, "TrRail_ene", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrRail_ene", index_col=0) - ct_totals["total rail"] = df["by fuel (EUROSTAT DATA)"] + ct_totals["total rail"] = df.loc["by fuel (EUROSTAT DATA)"] - ct_totals["electricity rail"] = df["Electricity"] + ct_totals["electricity rail"] = df.loc["Electricity"] assert df.index[15] == "Passenger transport" - ct_totals["total rail passenger"] = df[15] + ct_totals["total rail passenger"] = df.iloc[15] assert df.index[16] == "Metro and tram, urban light rail" assert df.index[19] == "Electric" assert df.index[20] == "High speed passenger trains" - ct_totals["electricity rail passenger"] = df[[16, 19, 20]].sum() + ct_totals["electricity rail passenger"] = df.iloc[[16, 19, 20]].sum() assert df.index[21] == "Freight transport" - ct_totals["total rail freight"] = df[21] + ct_totals["total rail freight"] = df.iloc[21] assert df.index[23] == "Electric" - ct_totals["electricity rail freight"] = df[23] + ct_totals["electricity rail freight"] = df.iloc[23] - df = pd.read_excel(fn_transport, "TrAvia_ene", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrAvia_ene", index_col=0) assert df.index[6] == "Passenger transport" - ct_totals["total aviation passenger"] = df[6] + ct_totals["total aviation passenger"] = df.iloc[6] assert df.index[10] == "Freight transport" - ct_totals["total aviation freight"] = df[10] + ct_totals["total aviation freight"] = df.iloc[10] assert df.index[7] == "Domestic" - ct_totals["total domestic aviation passenger"] = df[7] + ct_totals["total domestic aviation passenger"] = df.iloc[7] assert df.index[8] == "International - Intra-EU" assert df.index[9] == "International - Extra-EU" - ct_totals["total international aviation passenger"] = df[[8,9]].sum() + ct_totals["total international aviation passenger"] = df.iloc[[8,9]].sum() assert df.index[11] == "Domestic and International - Intra-EU" - ct_totals["total domestic aviation freight"] = df[11] + ct_totals["total domestic aviation freight"] = df.iloc[11] assert df.index[12] == "International - Extra-EU" - ct_totals["total international aviation freight"] = df[12] + ct_totals["total international aviation freight"] = df.iloc[12] ct_totals["total domestic aviation"] = ct_totals["total domestic aviation freight"] \ - + ct_totals["total domestic aviation passenger"] + + ct_totals["total domestic aviation passenger"] ct_totals["total international aviation"] = ct_totals["total international aviation freight"] \ - + ct_totals["total international aviation passenger"] + + ct_totals["total international aviation passenger"] - df = pd.read_excel(fn_transport, "TrNavi_ene", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrNavi_ene", index_col=0) # coastal and inland - ct_totals["total domestic navigation"] = df["by fuel (EUROSTAT DATA)"] + ct_totals["total domestic navigation"] = df.loc["by fuel (EUROSTAT DATA)"] - df = pd.read_excel(fn_transport, "TrRoad_act", index_col=0)[year] + df = pd.read_excel(fn_transport, "TrRoad_act", index_col=0) assert df.index[85] == "Passenger cars" - ct_totals["passenger cars"] = df[85] + ct_totals["passenger cars"] = df.iloc[85] - return pd.Series(ct_totals, name=ct) + return pd.DataFrame(ct_totals) -def build_idees(countries, year): +def build_idees(countries, year=None): nprocesses = snakemake.threads - func = partial(idees_per_country, year=year) tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries), desc='Build from IDEES database') with mp.Pool(processes=nprocesses) as pool: - totals_list = list(tqdm(pool.imap(func, countries), **tqdm_kwargs)) + dfs = list(tqdm(pool.imap(idees_per_country, countries), **tqdm_kwargs)) - - totals = pd.concat(totals_list, axis=1) + df = pd.concat(dfs, keys=countries, names=['country', 'year']) # convert ktoe to TWh - exclude = totals.index.str.fullmatch("passenger cars") - totals.loc[~exclude] *= 11.63 / 1e3 + exclude = df.columns.str.fullmatch("passenger cars") + df.loc[:,~exclude] *= 11.63 / 1e3 # convert TWh/100km to kWh/km - totals.loc["passenger car efficiency"] *= 10 + df["passenger car efficiency"] *= 10 # district heating share - district_heat = totals.loc[["derived heat residential", - "derived heat services"]].sum() - total_heat = totals.loc[["thermal uses residential", - "thermal uses services"]].sum() - totals.loc["district heat share"] = district_heat.div(total_heat) + subset = ["derived heat residential", "derived heat services"] + district_heat = df[subset].sum(axis=1) + subset = ["thermal uses residential", "thermal uses services"] + total_heat = df[subset].sum(axis=1) + df["district heat share"] = district_heat.div(total_heat) - return totals.T + if year: + df = df.xs(int(year), level='year') + + return df def build_energy_totals(countries, eurostat, swiss, idees): From ba8b3f9ce71ab40a610d6c03c39986ad47e877f0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 23 Jul 2022 17:05:07 +0200 Subject: [PATCH 15/31] energy_totals: combined energy totals for all years --- Snakefile | 1 + scripts/build_energy_totals.py | 87 +++++++++++++++++++++------------- 2 files changed, 54 insertions(+), 34 deletions(-) diff --git a/Snakefile b/Snakefile index fa2323f8..04356842 100644 --- a/Snakefile +++ b/Snakefile @@ -243,6 +243,7 @@ rule build_energy_totals: district_heat_share='data/district_heat_share.csv', eurostat=directory("data/eurostat-energy_balances-june_2021_edition"), output: + energy_name_full='resources/energy_totals_full.csv', energy_name='resources/energy_totals.csv', co2_name='resources/co2_totals.csv', transport_name='resources/transport_data.csv' diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 1845c93c..18f9ee6e 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -125,21 +125,25 @@ def build_eurostat(countries, year=None): # convert ktoe/a to TWh/a df *= 11.63 / 1e3 + df.index = df.index.set_levels(df.index.levels[1].astype(int), level=1) + if year: - df = df.xs(str(year), level='year') + df = df.xs(year, level='year') return df def build_swiss(year=None): - """Return a pd.Series of Swiss energy data in TWh/a""" + """Return a pd.DataFrame of Swiss energy data in TWh/a""" fn = snakemake.input.swiss - df = pd.read_csv(fn, index_col=[0,1]).loc["CH"] + df = pd.read_csv(fn, index_col=[0,1]).stack().unstack('item') + df.index.names = ["country", "year"] + df.index = df.index.set_levels(df.index.levels[1].astype(int), level=1) if year: - df = df[str(year)] + df = df.xs(year, level='year') # convert PJ/a to TWh/a df /= 3.6 @@ -373,6 +377,8 @@ def build_idees(countries, year=None): if year: df = df.xs(int(year), level='year') + df.columns.name = 'item' + return df @@ -391,15 +397,21 @@ def build_energy_totals(countries, eurostat, swiss, idees): ) to_drop = ["passenger cars", "passenger car efficiency"] - df = idees.reindex(countries).drop(to_drop, axis=1) + + new_index = pd.MultiIndex.from_product( + [countries, eurostat.index.levels[1]], + names=["country", "year"] + ) + + df = idees.reindex(new_index).drop(to_drop, axis=1) eurostat_countries = eurostat.index.levels[0] - in_eurostat = df.index.intersection(eurostat_countries) + in_eurostat = df.index.levels[0].intersection(eurostat_countries) # add international navigation - slicer = idx[in_eurostat, :, "International maritime bunkers", :] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() + slicer = idx[in_eurostat, :, :, "International maritime bunkers", :] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0,1]).sum() df.loc[in_eurostat, "total international navigation"] = fill_values # add swiss energy data @@ -409,17 +421,20 @@ def build_energy_totals(countries, eurostat, swiss, idees): # get values for missing countries based on Eurostat EnergyBalances # divide cooking/space/water according to averages in EU28 - missing = df.index[df["total residential"].isna()] - to_fill = missing.intersection(eurostat_countries) + to_fill = df.index[df["total residential"].isna() & df.index.get_level_values('country').isin(eurostat_countries)] uses = ["space", "cooking", "water"] + c = to_fill.get_level_values('country') + y = to_fill.get_level_values('year') + for sector in ["residential", "services", "road", "rail"]: # fuel use for fuel in ["electricity", "total"]: - slicer = idx[to_fill, :, :, eurostat_sectors[sector]] - fill_values = eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=0).sum() + + slicer = idx[c, y, :, :, eurostat_sectors[sector]] + fill_values = eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=[0,1]).sum() df.loc[to_fill, f"{fuel} {sector}"] = fill_values for sector in ["residential", "services"]: @@ -462,28 +477,28 @@ def build_energy_totals(countries, eurostat, swiss, idees): nonelectric_use = no_norway[f"total {sector} {use}"] - no_norway[f"electricity {sector} {use}"] nonelectric = no_norway[f"total {sector}"] - no_norway[f"electricity {sector}"] fraction = nonelectric_use.div(nonelectric).mean() - df.loc["NO", f"total {sector} {use}"] = total_heating * fraction - df.loc["NO", f"electricity {sector} {use}"] = total_heating * fraction * elec_fraction + df.loc["NO", f"total {sector} {use}"] = (total_heating * fraction).values + df.loc["NO", f"electricity {sector} {use}"] = (total_heating * fraction * elec_fraction).values # Missing aviation - slicer = idx[to_fill, :, :, "Domestic aviation"] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() + slicer = idx[c, y, :, :, "Domestic aviation"] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0,1]).sum() df.loc[to_fill, "total domestic aviation"] = fill_values - slicer = idx[to_fill, :, "International aviation", :] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() + slicer = idx[c, y, :, "International aviation", :] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0,1]).sum() df.loc[to_fill, "total international aviation"] = fill_values # missing domestic navigation - slicer = idx[to_fill, :, :, "Domestic navigation"] - fill_values = eurostat.loc[slicer, "Total"].groupby(level=0).sum() + slicer = idx[c, y, :, :, "Domestic navigation"] + fill_values = eurostat.loc[slicer, "Total"].groupby(level=[0,1]).sum() df.loc[to_fill, "total domestic navigation"] = fill_values # split road traffic for non-IDEES missing = df.index[df["total passenger cars"].isna()] - for fuel in ["total", "electricity"]: + for fuel in ["electricity", "total"]: selection = [ f"{fuel} passenger cars", f"{fuel} other road passenger", @@ -525,19 +540,20 @@ def build_energy_totals(countries, eurostat, swiss, idees): df.loc[missing, f"total aviation {purpose}"] = df.loc[missing, attrs].sum(axis=1) if "BA" in df.index: - # fill missing data for BA (services and road energy data) - # proportional to RS with ratio of total residential demand - missing = df.loc["BA"] == 0.0 - ratio = df.at["BA", "total residential"] / df.at["RS", "total residential"] - df.loc['BA', missing] = ratio * df.loc["RS", missing] + # fill missing data for BA proportional to RS + ratio = (df.loc["BA"].loc[2014:2020] / df.loc["RS"].loc[2014:2020]).mean() + df.loc["BA"] = (ratio * df.loc["RS"]).values # Missing district heating share dh_share = pd.read_csv(snakemake.input.district_heat_share, - index_col=0, usecols=[0, 1]) + index_col=0, usecols=[0, 1]) + + dh_share = pd.concat({y: dh_share for y in range(1990, 2021)}, names=["year", "country"]).swaplevel() + dh_share = dh_share.div(100).reindex(df.index) + # make conservative assumption and take minimum from both data sets - df["district heat share"] = (pd.concat([df["district heat share"], - dh_share.reindex(index=df.index)/100], - axis=1).min(axis=1)) + item = "district heat share" + df[item] = pd.concat([dh_share, df[item]], axis=1).min(axis=1) return df @@ -673,6 +689,7 @@ if __name__ == "__main__": snakemake = mock_snakemake('build_energy_totals') config = snakemake.config["energy"] + data_year = int(config["energy_totals_year"]) nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index") population = nuts3["pop"].groupby(nuts3.country).sum() @@ -680,12 +697,14 @@ if __name__ == "__main__": countries = population.index idees_countries = countries.intersection(eu28) - data_year = config["energy_totals_year"] - eurostat = build_eurostat(countries.difference(['CH']), data_year) - swiss = build_swiss(data_year) - idees = build_idees(idees_countries, data_year) + eurostat = build_eurostat(countries.difference(['CH'])) + swiss = build_swiss() + idees = build_idees(idees_countries) energy = build_energy_totals(countries, eurostat, swiss, idees) + energy.to_csv(snakemake.output.energy_name_full) + + energy = energy.xs(data_year, level='year') energy.to_csv(snakemake.output.energy_name) base_year_emissions = config["base_emissions_year"] From f70f006a47d65bcef63ef62b569dc7bd9a9032c0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 23 Jul 2022 17:05:56 +0200 Subject: [PATCH 16/31] energy_totals: co2 calculation with eurostat 2021 data --- scripts/build_energy_totals.py | 41 ++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 18f9ee6e..5482ec3b 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -606,23 +606,26 @@ def build_eea_co2(year=1990): return emissions / 1e3 -def build_eurostat_co2(countries, year=1990): +def build_eurostat_co2(countries, eurostat=None, year=1990): - eurostat = build_eurostat(countries, year) + if eurostat is None: + df = build_eurostat(countries, year) + else: + df = eurostat.xs(year, level='year') - specific_emissions = pd.Series(index=eurostat.columns, dtype=float) + specific_emissions = pd.Series(index=df.columns, dtype=float) # emissions in tCO2_equiv per MWh_th - specific_emissions["Solid fuels"] = 0.36 # Approximates coal - specific_emissions["Oil (total)"] = 0.285 # Average of distillate and residue - specific_emissions["Gas"] = 0.2 # For natural gas + specific_emissions["Solid fossil fuels"] = 0.36 # Approximates coal + specific_emissions["Oil and petroleum products"] = 0.285 # Average of distillate and residue + specific_emissions["Natural gas"] = 0.2 # For natural gas # oil values from https://www.eia.gov/tools/faqs/faq.cfm?id=74&t=11 # Distillate oil (No. 2) 0.276 # Residual oil (No. 6) 0.298 # https://www.eia.gov/electricity/annual/html/epa_a_03.html - return eurostat.multiply(specific_emissions).sum(axis=1) + return df.multiply(specific_emissions).sum(axis=1) def build_co2_totals(countries, eea_co2, eurostat_co2): @@ -632,19 +635,19 @@ def build_co2_totals(countries, eea_co2, eurostat_co2): for ct in countries.intersection(["BA", "RS", "AL", "ME", "MK"]): mappings = { - "electricity": (ct, "+", "Conventional Thermal Power Stations", "of which From Coal"), - "residential non-elec": (ct, "+", "+", "Residential"), - "services non-elec": (ct, "+", "+", "Services"), - "road non-elec": (ct, "+", "+", "Road"), - "rail non-elec": (ct, "+", "+", "Rail"), - "domestic navigation": (ct, "+", "+", "Domestic Navigation"), - "international navigation": (ct, "-", "Bunkers"), - "domestic aviation": (ct, "+", "+", "Domestic aviation"), - "international aviation": (ct, "+", "+", "International aviation"), + "electricity": (ct, "Transformation input", "Electricity & heat generation", "Main"), + "residential non-elec": (ct, "Final energy consumption", "Other sectors", "Households"), + "services non-elec": (ct, "Final energy consumption", "Other sectors", "Commercial & public services"), + "road non-elec": (ct, "Final energy consumption", "Transport sector", "Road"), + "rail non-elec": (ct, "Final energy consumption", "Transport sector", "Rail"), + "domestic navigation": (ct, "Final energy consumption", "Transport sector", "Domestic navigation"), + "international navigation": (ct, "Main", "International maritime bunkers"), + "domestic aviation": (ct, "Final energy consumption", "Transport sector", "Domestic aviation"), + "international aviation": (ct, "Main", "International aviation"), # does not include industrial process emissions or fuel processing/refining - "industrial non-elec": (ct, "+", "Industry"), + "industrial non-elec": (ct, "Final energy consumption", "Industry sector", "Non-energy use in industry sector"), # does not include non-energy emissions - "agriculture": (eurostat_co2.index.get_level_values(0) == ct) & eurostat_co2.index.isin(["Agriculture / Forestry", "Fishing"], level=3), + "agriculture": (eurostat_co2.index.get_level_values(0) == ct) & eurostat_co2.index.isin(["Agriculture & forestry", "Fishing"], level=3), } for i, mi in mappings.items(): @@ -709,7 +712,7 @@ if __name__ == "__main__": base_year_emissions = config["base_emissions_year"] eea_co2 = build_eea_co2(base_year_emissions) - eurostat_co2 = build_eurostat_co2(countries, base_year_emissions) + eurostat_co2 = build_eurostat_co2(countries, eurostat, base_year_emissions) co2 = build_co2_totals(countries, eea_co2, eurostat_co2) co2.to_csv(snakemake.output.co2_name) From 614909201ac3ae2a725d23e54c54e90f09345e77 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 23 Jul 2022 17:06:14 +0200 Subject: [PATCH 17/31] energy_totals: transport sector requires specific year --- scripts/build_energy_totals.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 5482ec3b..04147bb8 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -717,5 +717,6 @@ if __name__ == "__main__": co2 = build_co2_totals(countries, eea_co2, eurostat_co2) co2.to_csv(snakemake.output.co2_name) - transport = build_transport_data(countries, population, idees) + idees_transport = idees.xs(data_year, level='year') + transport = build_transport_data(countries, population, idees_transport) transport.to_csv(snakemake.output.transport_name) From 0bad2097b051c2bd34a00893ec30c04cd06b3372 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 25 Jul 2022 14:35:54 +0200 Subject: [PATCH 18/31] interpolate heat demands for years outside 2007-2015 --- Snakefile | 19 ++++-- data/era5-annual-HDD-per-country.csv | 34 ++++++++++ scripts/build_energy_totals.py | 3 - scripts/build_heat_totals.py | 63 +++++++++++++++++++ ...build_population_weighted_energy_totals.py | 21 ++++--- scripts/prepare_sector_network.py | 2 + 6 files changed, 128 insertions(+), 14 deletions(-) create mode 100644 data/era5-annual-HDD-per-country.csv create mode 100644 scripts/build_heat_totals.py diff --git a/Snakefile b/Snakefile index 04356842..617d30f2 100644 --- a/Snakefile +++ b/Snakefile @@ -243,7 +243,6 @@ rule build_energy_totals: district_heat_share='data/district_heat_share.csv', eurostat=directory("data/eurostat-energy_balances-june_2021_edition"), output: - energy_name_full='resources/energy_totals_full.csv', energy_name='resources/energy_totals.csv', co2_name='resources/co2_totals.csv', transport_name='resources/transport_data.csv' @@ -253,6 +252,18 @@ rule build_energy_totals: script: 'scripts/build_energy_totals.py' +rule build_heat_totals: + input: + hdd="data/era5-annual-HDD-per-country.csv", + energy_totals="resources/energy_totals.csv", + output: + heat_totals="resources/heat_totals.csv" + threads: 1 + resources: mem_mb=2000 + benchmark: "benchmarks/build_heat_totals" + script: "scripts/build_heat_totals.py" + + rule build_biomass_potentials: input: enspreso_biomass=HTTP.remote("https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", keep_local=True), @@ -435,9 +446,9 @@ else: rule build_population_weighted_energy_totals: input: - energy_totals='resources/energy_totals.csv', + totals='resources/{kind}_totals.csv', clustered_pop_layout="resources/pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" - output: "resources/pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv" + output: "resources/pop_weighted_{kind}_totals{weather_year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=2000 script: "scripts/build_population_weighted_energy_totals.py" @@ -465,8 +476,8 @@ rule prepare_sector_network: input: overrides="data/override_component_attrs", network=pypsaeur('networks/elec{weather_year}_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc'), - energy_totals_name='resources/energy_totals.csv', pop_weighted_energy_totals="resources/pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv", + pop_weighted_heat_totals="resources/pop_weighted_heat_totals{weather_year}_s{simpl}_{clusters}.csv", transport_demand="resources/transport_demand{weather_year}_s{simpl}_{clusters}.csv", transport_data="resources/transport_data{weather_year}_s{simpl}_{clusters}.csv", avail_profile="resources/avail_profile{weather_year}_s{simpl}_{clusters}.csv", diff --git a/data/era5-annual-HDD-per-country.csv b/data/era5-annual-HDD-per-country.csv new file mode 100644 index 00000000..472257c6 --- /dev/null +++ b/data/era5-annual-HDD-per-country.csv @@ -0,0 +1,34 @@ +name,1951,1952,1953,1954,1955,1956,1957,1958,1959,1960,1961,1962,1963,1964,1965,1966,1967,1968,1969,1970,1971,1972,1973,1974,1975,1976,1977,1978,1979,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021 +AL,56,58,65,67,55,70,58,59,61,54,54,62,62,62,68,59,60,57,57,58,61,57,64,60,61,61,54,63,57,65,63,58,61,57,57,58,61,62,56,53,67,62,62,53,61,61,60,60,56,58,57,53,61,58,65,64,55,55,54,53,58,58,50,46,54,50,54,47,49,50,54 +AT,374,416,379,422,419,449,387,393,378,389,367,437,425,406,424,376,385,400,409,413,394,407,408,379,384,400,380,414,405,428,397,384,377,411,420,406,409,378,369,372,415,373,383,334,380,416,371,373,368,343,371,337,370,370,384,362,333,340,351,388,337,353,358,302,324,336,347,315,320,322,358 +BE,95,105,94,102,109,113,91,97,88,89,84,111,114,100,100,91,94,101,102,101,96,102,101,90,97,97,92,103,108,105,99,94,96,99,112,107,108,85,84,83,101,92,94,82,89,110,90,89,84,79,89,78,89,90,87,86,76,87,87,107,75,90,99,69,81,88,82,81,80,71,89 +BG,283,294,334,367,295,363,306,283,320,268,277,303,324,313,323,259,297,292,320,286,300,306,323,303,296,314,287,310,293,326,301,312,295,306,326,313,333,320,276,272,332,303,318,266,307,335,325,303,273,275,284,272,322,277,298,296,259,266,266,273,313,295,258,250,261,265,281,261,232,250,277 +BA,144,164,167,183,158,194,157,157,152,143,141,173,180,176,176,155,162,157,165,165,163,154,170,156,153,168,145,170,153,176,163,154,157,159,167,160,166,158,146,139,174,152,162,143,157,171,156,158,147,132,145,130,156,146,164,148,134,132,139,147,144,147,133,113,137,134,144,128,125,127,141 +CH,214,225,210,226,221,240,211,215,203,214,200,234,231,214,229,213,217,219,225,227,217,221,224,213,216,220,208,219,220,229,216,208,208,227,224,217,217,200,202,200,217,204,208,183,206,217,195,205,203,192,200,189,199,204,209,193,187,194,193,213,179,196,204,178,181,189,191,174,184,176,197 +CZ,297,344,307,351,350,377,313,322,301,314,294,358,361,345,351,297,296,320,342,336,319,326,322,282,296,315,299,330,328,347,316,308,298,323,350,326,346,297,280,283,333,299,316,279,312,364,319,296,288,265,309,285,308,302,311,300,269,275,289,338,281,298,306,242,261,283,287,262,255,259,302 +DE,1254,1428,1237,1416,1484,1567,1282,1342,1229,1266,1183,1486,1528,1390,1399,1269,1211,1341,1447,1429,1300,1388,1356,1184,1258,1339,1221,1377,1433,1433,1348,1288,1273,1384,1492,1425,1491,1201,1156,1135,1367,1241,1312,1155,1295,1548,1288,1220,1172,1085,1250,1168,1269,1236,1239,1198,1072,1159,1202,1459,1112,1232,1305,992,1095,1188,1148,1092,1070,1019,1230 +DK,160,176,152,175,186,190,163,178,154,169,154,184,194,176,182,181,153,170,182,186,159,166,159,147,147,172,161,171,190,177,175,164,155,162,191,181,190,151,137,132,160,147,166,154,162,191,162,154,147,134,161,148,158,151,150,144,134,137,150,193,144,159,160,125,138,145,139,142,134,125,150 +ES,962,911,927,948,785,1092,934,848,821,890,742,952,981,967,949,878,890,839,933,899,971,961,943,931,928,928,797,906,887,911,793,787,807,920,860,891,781,783,684,767,911,852,889,724,627,759,605,735,796,750,757,677,746,813,856,714,776,785,736,856,673,810,830,649,684,722,688,768,708,672,728 +EE,297,310,283,297,321,336,278,310,281,305,253,298,323,292,306,324,283,310,333,317,285,285,289,256,245,327,302,324,299,313,289,278,260,271,328,294,332,281,232,240,257,257,285,288,265,306,281,278,262,235,273,271,278,270,270,260,250,230,268,315,253,286,259,254,225,261,255,254,235,209,270 +FI,3285,3442,3085,3179,3880,3819,3296,3626,3180,3386,3065,3513,3467,3329,3526,3873,3284,3693,3640,3494,3572,3178,3458,2963,3074,3691,3490,3743,3419,3607,3572,3331,3296,3192,3894,3516,3793,3402,2888,3075,3209,3163,3319,3341,3213,3343,3302,3468,3259,2915,3303,3307,3231,3159,2998,3101,3018,3009,3219,3604,2961,3351,2985,2983,2795,3030,3127,3061,3116,2726,3260 +FR,1430,1480,1422,1489,1396,1633,1338,1378,1240,1332,1196,1584,1630,1499,1471,1331,1408,1412,1473,1469,1475,1453,1545,1356,1462,1445,1312,1454,1502,1545,1396,1315,1424,1484,1599,1518,1520,1269,1237,1227,1501,1375,1389,1168,1280,1454,1222,1337,1263,1190,1299,1132,1294,1346,1376,1263,1199,1301,1277,1522,1060,1300,1408,1034,1143,1266,1236,1150,1165,1043,1279 +GB,906,934,809,901,925,935,813,890,787,849,825,1000,1046,912,970,917,839,874,894,869,805,880,862,842,838,851,863,873,972,880,896,823,824,838,923,944,903,801,764,746,859,824,857,793,794,886,738,752,745,771,816,726,751,743,748,734,706,790,777,930,707,832,841,683,759,769,720,764,750,725,764 +GR,166,162,210,211,160,215,184,169,196,164,172,182,184,192,199,162,185,176,171,164,184,185,195,186,188,190,160,186,172,191,184,192,190,179,174,180,194,193,176,165,206,197,195,161,179,191,192,180,167,177,174,164,191,172,184,189,161,158,155,143,188,179,148,136,165,150,164,139,146,148,156 +HR,116,140,140,155,137,166,131,132,122,119,114,149,158,152,144,124,133,129,143,138,137,131,144,124,124,139,121,140,130,149,133,132,130,133,148,141,144,128,118,114,143,123,138,115,126,145,132,129,124,106,120,109,135,127,140,123,109,110,116,130,123,123,118,92,114,116,119,111,102,105,119 +HU,236,291,282,321,289,333,272,273,246,248,235,288,321,309,298,244,272,268,293,283,276,264,283,244,255,281,267,287,271,311,272,282,265,274,313,294,305,274,243,238,299,262,288,240,271,310,291,272,267,232,269,243,300,267,289,265,229,230,243,275,267,259,250,198,235,251,255,234,211,228,259 +IE,216,215,191,208,213,211,190,204,184,205,195,234,250,208,232,210,216,213,222,214,190,226,204,213,198,211,210,205,239,212,209,204,201,211,229,241,217,200,192,192,209,209,213,201,195,219,180,186,191,204,206,189,192,193,187,188,173,202,204,241,192,205,210,186,205,199,184,200,191,194,189 +IT,652,683,660,700,615,778,640,653,616,631,587,715,726,674,710,665,655,656,682,672,678,639,695,659,655,670,604,681,675,729,684,645,657,695,688,671,676,630,616,610,712,626,650,555,623,639,578,618,619,563,585,546,629,612,675,597,548,568,590,637,562,602,587,481,541,528,570,529,540,520,577 +LT,344,371,342,368,368,405,318,353,329,346,303,355,386,362,371,354,322,365,401,372,330,345,339,312,289,388,348,370,370,382,342,326,301,326,393,352,397,335,275,272,313,311,336,329,325,377,335,325,306,274,323,313,328,324,325,316,299,277,316,366,305,332,314,298,273,308,296,302,267,252,327 +LU,8,9,8,9,10,10,8,9,8,8,7,10,10,9,9,8,8,9,9,9,9,9,9,8,9,9,8,9,10,10,9,9,9,9,10,10,10,8,8,8,9,8,9,8,8,10,8,8,8,7,8,7,8,8,8,8,7,8,8,9,7,8,9,6,7,8,8,7,7,7,8 +LV,383,402,372,394,407,439,351,391,361,384,326,386,420,385,402,401,360,401,434,407,361,372,371,337,316,424,385,411,392,411,372,357,330,354,425,379,428,362,300,301,334,337,366,366,348,402,365,358,334,302,353,346,357,352,354,337,324,300,346,402,330,365,342,329,297,337,330,328,297,272,351 +MK,64,66,74,79,66,79,67,66,71,63,64,72,73,70,77,66,69,66,67,67,70,67,75,69,70,69,62,72,66,73,71,69,70,67,69,67,70,71,65,63,76,68,69,59,69,70,71,68,64,65,65,62,69,64,70,68,60,59,61,59,67,66,56,53,61,58,62,54,55,57,63 +ME,40,43,45,48,41,50,42,42,42,39,39,44,45,45,48,43,43,42,42,43,43,41,44,43,42,45,39,45,42,46,45,41,43,42,42,41,44,44,42,39,47,43,43,38,43,44,42,43,39,39,40,37,42,41,45,43,38,38,38,38,39,40,36,32,37,36,39,34,35,35,39 +NL,113,126,110,121,130,137,108,118,107,107,104,132,141,121,120,114,105,119,124,122,111,120,116,104,111,116,107,120,133,122,118,112,110,118,134,127,130,99,99,94,119,106,113,102,109,138,111,102,97,93,106,98,109,103,100,100,87,102,103,131,93,108,117,82,95,104,96,98,93,85,105 +NO,3339,3541,3020,3307,3639,3731,3315,3547,3122,3353,3400,3807,3641,3496,3783,3951,3259,3533,3425,3431,3339,3128,3312,3028,3094,3405,3369,3436,3457,3380,3502,3181,3141,3117,3541,3359,3449,3200,2890,2848,3057,3032,3196,3207,3170,3301,3060,3203,3062,2878,3166,3028,2983,2943,2874,2828,2918,2954,3018,3442,2796,3157,2946,2754,2793,2859,2963,2930,2981,2711,3035 +PL,1249,1438,1294,1478,1446,1586,1279,1349,1269,1305,1215,1435,1527,1450,1467,1280,1193,1321,1504,1431,1293,1347,1314,1200,1184,1419,1270,1393,1431,1491,1336,1265,1178,1304,1494,1377,1507,1255,1077,1078,1312,1252,1310,1197,1302,1512,1326,1249,1171,1052,1268,1192,1296,1230,1262,1244,1117,1092,1220,1426,1177,1274,1251,1068,1063,1163,1158,1120,1019,1017,1252 +PT,107,103,99,109,80,131,106,93,93,97,78,106,104,115,103,98,103,97,112,110,115,116,106,109,109,109,99,139,107,102,89,90,95,104,99,108,85,89,75,87,104,96,104,85,66,88,62,82,96,90,90,78,87,97,103,90,91,91,84,96,80,101,100,81,79,87,79,96,81,72,78 +RO,736,811,883,948,822,968,810,789,826,731,744,826,890,867,873,706,791,776,858,792,787,798,838,782,752,855,772,836,780,880,801,811,763,807,917,815,882,827,710,700,846,801,848,701,806,869,842,805,740,710,760,721,844,754,799,780,684,695,695,757,793,781,709,662,681,721,720,690,624,648,740 +RS,222,252,273,300,249,313,248,249,254,231,226,270,283,271,274,233,253,243,267,254,251,246,272,242,243,265,233,265,241,273,251,255,244,253,275,256,268,254,231,220,275,243,263,223,250,270,259,250,237,217,234,217,262,238,265,247,217,214,222,229,251,244,214,192,222,223,232,211,197,212,234 +SK,178,210,195,217,208,228,192,193,185,185,178,208,213,211,219,179,187,192,203,200,192,188,195,178,178,197,188,204,193,215,191,190,180,193,212,198,207,187,172,173,202,186,191,171,186,204,193,184,177,162,184,174,190,184,191,182,163,161,170,188,175,179,174,142,161,170,176,157,152,158,180 +SI,61,72,67,73,69,78,65,66,62,61,59,75,75,73,73,62,65,67,71,69,67,68,71,62,61,69,62,71,68,74,67,65,65,69,73,71,71,63,59,58,70,61,63,55,61,71,63,62,62,54,59,55,65,63,67,61,54,57,57,65,58,59,59,46,55,57,58,53,52,53,61 +SE,3891,4219,3560,3919,4426,4488,3950,4223,3662,3988,3814,4451,4260,4021,4358,4613,3929,4280,4255,4254,4043,3806,3975,3634,3625,4238,4132,4314,4246,4287,4301,3913,3840,3819,4588,4139,4376,3931,3476,3446,3785,3695,3893,3991,3916,4073,3757,3950,3781,3446,3898,3778,3755,3769,3632,3561,3606,3590,3806,4397,3474,3935,3675,3452,3421,3635,3693,3705,3689,3247,3807 diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 04147bb8..d9763779 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -705,9 +705,6 @@ if __name__ == "__main__": idees = build_idees(idees_countries) energy = build_energy_totals(countries, eurostat, swiss, idees) - energy.to_csv(snakemake.output.energy_name_full) - - energy = energy.xs(data_year, level='year') energy.to_csv(snakemake.output.energy_name) base_year_emissions = config["base_emissions_year"] diff --git a/scripts/build_heat_totals.py b/scripts/build_heat_totals.py new file mode 100644 index 00000000..34d5dcdf --- /dev/null +++ b/scripts/build_heat_totals.py @@ -0,0 +1,63 @@ +"""Approximate heat demand for all weather years.""" + +import pandas as pd + +from itertools import product +from numpy.polynomial import Polynomial + +idx = pd.IndexSlice + + +def approximate_heat_demand(energy_totals, hdd): + + if isinstance(hdd, str): + hdd = pd.read_csv(hdd, index_col=0).T + hdd.index = hdd.index.astype(int) + + demands = {} + + for kind, sector in product(["total", "electricity"], ["services", "residential"]): + + row = idx[:, 2007:2015] + col = f"{kind} {sector} space" + demand = energy_totals.loc[row, col].unstack(0) + + demand_approx = {} + + for c in countries: + + Y = demand[c].dropna() + X = hdd.loc[Y.index, c] + + to_predict = hdd.index.difference(Y.index) + X_pred = hdd.loc[to_predict, c] + + p = Polynomial.fit(X, Y, 1) + Y_pred = p(X_pred) + + demand_approx[c] = pd.Series(Y_pred, index=to_predict) + + demand_approx = pd.DataFrame(demand_approx) + demand_approx = pd.concat([demand, demand_approx]).sort_index() + demands[f"{kind} {sector} space"] = demand_approx.groupby(demand_approx.index).sum() + + demands = pd.concat(demands).unstack().T.clip(lower=0) + demands.index.names = ["country", "year"] + + return demands + + +if __name__ == "__main__": + if 'snakemake' not in globals(): + from helper import mock_snakemake + snakemake = mock_snakemake('build_energy_totals') + + hdd = pd.read_csv(snakemake.input.hdd, index_col=0).T + + energy_totals = pd.read_csv(snakemake.input.energy_totals, index_col=[0,1]) + + countries = hdd.columns + + heat_demand = approximate_heat_demand(energy_totals, hdd) + + heat_demand.to_csv(snakemake.output.heat_totals) diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index 938983d5..933a0417 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -1,4 +1,4 @@ -"""Build population-weighted energy totals.""" +"""Build population-weighted energy and heat totals.""" import pandas as pd @@ -7,16 +7,23 @@ if __name__ == '__main__': from helper import mock_snakemake snakemake = mock_snakemake( 'build_population_weighted_energy_totals', + weather_year='', simpl='', - clusters=48, + clusters=37, ) + config = snakemake.config["energy"] + data_year = int(config["energy_totals_year"]) + if snakemake.wildcards.weather_year and snakemake.wildcards.kind == 'heat': + data_year = int(snakemake.wildcards.weather_year) + pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) - energy_totals = pd.read_csv(snakemake.input.energy_totals, index_col=0) + totals = pd.read_csv(snakemake.input.totals, index_col=[0,1]) + totals = totals.xs(data_year, level='year') - nodal_energy_totals = energy_totals.loc[pop_layout.ct].fillna(0.) - nodal_energy_totals.index = pop_layout.index - nodal_energy_totals = nodal_energy_totals.multiply(pop_layout.fraction, axis=0) + nodal_totals = totals.loc[pop_layout.ct].fillna(0.) + nodal_totals.index = pop_layout.index + nodal_totals = nodal_totals.multiply(pop_layout.fraction, axis=0) - nodal_energy_totals.to_csv(snakemake.output[0]) + nodal_totals.to_csv(snakemake.output[0]) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 14e824c0..3f25251f 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -2359,6 +2359,8 @@ if __name__ == "__main__": snakemake.config['costs']['lifetime']) pop_weighted_energy_totals = pd.read_csv(snakemake.input.pop_weighted_energy_totals, index_col=0) + pop_weighted_heat_totals = pd.read_csv(snakemake.input.pop_weighted_heat_totals, index_col=0) + pop_weighted_energy_totals.update(pop_weighted_heat_totals) patch_electricity_network(n) From 88ec493b3ab405a3ba307075366490e844ba6723 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 25 Jul 2022 14:40:13 +0200 Subject: [PATCH 19/31] Snakefile: correct updated eurostat dataset publication year! --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 617d30f2..1e53dc76 100644 --- a/Snakefile +++ b/Snakefile @@ -54,7 +54,7 @@ datafiles = [ "data/emobility/KFZ__count", "data/emobility/Pkw__count", "data/h2_salt_caverns_GWh_per_sqkm.geojson", - directory("data/eurostat-energy_balances-april_2022_edition"), + directory("data/eurostat-energy_balances-june_2021_edition"), directory("data/jrc-idees-2015"), ] From ee0c69727ffe365e6262e6683ec4f1e2346239a0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 29 Jul 2022 20:29:35 +0200 Subject: [PATCH 20/31] leap_days need to be filtered for daily data too in heat --- scripts/build_heat_demand.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 1e165c69..73e67633 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -25,9 +25,12 @@ if __name__ == '__main__': else: snapshots = snakemake.config['snapshots'] + drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) time = pd.date_range(freq='h', **snapshots) - if snakemake.config["atlite"].get("drop_leap_day", False): + daily = pd.date_range(freq='D', **snapshots) + if drop_leap_day: time = time[~((time.month == 2) & (time.day == 29))] + daily = daily[~((daily.month == 2) & (daily.day == 29))] cutout = atlite.Cutout(cutout_name).sel(time=time) @@ -44,6 +47,6 @@ if __name__ == '__main__': M = I.T.dot(np.diag(I.dot(stacked_pop))) heat_demand = cutout.heat_demand( - matrix=M.T, index=clustered_regions.index) + matrix=M.T, index=clustered_regions.index).sel(time=daily) heat_demand.to_netcdf(snakemake.output[f"heat_demand_{area}"]) From d4fa922a2a6af9abb8df01f03de58661de0fe3f7 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 29 Jul 2022 21:19:26 +0200 Subject: [PATCH 21/31] drop_leap_days when aggregating snapshots --- scripts/prepare_sector_network.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 3f25251f..45bd9abb 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -571,7 +571,7 @@ def add_co2limit(n, Nyears=1., limit=0.): ) # TODO PyPSA-Eur merge issue -def average_every_nhours(n, offset): +def average_every_nhours(n, offset, drop_leap_day=False): logger.info(f'Resampling the network to {offset}') m = n.copy(with_time=False) @@ -590,6 +590,10 @@ def average_every_nhours(n, offset): else: pnl[k] = df.resample(offset).mean() + if drop_leap_day: + sns = m.snapshots[~((m.snapshots.month == 2) & (m.snapshots.day == 29))] + m.set_snapshots(sns) + return m @@ -2428,7 +2432,8 @@ if __name__ == "__main__": for o in opts: m = re.match(r'^\d+h$', o, re.IGNORECASE) if m is not None: - n = average_every_nhours(n, m.group(0)) + drop_leap_day = snakemake.config["atlite"].get("drop_leap_day", False) + n = average_every_nhours(n, m.group(0), drop_leap_day) break limit_type = "config" From 64e1ef895d894262dc722004b8d40259a184adeb Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 1 Aug 2022 11:55:31 +0200 Subject: [PATCH 22/31] add rules to solve operations with perfect and myopic foresight --- Snakefile | 37 ++++++ scripts/solve_network.py | 13 +- scripts/solve_operations_network.py | 126 ++++++++++++++++++ scripts/solve_operations_network_myopic.py | 143 +++++++++++++++++++++ 4 files changed, 314 insertions(+), 5 deletions(-) create mode 100644 scripts/solve_operations_network.py create mode 100644 scripts/solve_operations_network_myopic.py diff --git a/Snakefile b/Snakefile index 1e53dc76..981c340d 100644 --- a/Snakefile +++ b/Snakefile @@ -13,6 +13,7 @@ configfile: "config.yaml" wildcard_constraints: weather_year="[0-9]{4}|", + capacity_year="[0-9]{4}|", lv="[a-z0-9\.]+", simpl="[a-zA-Z0-9]*", clusters="[0-9]+m?", @@ -680,3 +681,39 @@ if config["foresight"] == "myopic": resources: mem_mb=config['solving']['mem'] benchmark: RDIR + "/benchmarks/solve_network/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}" script: "scripts/solve_network.py" + + +rule solve_operations_network: + input: + pre=RDIR + "/prenetworks/elec{weather_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + post=RDIR + "/postnetworks/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + output: "elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc" + shadow: "shallow" + log: + solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", + python=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_python.log", + threads: 4 + resources: mem_mb=10000 + benchmark: RDIR + "/benchmarks/solve_operations_network/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}" + script: "scripts/solve_operations_network.py" + + +def solved_previous_year(wildcards): + previous_year = int(wildcards.weather_year) - 1 + return RDIR + "/postnetworks/elec" + previous_year + "_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc" + + +rule solve_operations_network_myopic: + input: + pre=RDIR + "/prenetworks/elec{weather_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + post=RDIR + "/postnetworks/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + previous=solved_previous_year + output: "elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc" + shadow: "shallow" + log: + solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", + python=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_python.log", + threads: 4 + resources: mem_mb=10000 + benchmark: RDIR + "/benchmarks/solve_operations_network_myopic/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}" + script: "scripts/solve_operations_network_myopic.py" diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 19ef5f52..4dd4d969 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -71,7 +71,7 @@ def prepare_network(n, solve_opts=None): df.where(df>solve_opts['clip_p_max_pu'], other=0., inplace=True) if solve_opts.get('load_shedding'): - n.add("Carrier", "Load") + n.add("Carrier", "load") n.madd("Generator", n.buses.index, " load", bus=n.buses.index, carrier='load', @@ -246,7 +246,7 @@ def extra_functionality(n, snapshots): add_co2_sequestration_limit(n, snapshots) -def solve_network(n, config, opts='', **kwargs): +def solve_network(n, config, opts='', snapshots=None, **kwargs): solver_options = config['solving']['solver'].copy() solver_name = solver_options.pop('name') cf_solving = config['solving']['options'] @@ -259,12 +259,15 @@ def solve_network(n, config, opts='', **kwargs): n.config = config n.opts = opts + if snapshots is None: + snapshots = n.snapshots + if cf_solving.get('skip_iterations', False): - network_lopf(n, solver_name=solver_name, solver_options=solver_options, + network_lopf(n, snapshots, solver_name=solver_name, solver_options=solver_options, extra_functionality=extra_functionality, keep_shadowprices=keep_shadowprices, **kwargs) else: - ilopf(n, solver_name=solver_name, solver_options=solver_options, + ilopf(n, snapshots, solver_name=solver_name, solver_options=solver_options, track_iterations=track_iterations, min_iterations=min_iterations, max_iterations=max_iterations, @@ -295,7 +298,7 @@ if __name__ == "__main__": if tmpdir is not None: from pathlib import Path Path(tmpdir).mkdir(parents=True, exist_ok=True) - opts = snakemake.wildcards.opts.split('-') + opts = snakemake.wildcards.sector_opts.split('-') solve_opts = snakemake.config['solving']['options'] fn = getattr(snakemake.log, 'memory', None) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py new file mode 100644 index 00000000..2021c7c9 --- /dev/null +++ b/scripts/solve_operations_network.py @@ -0,0 +1,126 @@ +"""Solve operations network.""" + + +import pypsa +import numpy as np + +from solve_network import solve_network, prepare_network +from helper import override_component_attrs + +import logging +logger = logging.getLogger(__name__) +pypsa.pf.logger.setLevel(logging.WARNING) + + +def set_parameters_from_optimized(n, n_optim): + lines_typed_i = n.lines.index[n.lines.type != ''] + n.lines.loc[lines_typed_i, 'num_parallel'] = \ + n_optim.lines['num_parallel'].reindex(lines_typed_i, fill_value=0.) + n.lines.loc[lines_typed_i, 's_nom'] = ( + np.sqrt(3) * n.lines['type'].map(n.line_types.i_nom) * + n.lines.bus0.map(n.buses.v_nom) * n.lines.num_parallel) + + lines_untyped_i = n.lines.index[n.lines.type == ''] + for attr in ('s_nom', 'r', 'x'): + n.lines.loc[lines_untyped_i, attr] = \ + n_optim.lines[attr].reindex(lines_untyped_i, fill_value=0.) + n.lines['s_nom_extendable'] = False + + links_dc_i = n.links.index[n.links.p_nom_extendable] + n.links.loc[links_dc_i, 'p_nom'] = \ + n_optim.links['p_nom_opt'].reindex(links_dc_i, fill_value=0.) + n.links.loc[links_dc_i, 'p_nom_extendable'] = False + + gen_extend_i = n.generators.index[n.generators.p_nom_extendable] + n.generators.loc[gen_extend_i, 'p_nom'] = \ + n_optim.generators['p_nom_opt'].reindex(gen_extend_i, fill_value=0.) + n.generators.loc[gen_extend_i, 'p_nom_extendable'] = False + + stor_units_extend_i = n.storage_units.index[n.storage_units.p_nom_extendable] + n.storage_units.loc[stor_units_extend_i, 'p_nom'] = \ + n_optim.storage_units['p_nom_opt'].reindex(stor_units_extend_i, fill_value=0.) + n.storage_units.loc[stor_units_extend_i, 'p_nom_extendable'] = False + + stor_extend_i = n.stores.index[n.stores.e_nom_extendable] + n.stores.loc[stor_extend_i, 'e_nom'] = \ + n_optim.stores['e_nom_opt'].reindex(stor_extend_i, fill_value=0.) + n.stores.loc[stor_extend_i, 'e_nom_extendable'] = False + + return n + + +def remove_unused_components(n, threshold=50): + logger.info("Remove assets that are barely used to speed things up.") + + for c in n.iterate_components({"Store", "Link", "Generator"}): + attr = "e_nom" if c.name == "Store" else "p_nom" + to_remove = c.df.loc[c.df[attr] < threshold].index + logger.info(f"Removing barely used {c.name}s:\n{to_remove}") + n.mremove(c.name, to_remove) + + return n + + +def add_load_shedding(n, voll=1e4): + logger.info("Add load shedding to all buses.") + + if "load" in n.generators.carrier.unique(): + to_remove = n.generators.query("carrier == 'load'").index + logger.info(f"Removing pre-existing load shedding:\n{to_remove}") + n.mremove("Generator", to_remove) + + n.madd("Generator", n.buses.index, + suffix=" load", + bus=n.buses.index, + carrier='load', + marginal_cost=voll, + p_nom=1e6 + ) + + return n + + +if __name__ == "__main__": + if 'snakemake' not in globals(): + from helper import mock_snakemake + snakemake = mock_snakemake( + 'solve_operations_network', + capacity_year=1952, + simpl='', + opts='', + clusters=37, + lv=2.0, + sector_opts='Co2L0-25H-T-H-B-I-A', + planning_horizons=2030, + weather_year=2013 + ) + + logging.basicConfig(filename=snakemake.log.python, + level=snakemake.config['logging_level']) + + tmpdir = snakemake.config['solving'].get('tmpdir') + if tmpdir is not None: + from pathlib import Path + Path(tmpdir).mkdir(parents=True, exist_ok=True) + + overrides = override_component_attrs(snakemake.input.overrides) + + n = pypsa.Network(snakemake.input.pre, override_component_attrs=overrides) + n_post = pypsa.Network(snakemake.input.post, override_component_attrs=overrides) + n = set_parameters_from_optimized(n, n_post) + del n_post + + n = remove_unused_components(n) + n = add_load_shedding(n) + + opts = snakemake.wildcards.sector_opts.split('-') + solve_opts = snakemake.config['solving']['options'] + solve_opts['skip_iterations'] = True + + n = prepare_network(n, solve_opts) + + n = solve_network(n, config=snakemake.config, opts=opts, + solver_dir=tmpdir, + solver_logfile=snakemake.log.solver) + + n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/solve_operations_network_myopic.py b/scripts/solve_operations_network_myopic.py new file mode 100644 index 00000000..a0e30b0a --- /dev/null +++ b/scripts/solve_operations_network_myopic.py @@ -0,0 +1,143 @@ +"""Solve myopic operations network.""" + + +import pypsa +import pandas as pd + +from solve_network import solve_network, prepare_network +from solve_operations_network import set_parameters_from_optimized, remove_unused_components, add_load_shedding +from helper import override_component_attrs + +import logging +logger = logging.getLogger(__name__) +pypsa.pf.logger.setLevel(logging.WARNING) + + +def prepare_myopic(n, config, store_soc, storage_unit_soc): + + n.stores.e_cyclic = False + n.storage_units.cyclic_state_of_charge = False + + biomass_stores = n.stores.carrier.str.isin(["solid biomass", "biogas"]) + biomass_potential = n.stores.loc[biomass_stores, "e_initial"] + + n.stores.e_initial = store_soc + n.storage_units.state_of_charge_initial = storage_unit_soc + + n.remove("GlobalConstraint", "CO2Limit") + n.stores.at["co2 atmosphere", "marginal_cost"] = -config["co2_price"] + + # handle co2 sequestration + assert sum(n.stores.carriers == "co2 stored") == 1, "Myopic operation not implemented for spatially resolved CO2 sequestration." + n.stores.at["co2 stored", 'e_nom'] = config['co2_sequestration_limit'] * 1e6 # t/a + + # reset co2 emissions + n.stores.loc[n.stores.carrier == 'co2 stored', "e_initial"] = 0. + n.stores.at["co2 atmosphere", "e_initial"] = 0. + + # replenish fossil gas and oil with 1000 TWh each + fossil_stores = n.stores.carrier.str.isin(["gas", "oil"]) + n.stores.loc[fossil_stores, 'e_initial'] = 1e9 + n.stores.loc[fossil_stores, 'e_nom'] = 10e9 + + # replenish annual solid biomass and biogas potentials + n.stores.loc[biomass_stores, "e_initial"] = biomass_potential + + # set storage bidding prices + # TODO bidding prices for Fischer-Tropsch, Methanation for H2? + bidding_prices = config["bidding_prices"] + for c in n.iterate_components({"Store", "Link", "StorageUnit"}): + c.df.marginal_cost.update(c.df.carrier.map(bidding_prices).dropna()) + + # deduct industry solid biomass + assert sum(n.stores.carriers == "solid biomass") == 1, "Myopic operation not implemented for spatially resolved solid biomass." + n.stores.at["EU solid biomass", "e_initial"] -= n.loads.at["solid biomass for industry", "p_set"] * 8760 + n.remove("Load", "solid biomass for industry") + + return n + + +def solve_network_myopic(n, config, opts='', **kwargs): + + rolling_horizon = config["operations"]["rolling_horizon"] + + freq = int(pd.infer_freq(n.snapshots)[:-1]) + window = rolling_horizon["window"] * 24 // freq + overlap = rolling_horizon["overlap"] * 24 // freq + kept = window - overlap + length = len(n.snapshots) + + assert kept > 0, f"Overlap ({overlap} days) must be smaller than windows ({window} days)." + + for i in range(length // kept): + + snapshots = n.snapshots[i * kept:(i + 1) * kept + overlap] + logger.info(f"Optimising operations from {snapshots[0]} to {snapshots[-1]}") + + n = solve_network(n, config, opts=opts, snapshots=snapshots, **kwargs) + + last_kept = n.snapshots[(i + 1) * kept - 1] + logger.info(f"Setting initial SOCs from {last_kept} for next iteration.\n") + + n.stores.e_initial = n.stores_t.e.loc[last_kept] + n.storage_units.state_of_charge_initial = n.storage_units_t.state_of_charge.loc[last_kept] + + return n + + +if __name__ == "__main__": + if 'snakemake' not in globals(): + from helper import mock_snakemake + snakemake = mock_snakemake( + 'solve_operations_network_myopic', + capacity_year=1952, + simpl='', + opts='', + clusters=37, + lv=2.0, + sector_opts='Co2L0-25H-T-H-B-I-A', + planning_horizons=2030, + weather_year=2013 + ) + + logging.basicConfig(filename=snakemake.log.python, + level=snakemake.config['logging_level']) + + tmpdir = snakemake.config['solving'].get('tmpdir') + if tmpdir is not None: + from pathlib import Path + Path(tmpdir).mkdir(parents=True, exist_ok=True) + + config = snakemake.config["operations"] + overrides = override_component_attrs(snakemake.input.overrides) + + n = pypsa.Network(snakemake.input.pre, override_component_attrs=overrides) + + n_post = pypsa.Network(snakemake.input.post, override_component_attrs=overrides) + n = set_parameters_from_optimized(n, n_post) + del n_post + + n_previous = pypsa.Network(snakemake.input.previous, override_component_attrs=overrides) + store_soc = n_previous.stores_t.e.iloc[-1] + storage_unit_soc = n_previous.storage_units_t.state_of_charge.iloc[-1] + del n_previous + + n = remove_unused_components(n) + n = add_load_shedding(n) + n = prepare_myopic(n, config, store_soc, storage_unit_soc) + + opts = snakemake.wildcards.sector_opts.split('-') + solve_opts = snakemake.config['solving']['options'] + solve_opts['skip_iterations'] = True + + n = prepare_network(n, solve_opts) + + n = solve_network_myopic( + n, + config=snakemake.config, + opts=opts, + solver_dir=tmpdir, + solver_logfile=snakemake.log.solver + ) + + n.export_to_netcdf(snakemake.output[0]) From 7554e4f47203fc7a8affea99ec83d0d45feacde4 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 1 Aug 2022 11:58:20 +0200 Subject: [PATCH 23/31] operations: adjust output file path --- Snakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 981c340d..79d78274 100644 --- a/Snakefile +++ b/Snakefile @@ -687,7 +687,7 @@ rule solve_operations_network: input: pre=RDIR + "/prenetworks/elec{weather_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", post=RDIR + "/postnetworks/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - output: "elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc" + output: RDIR + "/operations/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc" shadow: "shallow" log: solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", @@ -708,7 +708,7 @@ rule solve_operations_network_myopic: pre=RDIR + "/prenetworks/elec{weather_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", post=RDIR + "/postnetworks/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", previous=solved_previous_year - output: "elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc" + output: RDIR + "/operations/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc" shadow: "shallow" log: solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", From a58d67c6c12bb9389b79eb6102a49320c4a9d2af Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 1 Aug 2022 11:59:57 +0200 Subject: [PATCH 24/31] operations: add config options to config.default.yaml --- config.default.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/config.default.yaml b/config.default.yaml index 3e6f76a0..3f1637bf 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -385,6 +385,20 @@ solving: #feasopt_tolerance: 1.e-6 mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 +operations: + rolling_horizon: + window: 10 # days + overlap: 8 # days + bidding_prices: # EUR/MW + H2 Electrolysis: -10 + H2 Fuel Cell: 200 + urban central water tanks charger: -10 + urban central water tanks discharger: 10 + hydro: 70 + solid biomass: 150 + biogas: 100 + co2_price: 500 # EUR/t + co2_sequestation_limit: 200 # Mt/a plotting: map: From 26b21144c67cb0f14ba1b9b3a50ab13973c7d483 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 1 Aug 2022 12:03:24 +0200 Subject: [PATCH 25/31] operations: add some comments on multiyear contiguity --- scripts/solve_operations_network_myopic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/solve_operations_network_myopic.py b/scripts/solve_operations_network_myopic.py index a0e30b0a..fae86cb7 100644 --- a/scripts/solve_operations_network_myopic.py +++ b/scripts/solve_operations_network_myopic.py @@ -21,9 +21,11 @@ def prepare_myopic(n, config, store_soc, storage_unit_soc): biomass_stores = n.stores.carrier.str.isin(["solid biomass", "biogas"]) biomass_potential = n.stores.loc[biomass_stores, "e_initial"] + # storage level contiguity across years n.stores.e_initial = store_soc n.storage_units.state_of_charge_initial = storage_unit_soc + # replace co2 limit with co2 price n.remove("GlobalConstraint", "CO2Limit") n.stores.at["co2 atmosphere", "marginal_cost"] = -config["co2_price"] @@ -44,7 +46,6 @@ def prepare_myopic(n, config, store_soc, storage_unit_soc): n.stores.loc[biomass_stores, "e_initial"] = biomass_potential # set storage bidding prices - # TODO bidding prices for Fischer-Tropsch, Methanation for H2? bidding_prices = config["bidding_prices"] for c in n.iterate_components({"Store", "Link", "StorageUnit"}): c.df.marginal_cost.update(c.df.carrier.map(bidding_prices).dropna()) From 574b77fcc3dcd7f97bbccc71917a14ad2fc7695f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 1 Aug 2022 12:11:41 +0200 Subject: [PATCH 26/31] operations: add final iteration until end-of-year --- scripts/solve_operations_network_myopic.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/solve_operations_network_myopic.py b/scripts/solve_operations_network_myopic.py index fae86cb7..c403465f 100644 --- a/scripts/solve_operations_network_myopic.py +++ b/scripts/solve_operations_network_myopic.py @@ -83,6 +83,10 @@ def solve_network_myopic(n, config, opts='', **kwargs): n.stores.e_initial = n.stores_t.e.loc[last_kept] n.storage_units.state_of_charge_initial = n.storage_units_t.state_of_charge.loc[last_kept] + # final segment until end of year + snapshots = n.snapshots[(i + 1) * kept - 1:] + n = solve_network(n, config, opts=opts, snapshots=snapshots, **kwargs) + return n From 26e7e1246b7e748f1a3d3093d1fa9160ab925f9f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 1 Aug 2022 12:14:45 +0200 Subject: [PATCH 27/31] operations: final iteration one step later --- scripts/solve_operations_network_myopic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/solve_operations_network_myopic.py b/scripts/solve_operations_network_myopic.py index c403465f..0591ee70 100644 --- a/scripts/solve_operations_network_myopic.py +++ b/scripts/solve_operations_network_myopic.py @@ -84,7 +84,7 @@ def solve_network_myopic(n, config, opts='', **kwargs): n.storage_units.state_of_charge_initial = n.storage_units_t.state_of_charge.loc[last_kept] # final segment until end of year - snapshots = n.snapshots[(i + 1) * kept - 1:] + snapshots = n.snapshots[(i + 1) * kept:] n = solve_network(n, config, opts=opts, snapshots=snapshots, **kwargs) return n From a65cf7cac48c62371723c6d28d2dc8bef8370bcd Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 5 Aug 2022 17:44:08 +0200 Subject: [PATCH 28/31] make_summary: add weather_year wildcard to multiindex --- scripts/make_summary.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 8d5f4e48..358b7b5c 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -528,7 +528,7 @@ def make_summaries(networks_dict): columns = pd.MultiIndex.from_tuples( networks_dict.keys(), - names=["cluster", "lv", "opt", "planning_horizon"] + names=["weather_year", "cluster", "lv", "opt", "planning_horizon"] ) df = {} @@ -562,8 +562,9 @@ if __name__ == "__main__": snakemake = mock_snakemake('make_summary') networks_dict = { - (cluster, lv, opt+sector_opt, planning_horizon) : - snakemake.config['results_dir'] + snakemake.config['run'] + f'/postnetworks/elec_s{simpl}_{cluster}_lv{lv}_{opt}_{sector_opt}_{planning_horizon}.nc' \ + (weather_year, cluster, lv, opt+sector_opt, planning_horizon) : + snakemake.config['results_dir'] + snakemake.config['run'] + f'/postnetworks/elec{weather_year}_s{simpl}_{cluster}_lv{lv}_{opt}_{sector_opt}_{planning_horizon}.nc' \ + for weather_year in snakemake.config['scenario']['weather_year'] for simpl in snakemake.config['scenario']['simpl'] \ for cluster in snakemake.config['scenario']['clusters'] \ for opt in snakemake.config['scenario']['opts'] \ From c1a7a5476f0255716e2895bbf1849a4889d126c3 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 5 Aug 2022 17:44:31 +0200 Subject: [PATCH 29/31] plot_network: geomap for PyPSA 0.20 --- scripts/plot_network.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/plot_network.py b/scripts/plot_network.py index b9dd4567..9643cc44 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -307,7 +307,7 @@ def plot_h2_map(network): ) n.plot( - geomap=False, + # geomap=False, bus_sizes=0, link_colors='#72d3d6', link_widths=link_widths_retro, @@ -441,7 +441,7 @@ def plot_ch4_map(network): ) n.plot( - geomap=False, + # geomap=False, ax=ax, bus_sizes=0., link_colors='#e8d1d1', @@ -451,7 +451,7 @@ def plot_ch4_map(network): ) n.plot( - geomap=False, + # geomap=False, ax=ax, bus_sizes=0., link_colors=link_color_used, From b962f133d8d845b163c9f59f6ef4ad58c64d3df6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 31 Aug 2022 15:19:44 +0200 Subject: [PATCH 30/31] add simpl wildcard to operation rules --- Snakefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Snakefile b/Snakefile index 79d78274..ee7c5815 100644 --- a/Snakefile +++ b/Snakefile @@ -685,9 +685,9 @@ if config["foresight"] == "myopic": rule solve_operations_network: input: - pre=RDIR + "/prenetworks/elec{weather_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - post=RDIR + "/postnetworks/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - output: RDIR + "/operations/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc" + pre=RDIR + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + post=RDIR + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + output: RDIR + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc" shadow: "shallow" log: solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", @@ -705,10 +705,10 @@ def solved_previous_year(wildcards): rule solve_operations_network_myopic: input: - pre=RDIR + "/prenetworks/elec{weather_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", - post=RDIR + "/postnetworks/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + pre=RDIR + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", + post=RDIR + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", previous=solved_previous_year - output: RDIR + "/operations/elec{capacity_year}_s_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc" + output: RDIR + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_myopic.nc" shadow: "shallow" log: solver=RDIR + "/logs/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}_solver.log", From 7a7c7f0344f6d92230599e170845ab3075438791 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 10 Sep 2022 19:48:40 +0200 Subject: [PATCH 31/31] add missing overrides to snakemake.input for operational rules --- Snakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Snakefile b/Snakefile index ee7c5815..c11fba06 100644 --- a/Snakefile +++ b/Snakefile @@ -685,6 +685,7 @@ if config["foresight"] == "myopic": rule solve_operations_network: input: + overrides="data/override_component_attrs", pre=RDIR + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", post=RDIR + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", output: RDIR + "/operations/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_{weather_year}.nc" @@ -705,6 +706,7 @@ def solved_previous_year(wildcards): rule solve_operations_network_myopic: input: + overrides="data/override_component_attrs", pre=RDIR + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", post=RDIR + "/postnetworks/elec{capacity_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc", previous=solved_previous_year