bjoern/pypsa-eur
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

merge master

Browse Source
This commit is contained in:
lisazeyen 2023-08-30 12:03:49 +02:00
commit 9ec7442c69
14 changed files with 152 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.gitignore vendored
View File

@ -29,18 +29,18 @@ dconf
/data/links_p_nom.csv
/data/*totals.csv
/data/biomass*
/data/emobility/
/data/eea*
/data/jrc*
/data/bundle-sector/emobility/
/data/bundle-sector/eea*
/data/bundle-sector/jrc*
/data/heating/
/data/eurostat*
/data/bundle-sector/eurostat*
/data/odyssee/
/data/transport_data.csv
/data/switzerland*
/data/bundle-sector/switzerland*
/data/.nfs*
/data/Industrial_Database.csv
/data/bundle-sector/Industrial_Database.csv
/data/retro/tabula-calculator-calcsetbuilding.csv
/data/nuts*
/data/bundle-sector/nuts*
data/gas_network/scigrid-gas/
data/costs_*.csv

6
doc/release_notes.rst
View File

@ -14,6 +14,12 @@ Upcoming Release
* For industry distribution, use EPRTR as fallback if ETS data is not available.
* The minimum capacity for renewable generators when using the myopic option has been fixed.
* Files downloaded from zenodo are now write-protected to prevent accidental re-download.
* Files extracted from sector-coupled data bundle have been moved from ``data/`` to ``data/sector-bundle``.
* New feature multi-decade optimisation with perfect foresight.
PyPSA-Eur 0.8.1 (27th July 2023)

6
rules/build_electricity.smk
View File

@ -350,7 +350,9 @@ rule add_electricity:
hydro_capacities=ancient("data/bundle/hydro_capacities.csv"),
geth_hydro_capacities="data/geth2015_hydro_capacities.csv",
unit_commitment="data/unit_commitment.csv",
fuel_price=RESOURCES + "monthly_fuel_price.csv",
fuel_price=RESOURCES + "monthly_fuel_price.csv"
if config["conventional"]["dynamic_fuel_price"]
else [],
load=RESOURCES + "load.csv",
nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
output:
@ -478,7 +480,7 @@ rule prepare_network:
input:
RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
tech_costs=COSTS,
co2_price=RESOURCES + "co2_price.csv",
co2_price=lambda w: RESOURCES + "co2_price.csv" if "Ept" in w.opts else [],
output:
RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
log:

28
rules/build_sector.smk
View File

@ -242,9 +242,9 @@ rule build_energy_totals:
energy=config["energy"],
input:
nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
co2="data/eea/UNFCCC_v23.csv",
swiss="data/switzerland-sfoe/switzerland-new_format.csv",
idees="data/jrc-idees-2015",
co2="data/bundle-sector/eea/UNFCCC_v23.csv",
swiss="data/bundle-sector/switzerland-sfoe/switzerland-new_format.csv",
idees="data/bundle-sector/jrc-idees-2015",
district_heat_share="data/district_heat_share.csv",
eurostat=input_eurostat,
output:
@ -272,7 +272,7 @@ rule build_biomass_potentials:
"https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx",
keep_local=True,
),
nuts2="data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21
nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21
regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"),
swiss_cantons=ancient("data/bundle/ch_cantons.csv"),
@ -366,7 +366,7 @@ if not config["sector"]["regional_co2_sequestration_potential"]["enable"]:
rule build_salt_cavern_potentials:
input:
salt_caverns="data/h2_salt_caverns_GWh_per_sqkm.geojson",
salt_caverns="data/bundle-sector/h2_salt_caverns_GWh_per_sqkm.geojson",
regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}_{clusters}.geojson",
output:
@ -388,7 +388,7 @@ rule build_ammonia_production:
params:
countries=config["countries"],
input:
usgs="data/myb1-2017-nitro.xls",
usgs="data/bundle-sector/myb1-2017-nitro.xls",
output:
ammonia_production=RESOURCES + "ammonia_production.csv",
threads: 1
@ -410,7 +410,7 @@ rule build_industry_sector_ratios:
ammonia=config["sector"].get("ammonia", False),
input:
ammonia_production=RESOURCES + "ammonia_production.csv",
idees="data/jrc-idees-2015",
idees="data/bundle-sector/jrc-idees-2015",
output:
industry_sector_ratios=RESOURCES + "industry_sector_ratios.csv",
threads: 1
@ -432,8 +432,8 @@ rule build_industrial_production_per_country:
countries=config["countries"],
input:
ammonia_production=RESOURCES + "ammonia_production.csv",
jrc="data/jrc-idees-2015",
eurostat="data/eurostat-energy_balances-may_2018_edition",
jrc="data/bundle-sector/jrc-idees-2015",
eurostat="data/bundle-sector/eurostat-energy_balances-may_2018_edition",
output:
industrial_production_per_country=RESOURCES
+ "industrial_production_per_country.csv",
@ -483,7 +483,7 @@ rule build_industrial_distribution_key:
input:
regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
hotmaps_industrial_database="data/Industrial_Database.csv",
hotmaps_industrial_database="data/bundle-sector/Industrial_Database.csv",
output:
industrial_distribution_key=RESOURCES
+ "industrial_distribution_key_elec_s{simpl}_{clusters}.csv",
@ -558,7 +558,7 @@ rule build_industrial_energy_demand_per_country_today:
countries=config["countries"],
industry=config["industry"],
input:
jrc="data/jrc-idees-2015",
jrc="data/bundle-sector/jrc-idees-2015",
ammonia_production=RESOURCES + "ammonia_production.csv",
industrial_production_per_country=RESOURCES
+ "industrial_production_per_country.csv",
@ -684,8 +684,8 @@ rule build_transport_demand:
pop_weighted_energy_totals=RESOURCES
+ "pop_weighted_energy_totals_s{simpl}_{clusters}.csv",
transport_data=RESOURCES + "transport_data.csv",
traffic_data_KFZ="data/emobility/KFZ__count",
traffic_data_Pkw="data/emobility/Pkw__count",
traffic_data_KFZ="data/bundle-sector/emobility/KFZ__count",
traffic_data_Pkw="data/bundle-sector/emobility/Pkw__count",
temp_air_total=RESOURCES + "temp_air_total_elec_s{simpl}_{clusters}.nc",
output:
transport_demand=RESOURCES + "transport_demand_s{simpl}_{clusters}.csv",
@ -734,7 +734,7 @@ rule prepare_sector_network:
avail_profile=RESOURCES + "avail_profile_s{simpl}_{clusters}.csv",
dsm_profile=RESOURCES + "dsm_profile_s{simpl}_{clusters}.csv",
co2_totals_name=RESOURCES + "co2_totals.csv",
co2="data/eea/UNFCCC_v23.csv",
co2="data/bundle-sector/eea/UNFCCC_v23.csv",
biomass_potentials=RESOURCES + "biomass_potentials_s{simpl}_{clusters}.csv",
heat_profile="data/heat_load_profile_BDEW.csv",
costs="data/costs_{}.csv".format(config["costs"]["year"])

2
rules/common.smk
View File

@ -42,7 +42,7 @@ def has_internet_access(url="www.zenodo.org") -> bool:
def input_eurostat(w):
# 2016 includes BA, 2017 does not
report_year = config["energy"]["eurostat_report_year"]
return f"data/eurostat-energy_balances-june_{report_year}_edition"
return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition"
def solved_previous_horizon(wildcards):

4
rules/postprocess.smk
View File

@ -135,6 +135,8 @@ rule plot_summary:
countries=config["countries"],
planning_horizons=config["scenario"]["planning_horizons"],
sector_opts=config["scenario"]["sector_opts"],
emissions_scope=config["energy"]["emissions"],
eurostat_report_year=config["energy"]["eurostat_report_year"],
plotting=config["plotting"],
RDIR=RDIR,
input:
@ -142,7 +144,7 @@ rule plot_summary:
energy=RESULTS + "csvs/energy.csv",
balances=RESULTS + "csvs/supply_energy.csv",
eurostat=input_eurostat,
co2="data/eea/UNFCCC_v23.csv",
co2="data/bundle-sector/eea/UNFCCC_v23.csv",
output:
costs=RESULTS + "graphs/costs.pdf",
energy=RESULTS + "graphs/energy.pdf",

42
rules/retrieve.smk
View File

@ -27,7 +27,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle",
rule retrieve_databundle:
output:
expand("data/bundle/{file}", file=datafiles),
protected(expand("data/bundle/{file}", file=datafiles)),
log:
LOGS + "retrieve_databundle.log",
resources:
@ -92,7 +92,7 @@ if config["enable"]["retrieve"] and config["enable"].get(
static=True,
),
output:
RESOURCES + "natura.tiff",
protected(RESOURCES + "natura.tiff"),
log:
LOGS + "retrieve_natura_raster.log",
resources:
@ -106,22 +106,30 @@ if config["enable"]["retrieve"] and config["enable"].get(
"retrieve_sector_databundle", True
):
datafiles = [
"data/eea/UNFCCC_v23.csv",
"data/switzerland-sfoe/switzerland-new_format.csv",
"data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
"data/myb1-2017-nitro.xls",
"data/Industrial_Database.csv",
"data/emobility/KFZ__count",
"data/emobility/Pkw__count",
"data/h2_salt_caverns_GWh_per_sqkm.geojson",
directory("data/eurostat-energy_balances-june_2016_edition"),
directory("data/eurostat-energy_balances-may_2018_edition"),
directory("data/jrc-idees-2015"),
"eea/UNFCCC_v23.csv",
"switzerland-sfoe/switzerland-new_format.csv",
"nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
"myb1-2017-nitro.xls",
"Industrial_Database.csv",
"emobility/KFZ__count",
"emobility/Pkw__count",
"h2_salt_caverns_GWh_per_sqkm.geojson",
]
datafolders = [
protected(
directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition")
),
protected(
directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition")
),
protected(directory("data/bundle-sector/jrc-idees-2015")),
]
rule retrieve_sector_databundle:
output:
*datafiles,
protected(expand("data/bundle-sector/{files}", files=datafiles)),
*datafolders,
log:
LOGS + "retrieve_sector_databundle.log",
retries: 2
@ -143,7 +151,9 @@ if config["enable"]["retrieve"] and (
rule retrieve_gas_infrastructure_data:
output:
expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles),
protected(
expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles)
),
log:
LOGS + "retrieve_gas_infrastructure_data.log",
retries: 2
@ -187,7 +197,7 @@ if config["enable"]["retrieve"]:
static=True,
),
output:
"data/shipdensity_global.zip",
protected("data/shipdensity_global.zip"),
log:
LOGS + "retrieve_ship_raster.log",
resources:

2
scripts/add_electricity.py
View File

@ -165,7 +165,7 @@ def sanitize_carriers(n, config):
nice_names = (
pd.Series(config["plotting"]["nice_names"])
.reindex(carrier_i)
.fillna(carrier_i.to_series().str.title())
.fillna(carrier_i.to_series())
)
n.carriers["nice_name"] = n.carriers.nice_name.where(
n.carriers.nice_name != "", nice_names

14
scripts/add_existing_baseyear.py
View File

@ -446,15 +446,23 @@ def add_heating_capacities_installed_before_baseyear(
# split existing capacities between residential and services
# proportional to energy demand
p_set_sum = n.loads_t.p_set.sum()
ratio_residential = pd.Series(
[
(
n.loads_t.p_set.sum()[f"{node} residential rural heat"]
p_set_sum[f"{node} residential rural heat"]
/ (
n.loads_t.p_set.sum()[f"{node} residential rural heat"]
+ n.loads_t.p_set.sum()[f"{node} services rural heat"]
p_set_sum[f"{node} residential rural heat"]
+ p_set_sum[f"{node} services rural heat"]
)
)
# if rural heating demand for one of the nodes doesn't exist,
# then columns were dropped before and heating demand share should be 0.0
if all(
f"{node} {service} rural heat" in p_set_sum.index
for service in ["residential", "services"]
)
else 0.0
for node in nodal_df.index
],
index=nodal_df.index,

14
scripts/build_industrial_distribution_key.py
View File

@ -13,10 +13,13 @@ logger = logging.getLogger(__name__)
import uuid
from itertools import product
import country_converter as coco
import geopandas as gpd
import pandas as pd
from packaging.version import Version, parse
cc = coco.CountryConverter()
def locate_missing_industrial_sites(df):
"""
@ -107,6 +110,17 @@ def prepare_hotmaps_database(regions):
# concat not duplicated and filtered gdf
gdf = pd.concat([gdf.drop(duplicated_i), gdf_filtered]).sort_index()
# the .sjoin can lead to duplicates if a geom is in two overlapping regions
if gdf.index.duplicated().any():
# get all duplicated entries
duplicated_i = gdf.index[gdf.index.duplicated()]
# convert from raw data country name to iso-2-code
code = cc.convert(gdf.loc[duplicated_i, "Country"], to="iso2")
# screen out malformed country allocation
gdf_filtered = gdf.loc[duplicated_i].query("country == @code")
# concat not duplicated and filtered gdf
gdf = pd.concat([gdf.drop(duplicated_i), gdf_filtered])
return gdf

2
scripts/make_summary.py
View File

@ -711,5 +711,5 @@ if __name__ == "__main__":
if snakemake.params.foresight == "myopic":
cumulative_cost = calculate_cumulative_cost()
cumulative_cost.to_csv(
"results/" + snakemake.params.RDIR + "/csvs/cumulative_cost.csv"
"results/" + snakemake.params.RDIR + "csvs/cumulative_cost.csv"
)

25
scripts/plot_summary.py
View File

@ -457,7 +457,6 @@ def plot_carbon_budget_distribution(input_eurostat):
"""
Plot historical carbon emissions in the EU and decarbonization path.
"""
import seaborn as sns
sns.set()
@ -502,6 +501,14 @@ def plot_carbon_budget_distribution(input_eurostat):
# plot committed and under-discussion targets
# (notice that historical emissions include all countries in the
# network, but targets refer to EU)
ax1.plot(
[2020],
[0.8 * emissions[1990]],
marker="*",
markersize=12,
markerfacecolor="black",
markeredgecolor="black",
)
ax1.plot(
[2030],
@ -512,7 +519,23 @@ def plot_carbon_budget_distribution(input_eurostat):
markeredgecolor="black",
)
ax1.plot(
[2030],
[0.6 * emissions[1990]],
marker="*",
markersize=12,
markerfacecolor="black",
markeredgecolor="black",
)
ax1.plot(
[2050, 2050],
[x * emissions[1990] for x in [0.2, 0.05]],
color="gray",
linewidth=2,
marker="_",
alpha=0.5,
)
ax1.plot(
[2050],

61
scripts/prepare_sector_network.py
View File

@ -46,7 +46,6 @@ def define_spatial(nodes, options):
----------
nodes : list-like
"""
global spatial
spatial.nodes = nodes
@ -192,17 +191,17 @@ def get(item, investment_year=None):
def co2_emissions_year(
countries, input_eurostat, opts, snakemake, year
countries, input_eurostat, opts, emissions_scope, report_year, year
):
"""
Calculate CO2 emissions in one specific year (e.g. 1990 or 2018).
"""
emissions_scope = snakemake.config["energy"]["emissions"]
emissions_scope = snakemake.params.energy["emissions"]
eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope)
# TODO: read Eurostat data from year > 2014
# this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK
report_year = snakemake.config["energy"]["eurostat_report_year"]
report_year = snakemake.params.energy["eurostat_report_year"]
if year > 2014:
eurostat_co2 = build_eurostat_co2(
input_eurostat, countries, report_year, year=2014
@ -241,12 +240,24 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year):
countries = snakemake.params.countries
e_1990 = co2_emissions_year(
countries, input_eurostat, opts, snakemake, year=1990
countries,
input_eurostat,
opts,
emissions_scope,
report_year,
input_co2,
year=1990,
)
# emissions at the beginning of the path (last year available 2018)
e_0 = co2_emissions_year(
countries, input_eurostat, opts, snakemake, year=2018,
countries,
input_eurostat,
opts,
emissions_scope,
report_year,
input_co2,
year=2018,
)
planning_horizons = snakemake.params.planning_horizons
@ -357,7 +368,6 @@ def update_wind_solar_costs(n, costs):
Update costs for wind and solar generators added with pypsa-eur to those
cost in the planning year.
"""
# NB: solar costs are also manipulated for rooftop
# when distribution grid is inserted
n.generators.loc[n.generators.carrier == "solar", "capital_cost"] = costs.at[
@ -435,7 +445,6 @@ def add_carrier_buses(n, carrier, nodes=None):
"""
Add buses to connect e.g. coal, nuclear and oil plants.
"""
if nodes is None:
nodes = vars(spatial)[carrier].nodes
location = vars(spatial)[carrier].locations
@ -716,6 +725,7 @@ def average_every_nhours(n, offset):
return m
def cycling_shift(df, steps=1):
"""
Cyclic shift on index of pd.Series|pd.DataFrame by number of steps.
@ -1150,7 +1160,6 @@ def add_storage_and_grids(n, costs):
e_cyclic=True,
carrier="H2 Store",
capital_cost=h2_capital_cost,
lifetime=costs.at["hydrogen storage tank type 1 including compressor", "lifetime"],
)
if options["gas_network"] or options["H2_retrofit"]:
@ -3076,7 +3085,6 @@ def maybe_adjust_costs_and_potentials(n, opts):
logger.info(f"changing {attr} for {carrier} by factor {factor}")
# TODO this should rather be a config no wildcard
def limit_individual_line_extension(n, maxext):
logger.info(f"Limiting new HVAC and HVDC extensions to {maxext} MW")
n.lines["s_nom_max"] = n.lines["s_nom"] + maxext
@ -3211,7 +3219,7 @@ def apply_time_segmentation(
df = pnl.copy()
df.columns = pd.MultiIndex.from_product([[c.name], [attr], df.columns])
raw = pd.concat([raw, df], axis=1)
raw = raw.dropna(axis=1)
# normalise all time-dependent data
annual_max = raw.max().replace(0, 1)
raw = raw.div(annual_max, level=0)
@ -3268,28 +3276,26 @@ def set_temporal_aggregation(n, opts, solver_name):
# segments with package tsam
m = re.match(r"^(\d+)seg$", o, re.IGNORECASE)
if m is not None:
if snakemake.params.foresight!="perfect":
segments = int(m[1])
logger.info(f"Use temporal segmentation with {segments} segments")
n = apply_time_segmentation(n, segments, solver_name=solver_name)
break
else:
logger.info("Apply temporal segmentation at prepare_perfect_foresight.")
segments = int(m[1])
logger.info(f"Use temporal segmentation with {segments} segments")
n = apply_time_segmentation(n, segments, solver_name=solver_name)
break
return n
#%%
if __name__ == "__main__":
if "snakemake" not in globals():
from _helpers import mock_snakemake
snakemake = mock_snakemake(
"prepare_sector_network",
configfiles="test/config.overnight.yaml",
simpl="",
opts="",
clusters="37",
ll="v1.0",
sector_opts="60SEG-T-H-B-I-A-solar+p3-dist1",
planning_horizons="2050",
clusters="5",
ll="v1.5",
sector_opts="CO2L0-24H-T-H-B-I-A-solar+p3-dist1",
planning_horizons="2030",
)
logging.basicConfig(level=snakemake.config["logging"]["level"])
@ -3391,7 +3397,6 @@ if __name__ == "__main__":
add_allam(n, costs)
solver_name = snakemake.config["solving"]["solver"]["name"]
n = set_temporal_aggregation(n, opts, solver_name)
limit_type = "config"
@ -3404,8 +3409,14 @@ if __name__ == "__main__":
if not os.path.exists(fn):
emissions_scope = snakemake.params.emissions_scope
report_year = snakemake.params.eurostat_report_year
input_co2 = snakemake.input.co2
build_carbon_budget(
o, snakemake.input.eurostat, fn, emissions_scope, report_year
o,
snakemake.input.eurostat,
fn,
emissions_scope,
report_year,
input_co2,
)
co2_cap = pd.read_csv(fn, index_col=0).squeeze()
limit = co2_cap.loc[investment_year]

5
scripts/solve_network.py
View File

@ -110,6 +110,9 @@ def _add_land_use_constraint(n):
# warning: this will miss existing offwind which is not classed AC-DC and has carrier 'offwind'
for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]:
extendable_i = (n.generators.carrier == carrier) & n.generators.p_nom_extendable
n.generators.loc[extendable_i, "p_nom_min"] = 0
ext_i = (n.generators.carrier == carrier) & ~n.generators.p_nom_extendable
existing = (
n.generators.loc[ext_i, "p_nom"]
@ -126,7 +129,7 @@ def _add_land_use_constraint(n):
if len(existing_large):
logger.warning(
f"Existing capacities larger than technical potential for {existing_large},\
adjust technical potential to existing capacities"
adjust technical potential to existing capacities"
)
n.generators.loc[existing_large, "p_nom_max"] = n.generators.loc[
existing_large, "p_nom_min"