Merge branch 'master' into ci-review

This commit is contained in:
Fabian 2023-02-21 17:57:27 +01:00
commit 8bf6238785
16 changed files with 209 additions and 98 deletions

View File

@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: : 2021 The PyPSA-Eur Authors
# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors
#
# SPDX-License-Identifier: CC0-1.0

View File

@ -1,6 +1,6 @@
MIT License
Copyright 2017-2021 The PyPSA-Eur Authors
Copyright 2017-2023 The PyPSA-Eur Authors
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in

View File

@ -1,4 +1,4 @@
version: 0.6.0
version: 0.7.0
logging_level: INFO
@ -538,6 +538,7 @@ plotting:
natural gas: '#e05b09'
CCGT: '#a85522'
CCGT marginal: '#a85522'
allam: '#B98F76'
gas for industry co2 to atmosphere: '#692e0a'
gas for industry co2 to stored: '#8a3400'
gas for industry: '#853403'

View File

@ -63,17 +63,17 @@ master_doc = 'index'
# General information about the project.
project = u'PyPSA-Eur-Sec'
copyright = u'2019-2021 Tom Brown (KIT, TUB), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB), Fabian Neumann (TUB)'
author = u'2019-2021 Tom Brown (KIT, TUB), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB), Fabian Neumann (TUB)'
copyright = u'2019-2023 Tom Brown (KIT, TUB), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB), Fabian Neumann (TUB)'
author = u'2019-2023 Tom Brown (KIT, TUB), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB), Fabian Neumann (TUB)'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = u'0.6'
version = u'0.7'
# The full version, including alpha/beta/rc tags.
release = u'0.6.0'
release = u'0.7.0'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.

View File

@ -16,6 +16,7 @@ existing heating potentials,existing_infrastructure/existing_heating_raw.csv,unk
IRENA existing VRE capacities,existing_infrastructure/{solar|onwind|offwind}_capcity_IRENA.csv,unknown,https://www.irena.org/Statistics/Download-Data
USGS ammonia production,myb1-2017-nitro.xls,unknown,https://www.usgs.gov/centers/nmic/nitrogen-statistics-and-information
hydrogen salt cavern potentials,h2_salt_caverns_GWh_per_sqkm.geojson,CC BY 4.0,https://doi.org/10.1016/j.ijhydene.2019.12.161 https://doi.org/10.20944/preprints201910.0187.v1
international port trade volumes,attributed_ports.json,CC BY 4.0,https://datacatalog.worldbank.org/search/dataset/0038118/Global---International-Ports
hotmaps industrial site database,Industrial_Database.csv,CC BY 4.0,https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database
Hotmaps building stock data,data_building_stock.csv,CC BY 4.0,https://gitlab.com/hotmaps/building-stock
U-values Poland,u_values_poland.csv,unknown,https://data.europa.eu/euodp/de/data/dataset/building-stock-observatory

1 description file/folder licence source
16 IRENA existing VRE capacities existing_infrastructure/{solar|onwind|offwind}_capcity_IRENA.csv unknown https://www.irena.org/Statistics/Download-Data
17 USGS ammonia production myb1-2017-nitro.xls unknown https://www.usgs.gov/centers/nmic/nitrogen-statistics-and-information
18 hydrogen salt cavern potentials h2_salt_caverns_GWh_per_sqkm.geojson CC BY 4.0 https://doi.org/10.1016/j.ijhydene.2019.12.161 https://doi.org/10.20944/preprints201910.0187.v1
19 international port trade volumes attributed_ports.json CC BY 4.0 https://datacatalog.worldbank.org/search/dataset/0038118/Global---International-Ports
20 hotmaps industrial site database Industrial_Database.csv CC BY 4.0 https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database
21 Hotmaps building stock data data_building_stock.csv CC BY 4.0 https://gitlab.com/hotmaps/building-stock
22 U-values Poland u_values_poland.csv unknown https://data.europa.eu/euodp/de/data/dataset/building-stock-observatory

View File

@ -14,10 +14,6 @@ PyPSA-Eur-Sec: A Sector-Coupled Open Optimisation Model of the European Energy S
.. image:: https://img.shields.io/github/repo-size/pypsa/pypsa-eur-sec
:alt: GitHub repo size
.. image:: https://badges.gitter.im/PyPSA/community.svg
:target: https://gitter.im/PyPSA/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge
:alt: Chat on Gitter
PyPSA-Eur-Sec is an open model dataset of the European energy system at the
transmission network level that covers the full ENTSO-E area.
@ -101,6 +97,15 @@ Ministry for Education and Research (BMBF) <https://www.bmbf.de/en/index.html>`_
as part of the `Stromnetze Research Initiative
<http://forschung-stromnetze.info/projekte/grundlagen-und-konzepte-fuer-effiziente-dezentrale-stromnetze/>`_.
Workflow Outline
================
.. image:: ../graphics/workflow.png
.. note::
The graph above was generated using
``snakemake --rulegraph -F | sed -n "/digraph/,/}/p" | dot -Tpng -o workflow.png``
Documentation
=============

View File

@ -8,31 +8,49 @@ Future release
.. note::
This unreleased version currently may require the master branches of PyPSA, PyPSA-Eur, and the technology-data repository.
This release includes the addition of the European gas transmission network and
incorporates retrofitting options to hydrogen.
* new feature
PyPSA-Eur-Sec 0.7.0 (16th February 2023)
========================================
This release includes many new features. Highlights include new gas
infrastructure data with retrofitting options for hydrogen transport, improved
carbon management and infrastructure planning, regionalised potentials for
hydrogen underground storage and carbon sequestration, new applications for
biomass, and explicit modelling of methanol and ammonia as separate energy
carriers.
This release is known to work with `PyPSA-Eur
<https://github.com/PyPSA/pypsa-eur>`_ Version 0.7.0 and `Technology Data
<https://github.com/PyPSA/technology-data>`_ Version 0.5.0.
**Gas Transmission Network**
* New rule ``retrieve_gas_infrastructure_data`` that downloads and extracts the
SciGRID_gas `IGGIELGN <https://zenodo.org/record/4767098>`_ dataset from zenodo.
It includes data on the transmission routes, pipe diameters, capacities, pressure,
and whether the pipeline is bidirectional and carries H-Gas or L-Gas.
SciGRID_gas `IGGIELGN <https://zenodo.org/record/4767098>`_ dataset from
zenodo. It includes data on the transmission routes, pipe diameters,
capacities, pressure, and whether the pipeline is bidirectional and carries
H-Gas or L-Gas.
* New rule ``build_gas_network`` processes and cleans the pipeline data from SciGRID_gas.
Missing or uncertain pipeline capacities can be inferred by diameter.
* New rule ``build_gas_network`` processes and cleans the pipeline data from
SciGRID_gas. Missing or uncertain pipeline capacities can be inferred by
diameter.
* New rule ``build_gas_input_locations`` compiles the LNG import capacities
(including planned projects from gem.wiki), pipeline entry capacities and
local production capacities for each region of the model. These are the
regions where fossil gas can eventually enter the model.
(from the Global Energy Monitor's `Europe Gas Tracker
<https://globalenergymonitor.org/projects/europe-gas-tracker/>`_, pipeline
entry capacities and local production capacities for each region of the model.
These are the regions where fossil gas can eventually enter the model.
* New rule ``cluster_gas_network`` that clusters the gas transmission network
data to the model resolution. Cross-regional pipeline capacities are aggregated
(while pressure and diameter compatibility is ignored), intra-regional pipelines
are dropped. Lengths are recalculated based on the regions' centroids.
data to the model resolution. Cross-regional pipeline capacities are
aggregated (while pressure and diameter compatibility is ignored),
intra-regional pipelines are dropped. Lengths are recalculated based on the
regions' centroids.
* With the option ``sector: gas_network:``, the existing gas network is
added with a lossless transport model. A length-weighted `k-edge augmentation
* With the option ``sector: gas_network:``, the existing gas network is added
with a lossless transport model. A length-weighted `k-edge augmentation
algorithm
<https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_
can be run to add new candidate gas pipelines such that all regions of the
@ -41,69 +59,27 @@ incorporates retrofitting options to hydrogen.
the gas network is activated, all the gas demands are regionally disaggregated
as well.
* New constraint allows endogenous retrofitting of gas pipelines to hydrogen pipelines.
This option is activated via the setting ``sector: H2_retrofit:``. For every
unit of gas pipeline capacity dismantled, ``sector:
* New constraint allows endogenous retrofitting of gas pipelines to hydrogen
pipelines. This option is activated via the setting ``sector: H2_retrofit:``.
For every unit of gas pipeline capacity dismantled, ``sector:
H2_retrofit_capacity_per_CH4`` units are made available as hydrogen pipeline
capacity in the corresponding corridor. These repurposed hydrogen pipelines
have lower costs than new hydrogen pipelines. Both new and repurposed pipelines
can be built simultaneously. The retrofitting option ``sector: H2_retrofit:`` also works
with a copperplated methane infrastructure, i.e. when ``sector: gas_network: false``.
have lower costs than new hydrogen pipelines. Both new and repurposed
pipelines can be built simultaneously. The retrofitting option ``sector:
H2_retrofit:`` also works with a copperplated methane infrastructure, i.e.
when ``sector: gas_network: false``.
* New hydrogen pipelines can now be built where there are already power or gas
transmission routes. Previously, only the electricity transmission routes were
considered.
**New features and functionality**
* Add option to aggregate network temporally using representative snapshots or segments (with tsam package)
* Add option for biomass boilers (wood pellets) for decentral heating
* Add option for BioSNG (methane from biomass) with and without CC
* Add option for BtL (Biomass to liquid fuel/oil) with and without CC
* Add option for minimum part load for Fischer-Tropsch plants (default: 90%) and methanolisation plants (default: 50%).
* Units are assigned to the buses. These only provide a better understanding. The specifications of the units are not taken into account in the optimisation, which means that no automatic conversion of units takes place.
* Option ``retrieve_sector_databundle`` to automatically retrieve and extract data bundle.
* Add option to use waste heat of electrolysis in district heating networks (``use_electrolysis_waste_heat``).
* Add regionalised hydrogen salt cavern storage potentials from `Technical Potential of Salt Caverns for Hydrogen Storage in Europe <https://doi.org/10.20944/preprints201910.0187.v1>`_.
* Add option to sweep the global CO2 sequestration potentials with keyword ``seq200`` in the ``{sector_opts}`` wildcard (for limit of 200 Mt CO2).
* Add option to resolve ammonia as separate energy carrier with Haber-Bosch
synthesis, ammonia cracking, storage and industrial demand. The ammonia
carrier can be nodally resolved or copperplated across Europe. This feature is
controlled by ``sector: ammonia:``.
* Add methanol as energy carrier, methanolisation as process, and option for methanol demand in shipping sector.
* Updated `data bundle <https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz>`_ that includes the hydrogan salt cavern storage potentials.
* Updated and extended documentation in <https://pypsa-eur-sec.readthedocs.io/en/latest/>
* Shipping demand now defaults to (synthetic) oil rather than liquefied hydrogen until 2050.
* Improved network plots including better legends, hydrogen retrofitting network display, and change to EqualEarth projection.
* New config options for changing energy demands in aviation
(``aviation_demand_factor``) and HVC industry (``HVC_demand_factor``), as well
as explicit ICE shares for land transport (``land_transport_ice_share``) and
agriculture machinery (``agriculture_machinery_oil_share``).
**Carbon Management and Biomass**
* Add option to spatially resolve carrier representing stored carbon dioxide
(``co2_spatial``). This allows for more detailed modelling of CCUTS, e.g.
regarding the capturing of industrial process emissions, usage as feedstock
for electrofuels, transport of carbon dioxide, and geological sequestration sites.
* Add option for planning a new carbon dioxide network (``co2network``).
for electrofuels, transport of carbon dioxide, and geological sequestration
sites.
* Add option for regionally-resolved geological carbon dioxide sequestration
potentials through new rule ``build_sequestration_potentials`` based on
@ -115,18 +91,135 @@ incorporates retrofitting options to hydrogen.
potential. The defaults are preliminary and will be validated the next
release.
* Separate option to regionally resolve biomass (``biomass_spatial``) from
option to allow biomass transport (``biomass_transport``).
* Add option to sweep the global CO2 sequestration potentials with keyword
``seq200`` in the ``{sector_opts}`` wildcard (for limit of 200 Mt CO2).
* Add option to include `Allam cycle gas power plants
<https://en.wikipedia.org/wiki/Allam_power_cycle>`_ (``allam_cycle``).
* Add option for planning a new carbon dioxide network (``co2network``).
* Separate option to regionally resolve biomass (``biomass_spatial``) from
option to allow biomass transport (``biomass_transport``).
* Add option for biomass boilers (wood pellets) for decentral heating.
* Add option for BioSNG (methane from biomass) with and without carbon capture.
* Add option for BtL (biomass to liquid fuel/oil) with and without carbon
capture.
**Other new features**
* Add regionalised hydrogen salt cavern storage potentials from `Technical
Potential of Salt Caverns for Hydrogen Storage in Europe
<https://doi.org/10.20944/preprints201910.0187.v1>`_. This data is compiled in
a new rule ``build_salt_cavern_potentials``.
* Add option to resolve ammonia as separate energy carrier with Haber-Bosch
synthesis, ammonia cracking, storage and industrial demand. The ammonia
carrier can be nodally resolved or copperplated across Europe (see
``ammonia``).
* Add methanol as energy carrier, methanolisation as process, and option for
methanol demand in shipping sector.
* Shipping demand now defaults to methanol rather than liquefied hydrogen
until 2050.
* Demand for liquid hydrogen in international shipping is now geographically
distributed by port trade volumes in a new rule ``build_shipping_demand``
using data from the `World Bank Data Catalogue
<https://datacatalog.worldbank.org/search/dataset/0038118/Global---International-Ports>`_.
Domestic shipping remains distributed by population.
* Add option to aggregate network temporally using representative snapshots or
segments (with `tsam <https://github.com/FZJ-IEK3-VSA/tsam>`_).
* Add option for minimum part load for Fischer-Tropsch plants (default: 90%) and
methanolisation plants (default: 50%).
* Add option to use waste heat of electrolysis in district heating networks
(``use_electrolysis_waste_heat``).
* Add option for coal CHPs with carbon capture (see ``coal_cc``).
* In overnight optimisation, it is now possible to specify a year for the
technology cost projections separate from the planning horizon.
* New config options for changing energy demands in aviation
(``aviation_demand_factor``) and HVC industry (``HVC_demand_factor``), as well
as explicit ICE shares for land transport (``land_transport_ice_share``) and
agriculture machinery (``agriculture_machinery_oil_share``).
* It is now possible to merge residential and services heat buses to reduce the
problem size (see ``cluster_heat_nodes``).
* Added option to tweak (almost) any configuration parameter through the
``{sector_opts}`` wildcard. The regional_co2_sequestration_potential is
triggered by the prefix ``CF+`` after which it is possible to pipe to any
setting that does not contain underscores (``_``). Example:
``CF+sector+v2g+false`` disables vehicle-to-grid flexibility.
* Option ``retrieve_sector_databundle`` to automatically retrieve and extract
data bundle.
* Removed the need to clone ``technology-data`` repository in a parallel
directory. The new approach automatically retrieves the technology data from
remote in the rule ``retrieve_cost_data``.
* Improved network plots including better legends, hydrogen retrofitting network
display, and change to EqualEarth projection. A new color scheme for
technologies was also introduced.
* Add two new rules ``build_transport_demand`` and
``build_population_weighted_energy_totals`` using code previously contained in
``prepare_sector_network``.
* Rules that convert weather data with ``atlite`` now largely run separately for
categories residential, rural and total.
* Units are assigned to the buses. These only provide a better understanding.
The specifications of the units are not taken into account in the
optimisation, which means that no automatic conversion of units takes place.
* Configuration file and wildcards are now stored under ``n.meta`` in every
PyPSA network.
* Updated `data bundle
<https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz>`_
that includes the hydrogan salt cavern storage potentials.
* Updated and extended documentation in
<https://pypsa-eur-sec.readthedocs.io/en/latest/>
* Added new rule ``copy_conda_env`` that exports a list of packages with which
the workflow was executed.
* Add basic continuous integration using Github Actions.
* Add basic ``rsync`` setup.
**Bugfixes**
* The CO2 sequestration limit implemented as GlobalConstraint (introduced in the previous version)
caused a failure to read in the shadow prices of other global constraints.
* The CO2 sequestration limit implemented as GlobalConstraint (introduced in the
previous version) caused a failure to read in the shadow prices of other
global constraints.
* Correct capital cost of Fischer-Tropsch according to new units in ``technology-data``.
* Correct capital cost of Fischer-Tropsch according to new units in
``technology-data`` repository.
* Fix unit conversion error for thermal energy storage.
* For myopic pathway optimisation, set optimised capacities of power grid
expansion of previous iteration as minimum capacity for next iteration.
* Further rather minor bugfixes for myopic optimisation code (see `#256
<https://github.com/PyPSA/pypsa-eur-sec/pull/256>`_).
Many thanks to all who contributed to this release!
PyPSA-Eur-Sec 0.6.0 (4 October 2021)

BIN
graphics/workflow.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 324 KiB

View File

@ -178,7 +178,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
busmap = pd.read_csv(snakemake.input.busmap, index_col=0).squeeze()
inv_busmap = {}
for k, v in busmap.iteritems():
for k, v in busmap.items():
inv_busmap[v] = inv_busmap.get(v, []) + [k]
clustermaps = busmap_s.map(busmap)

View File

@ -1,5 +1,6 @@
from functools import partial
from tqdm import tqdm
from helper import mute
import multiprocessing as mp
import pandas as pd
@ -8,6 +9,8 @@ import numpy as np
idx = pd.IndexSlice
mute()
def cartesian(s1, s2):
"""Cartesian product of two pd.Series"""
return pd.DataFrame(np.outer(s1, s2), index=s1.index, columns=s2.index)
@ -126,7 +129,6 @@ to_ipcc = {
"total woL": "Total (without LULUCF)",
}
def build_eurostat(input_eurostat, countries, report_year, year):
"""Return multi-index for all countries' energy data in TWh/a."""
@ -380,7 +382,7 @@ def build_idees(countries, year):
func = partial(idees_per_country, year=year)
tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries),
desc='Build from IDEES database')
with mp.Pool(processes=nprocesses) as pool:
with mp.Pool(processes=nprocesses, initializer=mute) as pool:
totals_list = list(tqdm(pool.imap(func, countries), **tqdm_kwargs))

View File

@ -4,7 +4,7 @@ import pandas as pd
import numpy as np
import multiprocessing as mp
from tqdm import tqdm
from helper import mute
tj_to_ktoe = 0.0238845
ktoe_to_twh = 0.01163
@ -99,7 +99,6 @@ e_switzerland = pd.Series({'Iron and steel': 7889.,
'Other Industrial Sectors': 10825.,
'current electricity': 53760.})
def find_physical_output(df):
start = np.where(df.index.str.contains('Physical output', na=''))[0][0]
empty_row = np.where(df.index.isnull())[0]
@ -169,7 +168,7 @@ def industry_production(countries):
func = industry_production_per_country
tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries),
desc="Build industry production")
with mp.Pool(processes=nprocesses) as pool:
with mp.Pool(processes=nprocesses, initializer=mute) as pool:
demand_l = list(tqdm(pool.imap(func, countries), **tqdm_kwargs))
demand = pd.concat(demand_l, axis=1).T

View File

@ -1,6 +1,9 @@
"""Build industry sector ratios."""
import pandas as pd
from helper import mute
mute()
# GWh/ktoe OR MWh/toe
toe_to_MWh = 11.630

View File

@ -40,7 +40,7 @@ if __name__ == '__main__':
reference = ["RS", "BA"]
average = urban_fraction[reference].mean()
fill_values = pd.Series({ct: average for ct in missing})
urban_fraction = urban_fraction.append(fill_values)
urban_fraction = pd.concat([urban_fraction, fill_values])
# population in each grid cell
pop_cells = pd.Series(I.dot(nuts3['pop']))

View File

@ -28,8 +28,8 @@ def build_transport_demand(traffic_fn, airtemp_fn, nodes, nodal_transport_data):
## Get overall demand curve for all vehicles
traffic = pd.read_csv(
traffic_fn, skiprows=2, usecols=["count"], squeeze=True
)
traffic_fn, skiprows=2, usecols=["count"]
).squeeze("columns")
transport_shape = generate_periodic_profiles(
dt_index=snapshots,
@ -118,7 +118,7 @@ def bev_availability_profile(fn, snapshots, nodes, options):
Derive plugged-in availability for passenger electric vehicles.
"""
traffic = pd.read_csv(fn, skiprows=2, usecols=["count"], squeeze=True)
traffic = pd.read_csv(fn, skiprows=2, usecols=["count"]).squeeze("columns")
avail_max = options["bev_avail_max"]
avail_mean = options["bev_avail_mean"]

View File

@ -1,4 +1,5 @@
import os
import sys
import yaml
import pytz
import pandas as pd
@ -10,6 +11,10 @@ from pypsa.components import components, component_attrs
import logging
logger = logging.getLogger(__name__)
def mute():
"""hide irrelevant outputs of subprocess in multiprocessing pools.
also hide irrelevant outputs caused by pd.read_excel"""
sys.stdout = open(os.devnull, 'w')
def override_component_attrs(directory):
"""Tell PyPSA that links can have multiple outputs by

View File

@ -527,6 +527,8 @@ def add_co2_tracking(n, options):
e_nom_max = pd.read_csv(snakemake.input.sequestration_potential, index_col=0).squeeze()
e_nom_max = e_nom_max.reindex(spatial.co2.locations).fillna(0.).clip(upper=upper_limit).mul(1e6) / annualiser # t
e_nom_max = e_nom_max.rename(index=lambda x: x + " co2 stored")
else:
e_nom_max = np.inf
n.madd("Store",
spatial.co2.nodes,
@ -759,8 +761,8 @@ def add_ammonia(n, costs):
carrier="Haber-Bosch",
efficiency=1 / (cf_industry["MWh_elec_per_tNH3_electrolysis"] / cf_industry["MWh_NH3_per_tNH3"]), # output: MW_NH3 per MW_elec
efficiency2=-cf_industry["MWh_H2_per_tNH3_electrolysis"] / cf_industry["MWh_elec_per_tNH3_electrolysis"], # input: MW_H2 per MW_elec
capital_cost=costs.at["Haber-Bosch synthesis", "fixed"],
lifetime=costs.at["Haber-Bosch synthesis", 'lifetime']
capital_cost=costs.at["Haber-Bosch", "fixed"],
lifetime=costs.at["Haber-Bosch", 'lifetime']
)
n.madd("Link",
@ -2905,6 +2907,9 @@ if __name__ == "__main__":
if "B" in opts:
add_biomass(n, costs)
if options['ammonia']:
add_ammonia(n, costs)
if "I" in opts:
add_industry(n, costs)
@ -2917,9 +2922,6 @@ if __name__ == "__main__":
if options['dac']:
add_dac(n, costs)
if options['ammonia']:
add_ammonia(n, costs)
if "decentral" in opts:
decentral(n)