diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 76dcf552..0b3c2d4f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: : 2021 The PyPSA-Eur Authors +# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors # # SPDX-License-Identifier: CC0-1.0 diff --git a/LICENSE.txt b/LICENSE.txt index dc10fd32..87f6d959 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ MIT License -Copyright 2017-2021 The PyPSA-Eur Authors +Copyright 2017-2023 The PyPSA-Eur Authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/config.default.yaml b/config.default.yaml index ed1035df..a23cdc79 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -1,4 +1,4 @@ -version: 0.6.0 +version: 0.7.0 logging_level: INFO @@ -538,6 +538,7 @@ plotting: natural gas: '#e05b09' CCGT: '#a85522' CCGT marginal: '#a85522' + allam: '#B98F76' gas for industry co2 to atmosphere: '#692e0a' gas for industry co2 to stored: '#8a3400' gas for industry: '#853403' diff --git a/doc/conf.py b/doc/conf.py index de9908be..85ceb66e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -63,17 +63,17 @@ master_doc = 'index' # General information about the project. project = u'PyPSA-Eur-Sec' -copyright = u'2019-2021 Tom Brown (KIT, TUB), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB), Fabian Neumann (TUB)' -author = u'2019-2021 Tom Brown (KIT, TUB), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB), Fabian Neumann (TUB)' +copyright = u'2019-2023 Tom Brown (KIT, TUB), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB), Fabian Neumann (TUB)' +author = u'2019-2023 Tom Brown (KIT, TUB), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB), Fabian Neumann (TUB)' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = u'0.6' +version = u'0.7' # The full version, including alpha/beta/rc tags. -release = u'0.6.0' +release = u'0.7.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/data.csv b/doc/data.csv index 49fbc536..012e5be0 100644 --- a/doc/data.csv +++ b/doc/data.csv @@ -16,6 +16,7 @@ existing heating potentials,existing_infrastructure/existing_heating_raw.csv,unk IRENA existing VRE capacities,existing_infrastructure/{solar|onwind|offwind}_capcity_IRENA.csv,unknown,https://www.irena.org/Statistics/Download-Data USGS ammonia production,myb1-2017-nitro.xls,unknown,https://www.usgs.gov/centers/nmic/nitrogen-statistics-and-information hydrogen salt cavern potentials,h2_salt_caverns_GWh_per_sqkm.geojson,CC BY 4.0,https://doi.org/10.1016/j.ijhydene.2019.12.161 https://doi.org/10.20944/preprints201910.0187.v1 +international port trade volumes,attributed_ports.json,CC BY 4.0,https://datacatalog.worldbank.org/search/dataset/0038118/Global---International-Ports hotmaps industrial site database,Industrial_Database.csv,CC BY 4.0,https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database Hotmaps building stock data,data_building_stock.csv,CC BY 4.0,https://gitlab.com/hotmaps/building-stock U-values Poland,u_values_poland.csv,unknown,https://data.europa.eu/euodp/de/data/dataset/building-stock-observatory diff --git a/doc/index.rst b/doc/index.rst index d366f5b7..e6fdd415 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -14,10 +14,6 @@ PyPSA-Eur-Sec: A Sector-Coupled Open Optimisation Model of the European Energy S .. image:: https://img.shields.io/github/repo-size/pypsa/pypsa-eur-sec :alt: GitHub repo size -.. image:: https://badges.gitter.im/PyPSA/community.svg - :target: https://gitter.im/PyPSA/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge - :alt: Chat on Gitter - PyPSA-Eur-Sec is an open model dataset of the European energy system at the transmission network level that covers the full ENTSO-E area. @@ -101,6 +97,15 @@ Ministry for Education and Research (BMBF) `_ as part of the `Stromnetze Research Initiative `_. +Workflow Outline +================ + +.. image:: ../graphics/workflow.png + +.. note:: + The graph above was generated using + ``snakemake --rulegraph -F | sed -n "/digraph/,/}/p" | dot -Tpng -o workflow.png`` + Documentation ============= diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 4700f27b..f80d78f0 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -8,31 +8,49 @@ Future release .. note:: This unreleased version currently may require the master branches of PyPSA, PyPSA-Eur, and the technology-data repository. -This release includes the addition of the European gas transmission network and -incorporates retrofitting options to hydrogen. +* new feature + + +PyPSA-Eur-Sec 0.7.0 (16th February 2023) +======================================== + +This release includes many new features. Highlights include new gas +infrastructure data with retrofitting options for hydrogen transport, improved +carbon management and infrastructure planning, regionalised potentials for +hydrogen underground storage and carbon sequestration, new applications for +biomass, and explicit modelling of methanol and ammonia as separate energy +carriers. + +This release is known to work with `PyPSA-Eur +`_ Version 0.7.0 and `Technology Data +`_ Version 0.5.0. **Gas Transmission Network** * New rule ``retrieve_gas_infrastructure_data`` that downloads and extracts the - SciGRID_gas `IGGIELGN `_ dataset from zenodo. - It includes data on the transmission routes, pipe diameters, capacities, pressure, - and whether the pipeline is bidirectional and carries H-Gas or L-Gas. + SciGRID_gas `IGGIELGN `_ dataset from + zenodo. It includes data on the transmission routes, pipe diameters, + capacities, pressure, and whether the pipeline is bidirectional and carries + H-Gas or L-Gas. -* New rule ``build_gas_network`` processes and cleans the pipeline data from SciGRID_gas. - Missing or uncertain pipeline capacities can be inferred by diameter. +* New rule ``build_gas_network`` processes and cleans the pipeline data from + SciGRID_gas. Missing or uncertain pipeline capacities can be inferred by + diameter. * New rule ``build_gas_input_locations`` compiles the LNG import capacities - (including planned projects from gem.wiki), pipeline entry capacities and - local production capacities for each region of the model. These are the - regions where fossil gas can eventually enter the model. + (from the Global Energy Monitor's `Europe Gas Tracker + `_, pipeline + entry capacities and local production capacities for each region of the model. + These are the regions where fossil gas can eventually enter the model. * New rule ``cluster_gas_network`` that clusters the gas transmission network - data to the model resolution. Cross-regional pipeline capacities are aggregated - (while pressure and diameter compatibility is ignored), intra-regional pipelines - are dropped. Lengths are recalculated based on the regions' centroids. + data to the model resolution. Cross-regional pipeline capacities are + aggregated (while pressure and diameter compatibility is ignored), + intra-regional pipelines are dropped. Lengths are recalculated based on the + regions' centroids. -* With the option ``sector: gas_network:``, the existing gas network is - added with a lossless transport model. A length-weighted `k-edge augmentation +* With the option ``sector: gas_network:``, the existing gas network is added + with a lossless transport model. A length-weighted `k-edge augmentation algorithm `_ can be run to add new candidate gas pipelines such that all regions of the @@ -41,69 +59,27 @@ incorporates retrofitting options to hydrogen. the gas network is activated, all the gas demands are regionally disaggregated as well. -* New constraint allows endogenous retrofitting of gas pipelines to hydrogen pipelines. - This option is activated via the setting ``sector: H2_retrofit:``. For every - unit of gas pipeline capacity dismantled, ``sector: +* New constraint allows endogenous retrofitting of gas pipelines to hydrogen + pipelines. This option is activated via the setting ``sector: H2_retrofit:``. + For every unit of gas pipeline capacity dismantled, ``sector: H2_retrofit_capacity_per_CH4`` units are made available as hydrogen pipeline capacity in the corresponding corridor. These repurposed hydrogen pipelines - have lower costs than new hydrogen pipelines. Both new and repurposed pipelines - can be built simultaneously. The retrofitting option ``sector: H2_retrofit:`` also works - with a copperplated methane infrastructure, i.e. when ``sector: gas_network: false``. + have lower costs than new hydrogen pipelines. Both new and repurposed + pipelines can be built simultaneously. The retrofitting option ``sector: + H2_retrofit:`` also works with a copperplated methane infrastructure, i.e. + when ``sector: gas_network: false``. * New hydrogen pipelines can now be built where there are already power or gas transmission routes. Previously, only the electricity transmission routes were considered. -**New features and functionality** - - -* Add option to aggregate network temporally using representative snapshots or segments (with tsam package) - -* Add option for biomass boilers (wood pellets) for decentral heating - -* Add option for BioSNG (methane from biomass) with and without CC - -* Add option for BtL (Biomass to liquid fuel/oil) with and without CC - -* Add option for minimum part load for Fischer-Tropsch plants (default: 90%) and methanolisation plants (default: 50%). - -* Units are assigned to the buses. These only provide a better understanding. The specifications of the units are not taken into account in the optimisation, which means that no automatic conversion of units takes place. - -* Option ``retrieve_sector_databundle`` to automatically retrieve and extract data bundle. - -* Add option to use waste heat of electrolysis in district heating networks (``use_electrolysis_waste_heat``). - -* Add regionalised hydrogen salt cavern storage potentials from `Technical Potential of Salt Caverns for Hydrogen Storage in Europe `_. - -* Add option to sweep the global CO2 sequestration potentials with keyword ``seq200`` in the ``{sector_opts}`` wildcard (for limit of 200 Mt CO2). - -* Add option to resolve ammonia as separate energy carrier with Haber-Bosch - synthesis, ammonia cracking, storage and industrial demand. The ammonia - carrier can be nodally resolved or copperplated across Europe. This feature is - controlled by ``sector: ammonia:``. - -* Add methanol as energy carrier, methanolisation as process, and option for methanol demand in shipping sector. - -* Updated `data bundle `_ that includes the hydrogan salt cavern storage potentials. - -* Updated and extended documentation in - -* Shipping demand now defaults to (synthetic) oil rather than liquefied hydrogen until 2050. - -* Improved network plots including better legends, hydrogen retrofitting network display, and change to EqualEarth projection. - -* New config options for changing energy demands in aviation - (``aviation_demand_factor``) and HVC industry (``HVC_demand_factor``), as well - as explicit ICE shares for land transport (``land_transport_ice_share``) and - agriculture machinery (``agriculture_machinery_oil_share``). +**Carbon Management and Biomass** * Add option to spatially resolve carrier representing stored carbon dioxide (``co2_spatial``). This allows for more detailed modelling of CCUTS, e.g. regarding the capturing of industrial process emissions, usage as feedstock - for electrofuels, transport of carbon dioxide, and geological sequestration sites. - -* Add option for planning a new carbon dioxide network (``co2network``). - + for electrofuels, transport of carbon dioxide, and geological sequestration + sites. * Add option for regionally-resolved geological carbon dioxide sequestration potentials through new rule ``build_sequestration_potentials`` based on @@ -115,18 +91,135 @@ incorporates retrofitting options to hydrogen. potential. The defaults are preliminary and will be validated the next release. -* Separate option to regionally resolve biomass (``biomass_spatial``) from - option to allow biomass transport (``biomass_transport``). +* Add option to sweep the global CO2 sequestration potentials with keyword + ``seq200`` in the ``{sector_opts}`` wildcard (for limit of 200 Mt CO2). * Add option to include `Allam cycle gas power plants `_ (``allam_cycle``). +* Add option for planning a new carbon dioxide network (``co2network``). + +* Separate option to regionally resolve biomass (``biomass_spatial``) from + option to allow biomass transport (``biomass_transport``). + +* Add option for biomass boilers (wood pellets) for decentral heating. + +* Add option for BioSNG (methane from biomass) with and without carbon capture. + +* Add option for BtL (biomass to liquid fuel/oil) with and without carbon + capture. + + +**Other new features** + +* Add regionalised hydrogen salt cavern storage potentials from `Technical + Potential of Salt Caverns for Hydrogen Storage in Europe + `_. This data is compiled in + a new rule ``build_salt_cavern_potentials``. + +* Add option to resolve ammonia as separate energy carrier with Haber-Bosch + synthesis, ammonia cracking, storage and industrial demand. The ammonia + carrier can be nodally resolved or copperplated across Europe (see + ``ammonia``). + +* Add methanol as energy carrier, methanolisation as process, and option for + methanol demand in shipping sector. + +* Shipping demand now defaults to methanol rather than liquefied hydrogen + until 2050. + +* Demand for liquid hydrogen in international shipping is now geographically + distributed by port trade volumes in a new rule ``build_shipping_demand`` + using data from the `World Bank Data Catalogue + `_. + Domestic shipping remains distributed by population. + +* Add option to aggregate network temporally using representative snapshots or + segments (with `tsam `_). + +* Add option for minimum part load for Fischer-Tropsch plants (default: 90%) and + methanolisation plants (default: 50%). + +* Add option to use waste heat of electrolysis in district heating networks + (``use_electrolysis_waste_heat``). + +* Add option for coal CHPs with carbon capture (see ``coal_cc``). + +* In overnight optimisation, it is now possible to specify a year for the + technology cost projections separate from the planning horizon. + +* New config options for changing energy demands in aviation + (``aviation_demand_factor``) and HVC industry (``HVC_demand_factor``), as well + as explicit ICE shares for land transport (``land_transport_ice_share``) and + agriculture machinery (``agriculture_machinery_oil_share``). + +* It is now possible to merge residential and services heat buses to reduce the + problem size (see ``cluster_heat_nodes``). + +* Added option to tweak (almost) any configuration parameter through the + ``{sector_opts}`` wildcard. The regional_co2_sequestration_potential is + triggered by the prefix ``CF+`` after which it is possible to pipe to any + setting that does not contain underscores (``_``). Example: + ``CF+sector+v2g+false`` disables vehicle-to-grid flexibility. + +* Option ``retrieve_sector_databundle`` to automatically retrieve and extract + data bundle. + +* Removed the need to clone ``technology-data`` repository in a parallel + directory. The new approach automatically retrieves the technology data from + remote in the rule ``retrieve_cost_data``. + +* Improved network plots including better legends, hydrogen retrofitting network + display, and change to EqualEarth projection. A new color scheme for + technologies was also introduced. + +* Add two new rules ``build_transport_demand`` and + ``build_population_weighted_energy_totals`` using code previously contained in + ``prepare_sector_network``. + +* Rules that convert weather data with ``atlite`` now largely run separately for + categories residential, rural and total. + +* Units are assigned to the buses. These only provide a better understanding. + The specifications of the units are not taken into account in the + optimisation, which means that no automatic conversion of units takes place. + +* Configuration file and wildcards are now stored under ``n.meta`` in every + PyPSA network. + +* Updated `data bundle + `_ + that includes the hydrogan salt cavern storage potentials. + +* Updated and extended documentation in + + +* Added new rule ``copy_conda_env`` that exports a list of packages with which + the workflow was executed. + +* Add basic continuous integration using Github Actions. + +* Add basic ``rsync`` setup. + **Bugfixes** -* The CO2 sequestration limit implemented as GlobalConstraint (introduced in the previous version) - caused a failure to read in the shadow prices of other global constraints. +* The CO2 sequestration limit implemented as GlobalConstraint (introduced in the + previous version) caused a failure to read in the shadow prices of other + global constraints. -* Correct capital cost of Fischer-Tropsch according to new units in ``technology-data``. +* Correct capital cost of Fischer-Tropsch according to new units in + ``technology-data`` repository. + +* Fix unit conversion error for thermal energy storage. + +* For myopic pathway optimisation, set optimised capacities of power grid + expansion of previous iteration as minimum capacity for next iteration. + +* Further rather minor bugfixes for myopic optimisation code (see `#256 + `_). + + +Many thanks to all who contributed to this release! PyPSA-Eur-Sec 0.6.0 (4 October 2021) diff --git a/graphics/workflow.png b/graphics/workflow.png new file mode 100644 index 00000000..ca3e4e29 Binary files /dev/null and b/graphics/workflow.png differ diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 8e274d62..74b26211 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -178,7 +178,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas busmap = pd.read_csv(snakemake.input.busmap, index_col=0).squeeze() inv_busmap = {} - for k, v in busmap.iteritems(): + for k, v in busmap.items(): inv_busmap[v] = inv_busmap.get(v, []) + [k] clustermaps = busmap_s.map(busmap) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 575070ad..38a9895c 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -1,5 +1,6 @@ from functools import partial from tqdm import tqdm +from helper import mute import multiprocessing as mp import pandas as pd @@ -8,6 +9,8 @@ import numpy as np idx = pd.IndexSlice +mute() + def cartesian(s1, s2): """Cartesian product of two pd.Series""" return pd.DataFrame(np.outer(s1, s2), index=s1.index, columns=s2.index) @@ -126,7 +129,6 @@ to_ipcc = { "total woL": "Total (without LULUCF)", } - def build_eurostat(input_eurostat, countries, report_year, year): """Return multi-index for all countries' energy data in TWh/a.""" @@ -380,7 +382,7 @@ def build_idees(countries, year): func = partial(idees_per_country, year=year) tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries), desc='Build from IDEES database') - with mp.Pool(processes=nprocesses) as pool: + with mp.Pool(processes=nprocesses, initializer=mute) as pool: totals_list = list(tqdm(pool.imap(func, countries), **tqdm_kwargs)) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index f21b711a..c88919ab 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -4,7 +4,7 @@ import pandas as pd import numpy as np import multiprocessing as mp from tqdm import tqdm - +from helper import mute tj_to_ktoe = 0.0238845 ktoe_to_twh = 0.01163 @@ -99,7 +99,6 @@ e_switzerland = pd.Series({'Iron and steel': 7889., 'Other Industrial Sectors': 10825., 'current electricity': 53760.}) - def find_physical_output(df): start = np.where(df.index.str.contains('Physical output', na=''))[0][0] empty_row = np.where(df.index.isnull())[0] @@ -169,7 +168,7 @@ def industry_production(countries): func = industry_production_per_country tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries), desc="Build industry production") - with mp.Pool(processes=nprocesses) as pool: + with mp.Pool(processes=nprocesses, initializer=mute) as pool: demand_l = list(tqdm(pool.imap(func, countries), **tqdm_kwargs)) demand = pd.concat(demand_l, axis=1).T diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index d2a9ca48..a8c00941 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -1,6 +1,9 @@ """Build industry sector ratios.""" import pandas as pd +from helper import mute + +mute() # GWh/ktoe OR MWh/toe toe_to_MWh = 11.630 diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index fa5c7d28..ba64e10f 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -40,7 +40,7 @@ if __name__ == '__main__': reference = ["RS", "BA"] average = urban_fraction[reference].mean() fill_values = pd.Series({ct: average for ct in missing}) - urban_fraction = urban_fraction.append(fill_values) + urban_fraction = pd.concat([urban_fraction, fill_values]) # population in each grid cell pop_cells = pd.Series(I.dot(nuts3['pop'])) diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index a5aabc31..0045b882 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -28,8 +28,8 @@ def build_transport_demand(traffic_fn, airtemp_fn, nodes, nodal_transport_data): ## Get overall demand curve for all vehicles traffic = pd.read_csv( - traffic_fn, skiprows=2, usecols=["count"], squeeze=True - ) + traffic_fn, skiprows=2, usecols=["count"] + ).squeeze("columns") transport_shape = generate_periodic_profiles( dt_index=snapshots, @@ -118,7 +118,7 @@ def bev_availability_profile(fn, snapshots, nodes, options): Derive plugged-in availability for passenger electric vehicles. """ - traffic = pd.read_csv(fn, skiprows=2, usecols=["count"], squeeze=True) + traffic = pd.read_csv(fn, skiprows=2, usecols=["count"]).squeeze("columns") avail_max = options["bev_avail_max"] avail_mean = options["bev_avail_mean"] diff --git a/scripts/helper.py b/scripts/helper.py index 62ae33c0..b099061c 100644 --- a/scripts/helper.py +++ b/scripts/helper.py @@ -1,4 +1,5 @@ import os +import sys import yaml import pytz import pandas as pd @@ -10,6 +11,10 @@ from pypsa.components import components, component_attrs import logging logger = logging.getLogger(__name__) +def mute(): + """hide irrelevant outputs of subprocess in multiprocessing pools. + also hide irrelevant outputs caused by pd.read_excel""" + sys.stdout = open(os.devnull, 'w') def override_component_attrs(directory): """Tell PyPSA that links can have multiple outputs by diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index d369df16..d0264c7d 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -527,6 +527,8 @@ def add_co2_tracking(n, options): e_nom_max = pd.read_csv(snakemake.input.sequestration_potential, index_col=0).squeeze() e_nom_max = e_nom_max.reindex(spatial.co2.locations).fillna(0.).clip(upper=upper_limit).mul(1e6) / annualiser # t e_nom_max = e_nom_max.rename(index=lambda x: x + " co2 stored") + else: + e_nom_max = np.inf n.madd("Store", spatial.co2.nodes, @@ -759,8 +761,8 @@ def add_ammonia(n, costs): carrier="Haber-Bosch", efficiency=1 / (cf_industry["MWh_elec_per_tNH3_electrolysis"] / cf_industry["MWh_NH3_per_tNH3"]), # output: MW_NH3 per MW_elec efficiency2=-cf_industry["MWh_H2_per_tNH3_electrolysis"] / cf_industry["MWh_elec_per_tNH3_electrolysis"], # input: MW_H2 per MW_elec - capital_cost=costs.at["Haber-Bosch synthesis", "fixed"], - lifetime=costs.at["Haber-Bosch synthesis", 'lifetime'] + capital_cost=costs.at["Haber-Bosch", "fixed"], + lifetime=costs.at["Haber-Bosch", 'lifetime'] ) n.madd("Link", @@ -2905,6 +2907,9 @@ if __name__ == "__main__": if "B" in opts: add_biomass(n, costs) + if options['ammonia']: + add_ammonia(n, costs) + if "I" in opts: add_industry(n, costs) @@ -2917,9 +2922,6 @@ if __name__ == "__main__": if options['dac']: add_dac(n, costs) - if options['ammonia']: - add_ammonia(n, costs) - if "decentral" in opts: decentral(n)