diff --git a/.travis.yml b/.travis.yml index b7b8c4d9..43b25200 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ before_install: - conda activate pypsa-eur # install open-source solver - - mamba install -c conda-forge glpk ipopt + - mamba install -c conda-forge glpk ipopt'<3.13.3' # list packages for easier debugging - conda list diff --git a/README.md b/README.md index c281e589..dc6b4791 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ SPDX-License-Identifier: CC-BY-4.0 [![Build Status](https://travis-ci.org/PyPSA/pypsa-eur.svg?branch=master)](https://travis-ci.org/PyPSA/pypsa-eur) [![Documentation](https://readthedocs.org/projects/pypsa-eur/badge/?version=latest)](https://pypsa-eur.readthedocs.io/en/latest/?badge=latest) ![Size](https://img.shields.io/github/repo-size/pypsa/pypsa-eur) -[![Zenodo](https://zenodo.org/badge/DOI/10.5281/zenodo.3520875.svg)](https://doi.org/10.5281/zenodo.3520875) +[![Zenodo](https://zenodo.org/badge/DOI/10.5281/zenodo.3520874.svg)](https://doi.org/10.5281/zenodo.3520874) [![Gitter](https://badges.gitter.im/PyPSA/community.svg)](https://gitter.im/PyPSA/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Snakemake](https://img.shields.io/badge/snakemake-≥5.0.0-brightgreen.svg?style=flat)](https://snakemake.readthedocs.io) [![REUSE status](https://api.reuse.software/badge/github.com/pypsa/pypsa-eur)](https://api.reuse.software/info/github.com/pypsa/pypsa-eur) @@ -42,7 +42,7 @@ discussion in Section 3.4 "Model validation" of the paper. ![PyPSA-Eur Grid Model Simplified](doc/img/elec_s_X.png) -The model is designed to be imported into the open toolbox +The model building routines are defined through a snakemake workflow. The model is designed to be imported into the open toolbox [PyPSA](https://github.com/PyPSA/PyPSA) for operational studies as well as generation and transmission expansion planning studies. @@ -61,7 +61,7 @@ The dataset consists of: - Geographical potentials for wind and solar generators based on land use (CORINE) and excluding nature reserves (Natura2000) are computed with the [vresutils library](https://github.com/FRESNA/vresutils) and the [glaes library](https://github.com/FZJ-IEK3-VSA/glaes). Already-built versions of the model can be found in the accompanying [Zenodo -repository](https://doi.org/10.5281/zenodo.3601882). +repository](https://doi.org/10.5281/zenodo.3601881). A version of the model that adds building heating, transport and industry sectors to the model, as well as gas networks, can be found diff --git a/Snakefile b/Snakefile index b8bc284a..c2420cd4 100644 --- a/Snakefile +++ b/Snakefile @@ -51,8 +51,8 @@ datafiles = ['ch_cantons.csv', 'je-e-21.03.02.xls', 'eez/World_EEZ_v8_2014.shp', 'EIA_hydro_generation_2000_2014.csv', 'hydro_capacities.csv', 'naturalearth/ne_10m_admin_0_countries.shp', 'NUTS_2013_60M_SH/data/NUTS_RG_60M_2013.shp', 'nama_10r_3popgdp.tsv.gz', - 'nama_10r_3gdp.tsv.gz', 'time_series_60min_singleindex_filtered.csv', - 'corine/g250_clc06_V18_5.tif'] + 'nama_10r_3gdp.tsv.gz', 'corine/g250_clc06_V18_5.tif'] + if not config.get('tutorial', False): @@ -66,6 +66,12 @@ if config['enable'].get('retrieve_databundle', True): script: 'scripts/retrieve_databundle.py' +rule build_load_data: + output: "resources/load.csv" + log: "logs/build_load_data.log" + script: 'scripts/build_load_data.py' + + rule build_powerplants: input: base_network="networks/base.nc", @@ -211,7 +217,7 @@ rule add_electricity: powerplants='resources/powerplants.csv', hydro_capacities='data/bundle/hydro_capacities.csv', geth_hydro_capacities='data/geth2015_hydro_capacities.csv', - opsd_load='data/bundle/time_series_60min_singleindex_filtered.csv', + load='resources/load.csv', nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": "resources/profile{year}_" + f"{tech}.nc" for tech in config['renewable']} @@ -290,6 +296,11 @@ def memory(w): if m is not None: factor /= int(m.group(1)) break + for o in w.opts.split('-'): + m = re.match(r'^(\d+)seg$', o, re.IGNORECASE) + if m is not None: + factor *= int(m.group(1)) / 8760 + break if w.clusters.endswith('m'): return int(factor * (18000 + 180 * int(w.clusters[:-1]))) else: diff --git a/config.default.yaml b/config.default.yaml index 9f096fff..68fdb4c7 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: CC0-1.0 -version: 0.2.0 +version: 0.3.0 tutorial: false logging: @@ -32,17 +32,18 @@ enable: retrieve_cutout: true build_natura_raster: false retrieve_natura_raster: true + custom_busmap: false electricity: voltages: [220., 300., 380.] co2limit: 7.75e+7 # 0.05 * 3.1e9*0.5 - co2base: 3.1e+9 # 1 * 3.1e9*0.5 + co2base: 1.487e9 agg_p_nom_limits: data/agg_p_nom_minmax.csv extendable_carriers: Generator: [] - StorageUnit: [battery, H2] - Store: [] # battery, H2 + StorageUnit: [] # battery, H2 + Store: [battery, H2] Link: [] max_hours: @@ -52,6 +53,7 @@ electricity: powerplants_filter: false # use pandas query strings here, e.g. Country not in ['Germany'] custom_powerplants: false # use pandas query strings here, e.g. Country in ['Germany'] conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] + renewable_capacities_from_OPSD: [] # onwind, offwind, solar # estimate_renewable_capacities_from_capacity_stats: # # Wind is the Fueltype in ppm.data.Capacity_stats, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur @@ -142,8 +144,7 @@ renewable: cutout: europe-2013-era5 carriers: [ror, PHS, hydro] PHS_max_hours: 6 - hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, - # estimate_by_large_installations or a float + hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float clip_min_inflow: 1.0 lines: @@ -168,6 +169,11 @@ transformers: type: '' load: + url: https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv + power_statistics: True # only for files from <2019; set false in order to get ENTSOE transparency data + interpolate_limit: 3 # data gaps up until this size are interpolated linearly + time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from + manual_adjustments: true # false scaling_factor: 1.0 fallback_year: 2013 @@ -175,13 +181,16 @@ costs: year: 2030 discountrate: 0.07 # From a Lion Hirth paper, also reflects average of Noothout et al 2016 USD2013_to_EUR2013: 0.7532 # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html - marginal_cost: + marginal_cost: # EUR/MWh solar: 0.01 onwind: 0.015 offwind: 0.015 hydro: 0. H2: 0. + electrolysis: 0. + fuel cell: 0. battery: 0. + battery inverter: 0. emission_prices: # in currency per tonne emission, only used with the option Ep co2: 0. diff --git a/config.tutorial.yaml b/config.tutorial.yaml index e74a2967..c78a75fd 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -2,8 +2,9 @@ # # SPDX-License-Identifier: CC0-1.0 -version: 0.2.0 +version: 0.3.0 tutorial: true + logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' @@ -31,6 +32,7 @@ enable: retrieve_cutout: true build_natura_raster: false retrieve_natura_raster: true + custom_busmap: false electricity: voltages: [220., 300., 380.] @@ -38,8 +40,8 @@ electricity: extendable_carriers: Generator: [OCGT] - StorageUnit: [battery, H2] - Store: [] #battery, H2 + StorageUnit: [] #battery, H2 + Store: [battery, H2] Link: [] max_hours: @@ -146,6 +148,11 @@ transformers: type: '' load: + url: https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv + power_statistics: True # only for files from <2019; set false in order to get ENTSOE transparency data + interpolate_limit: 3 # data gaps up until this size are interpolated linearly + time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from + manual_adjustments: true # false scaling_factor: 1.0 fallback_year: 2013 diff --git a/data/costs.csv b/data/costs.csv index 330cc3bb..8953eb8a 100644 --- a/data/costs.csv +++ b/data/costs.csv @@ -114,7 +114,7 @@ DAC,2030,lifetime,30,years,Fasihi DAC,2030,FOM,4,%/year,Fasihi battery inverter,2030,investment,411,USD/kWel,budischak2013 battery inverter,2030,lifetime,20,years,budischak2013 -battery inverter,2030,efficiency,0.81,per unit,budischak2013; Lund and Kempton (2008) http://dx.doi.org/10.1016/j.enpol.2008.06.007 +battery inverter,2030,efficiency,0.9,per unit charge/discharge,budischak2013; Lund and Kempton (2008) http://dx.doi.org/10.1016/j.enpol.2008.06.007 battery inverter,2030,FOM,3,%/year,budischak2013 battery storage,2030,investment,192,USD/kWh,budischak2013 battery storage,2030,lifetime,15,years,budischak2013 diff --git a/data/links_tyndp.csv b/data/links_tyndp.csv index ca977c27..8079be72 100644 --- a/data/links_tyndp.csv +++ b/data/links_tyndp.csv @@ -6,8 +6,8 @@ Italy-Montenegro,Villanova (IT),Latsva (MT),445,,1200,under construction,Link.14 NordLink,Tonstad (NO),Wilster (DE),514,,1400,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/37,6.716948,58.662631,9.373979,53.922479 COBRA cable,Endrup (DK),Eemshaven (NL),325,,700,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/71,8.718392,55.523115,6.835494,53.438589 Thames Estuary Cluster (NEMO-Link),Richborough (GB),Gezelle (BE),140,,1000,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/74,1.324854,51.295891,3.23043,51.24902 -Anglo-Scottish -1,Hunterston (UK),Deeside (UK),422,,2400,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/77,-4.898329,55.723331,-3.032972,53.199735 -ALEGrO,Lixhe (BE),Oberzier (DE),100,,1000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/92,5.67933,50.7567965,6.474704,50.867532 +Anglo-Scottish -1,Hunterston (UK),Deeside (UK),422,,2400,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/77,-4.898329,55.723331,-3.032972,53.199735 +ALEGrO,Lixhe (BE),Oberzier (DE),100,,1000,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/92,5.67933,50.7567965,6.474704,50.867532 North Sea Link,Kvilldal (NO),Blythe (GB),720,,1400,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/110,6.637527,59.515096,-1.510277,55.126957 HVDC SuedOstLink,Wolmirstedt (DE),Isar (DE),,557,2000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/130,11.629014,52.252137,12.091596,48.080837 HVDC Line A-North,Emden East (DE),Osterath (DE),,284,2000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/132,7.206009,53.359403,6.619451,51.272935 diff --git a/data/parameter_corrections.yaml b/data/parameter_corrections.yaml index 0962b1b8..be8d2d42 100644 --- a/data/parameter_corrections.yaml +++ b/data/parameter_corrections.yaml @@ -33,12 +33,13 @@ Link: "14559": "6240" # fix wrong bus allocation from 6241 "12998": "1333" # combine link 12998 + 12997 in 12998 "5627": '2309' # combine link 5627 + 5628 in 5627 + "8068": "5819" # fix GB location of Anglo-Scottish interconnector length: index: "12998": 409.0 "5627": 26.39 bus0: index: - # set bus0 == bus1 for removing the link in remove_unconnected_components - "5628": "7276" - "12997": "7276" + "14552": "5819" # fix GB location of GB-IE interconnector + "5628": "7276" # bus0 == bus1 to remove link in remove_unconnected_components + "12997": "7276" # bus0 == bus1 to remove link in remove_unconnected_components diff --git a/doc/_static/theme_overrides.css b/doc/_static/theme_overrides.css index 778756c1..3480a661 100644 --- a/doc/_static/theme_overrides.css +++ b/doc/_static/theme_overrides.css @@ -2,22 +2,73 @@ SPDX-License-Identifier: GPL-3.0-or-later */ +.wy-side-nav-search { + background-color: #eeeeee; +} + +.wy-side-nav-search .wy-dropdown>a, +.wy-side-nav-search>a { + color: rgb(34, 97, 156) +} + +.wy-side-nav-search>div.version { + color: rgb(34, 97, 156) +} + +.wy-menu-vertical header, +.wy-menu-vertical p.caption, +.rst-versions a { + color: #999999; +} + +.wy-menu-vertical a.reference:hover, +.wy-menu-vertical a.reference.internal:hover { + background: #dddddd; + color: #fff; +} + +.wy-nav-side { + background: #efefef; +} + +.wy-menu-vertical a.reference { + color: #000; +} + +.rst-versions .rst-current-version, +.wy-nav-top, +.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a:hover { + background: #002221; +} + +.wy-nav-content .highlight { + background: #ffffff; +} + +.rst-content code.literal, +.rst-content tt.literal { + color: rgb(34, 97, 156) +} + +.wy-nav-content a.reference { + color: rgb(34, 97, 156); +} + + /* override table width restrictions */ + @media screen and (min-width: 767px) { - .wy-table-responsive table td { - /* !important prevents the common CSS stylesheets from overriding + /* !important prevents the common CSS stylesheets from overriding this as on RTD they are loaded after this stylesheet */ - white-space: normal !important; - /* background: #eeeeee !important; */ + white-space: normal !important; + background: rgb(250, 250, 250) !important; } - .wy-table-responsive { - max-width: 100%; - overflow: visible !important; + max-width: 100%; + overflow: visible !important; } - .wy-nav-content { max-width: 910px !important; } - } +} diff --git a/doc/cloudcomputing.rst b/doc/cloudcomputing.rst index 95bfa2c7..f20e1b23 100644 --- a/doc/cloudcomputing.rst +++ b/doc/cloudcomputing.rst @@ -60,7 +60,7 @@ Now a window with the machine details will open. You have to configure the follo You can edit your machine configuration later. So use a cheap machine type configuration to transfer data and only when everything is ready and tested, your expensive machine type, for instance a custom 8 vCPU with 160 GB memory. Solvers do not parallelise well, so we recommend not to choose more than 8 vCPU. - Check ``snakemake -j -n 1 solve_all_networks`` as a dry run to see how much memory is required. + Check ``snakemake -n -j 1 solve_all_networks`` as a dry run to see how much memory is required. The memory requirements will vary depending on the spatial and temporal resoulution of your optimisation. Example: for an hourly, 181 node full European network, set 8 vCPU and 150 GB memory since the dry-run calculated a 135 GB memory requirement.) - Boot disk: As default, your VM is created with 10 GB. Depending on how much you want to handle on one VM you should increase the disk size. @@ -85,7 +85,7 @@ Step 3 - Installation of Cloud SDK sudo apt-get update sudo apt-get install bzip2 libxml2-dev sudo apt-get install wget - wget https://repo.anaconda.com/archive/Anaconda3-2020.07-Linux-x86_64.sh (Check the link. To be up to date with anaconda, check the Anaconda website https://www.anaconda.com/products/individual ) + wget https://repo.anaconda.com/archive/Anaconda3-2020.07-Linux-x86_64.sh ls (to see what anaconda file to bash) bash Anaconda3-2020.07-Linux-x86_64.sh source ~/.bashrc diff --git a/doc/conf.py b/doc/conf.py index 899cbe1a..b2b220a0 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -74,9 +74,9 @@ author = u'Jonas Hoersch (KIT, FIAS), Fabian Hofmann (FIAS), David Schlachtberge # built documents. # # The short X.Y version. -version = u'0.2' +version = u'0.3' # The full version, including alpha/beta/rc tags. -release = u'0.2.0' +release = u'0.3.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index be447136..aaeab239 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -1,16 +1,19 @@ -,Unit,Values,Description -voltages,kV,"Any subset of {220., 300., 380.}","Voltage levels to consider when" -co2limit,:math:`t_{CO_2-eq}/a`,float,"Cap on total annual system carbon dioxide emissions" -co2base,:math:`t_{CO_2-eq}/a`,float,"Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard." -agg_p_nom_limits,--,file path,"Reference to ``.csv`` file specifying per carrier generator nominal capacity constraints for individual countries if ``'CCL'`` is in ``{opts}`` wildcard. Defaults to ``data/agg_p_nom_minmax.csv``." -extendable_carriers,,, --- Generator,--,"Any subset of {'OCGT','CCGT'}","Places extendable conventional power plants (OCGT and/or CCGT) where gas power plants are located today without capacity limits." --- StorageUnit,--,"Any subset of {'battery','H2'}","Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity." --- Store,--,"Any subset of {'battery','H2'}","Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity." --- Link,--,"Any subset of {'H2 pipeline'}","Adds extendable links (H2 pipelines only) at every connection where there are lines or HVDC links without capacity limits and with zero initial capacity. Hydrogen pipelines require hydrogen storage to be modelled as ``Store``." -max_hours,,, --- battery,h,float,"Maximum state of charge capacity of the battery in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_." --- H2,h,float,"Maximum state of charge capacity of the hydrogen storage in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_." -powerplants_filter,--,"use `pandas.query `_ strings here, e.g. Country not in ['Germany']","Filter query for the default powerplant database." -custom_powerplants,--,"use `pandas.query `_ strings here, e.g. Country in ['Germany']","Filter query for the custom powerplant database." -conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants.csv``." +,Unit,Values,Description, +voltages,kV,"Any subset of {220., 300., 380.}",Voltage levels to consider when, +co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide emissions, +co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard., +agg_p_nom_limits,file,path,Reference to ``.csv`` file specifying per carrier generator nominal capacity constraints for individual countries if ``'CCL'`` is in ``{opts}`` wildcard. Defaults to ``data/agg_p_nom_minmax.csv``. +extendable_carriers,,,, +-- Generator,--,"Any subset of {'OCGT','CCGT'}",Places extendable conventional power plants (OCGT and/or CCGT) where gas power plants are located today without capacity limits. +-- StorageUnit,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. +-- Store,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. +-- Link,--,Any subset of {'H2 pipeline'},Adds extendable links (H2 pipelines only) at every connection where there are lines or HVDC links without capacity limits and with zero initial capacity. Hydrogen pipelines require hydrogen storage to be modelled as ``Store``. +max_hours,,,, +-- battery,h,float,Maximum state of charge capacity of the battery in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. +-- H2,h,float,Maximum state of charge capacity of the hydrogen storage in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. +powerplants_filter,--,"use `pandas.query `_ strings here, e.g. Country not in ['Germany']",Filter query for the default powerplant database., +custom_powerplants,--,"use `pandas.query `_ strings here, e.g. Country in ['Germany']",Filter query for the custom powerplant database., +conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}",List of conventional power plants to include in the model from ``resources/powerplants.csv``., +renewable_capacities_from_OPSD,,"[solar, onwind, offwind]",List of carriers (offwind-ac and offwind-dc are included in offwind) whose capacities 'p_nom' are aligned to the `OPSD renewable power plant list `_, +estimate_renewable_capacities_from_capacitiy_stats,,,, +"-- Fueltype [ppm], e.g. Wind",,"list of fueltypes strings in PyPSA-Eur, e.g. [onwind, offwind-ac, offwind-dc]",converts ppm Fueltype to PyPSA-EUR Fueltype, diff --git a/doc/configtables/load.csv b/doc/configtables/load.csv index 760f912c..d6cfa686 100644 --- a/doc/configtables/load.csv +++ b/doc/configtables/load.csv @@ -1,2 +1,7 @@ ,Unit,Values,Description +url,--,string,"Link to open power system data time series data." +power_statistics,bool,"{true, false}",Whether to load the electricity consumption data of the ENTSOE power statistics (only for files from 2019 and before) or from the ENTSOE transparency data (only has load data from 2015 onwards). +interpolate_limit,hours,integer,"Maximum gap size (consecutive nans) which interpolated linearly." +time_shift_for_large_gaps,string,string,"Periods which are used for copying time-slices in order to fill large gaps of nans. Have to be valid ``pandas`` period strings." +manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`." scaling_factor,--,float,"Global correction factor for the load time series." diff --git a/doc/configtables/opts.csv b/doc/configtables/opts.csv index 88e87e44..da673ac8 100644 --- a/doc/configtables/opts.csv +++ b/doc/configtables/opts.csv @@ -1,5 +1,6 @@ Trigger, Description, Definition, Status ``nH``; i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() `_ and its `caller `_), In active use +``nSEG``; e.g. ``4380SEG``, "Apply time series segmentation with `tsam `_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use ``Co2L``, Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() `_ and its `caller `_, In active use ``Ep``, Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() `_ and its `caller `_, In active use ``CCL``, Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use @@ -7,4 +8,4 @@ Trigger, Description, Definition, Status ``ATK``, "Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links.", ``prepare_network``, In active use ``BAU``, Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() `_, Untested ``SAFE``, Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() `_, Untested -``carrier+factor``, "Alter the capital cost of a carrier by a factor. Example: ``solar+0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use +``carrier+{c|p}factor``, "Alter the capital cost (``c``) or installable potential (``p``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv index 27df2344..40715b7b 100644 --- a/doc/configtables/toplevel.csv +++ b/doc/configtables/toplevel.csv @@ -3,7 +3,7 @@ version,--,0.x.x,"Version of PyPSA-Eur" tutorial,bool,"{true, false}","Switch to retrieve the tutorial data set instead of the full data set." logging,,, -- level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only" --- format,--,"e.g. ``%(levelname)s:%(name)s:%(message)s``","Custom format for log messages. See `LogRecord `_ attributes." +-- format,--,"","Custom format for log messages. See `LogRecord `_ attributes." summary_dir,--,"e.g. 'results'","Directory into which results are written." countries,--,"Subset of {'AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'}","European countries defined by their `Two-letter country codes (ISO 3166-1) `_ which should be included in the energy system model." focus_weights,--,"Keys should be two-digit country codes (e.g. DE) and values should range between 0 and 1","Ratio of total clusters for particular countries. the remaining weight is distributed according to mean load. An example: ``focus_weights: DE: 0.6 FR: 0.2``." @@ -14,3 +14,4 @@ enable,,, -- retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts from zenodo with :mod:`retrieve_cutout`." -- build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`." -- retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`." +-- custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec{year}_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec{year}_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``." diff --git a/doc/configuration.rst b/doc/configuration.rst index 95d7af31..56798ee5 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -18,7 +18,7 @@ Top-level configuration .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 5-12,21,28-34 + :lines: 5-12,20,27-34 .. csv-table:: :header-rows: 1 @@ -50,7 +50,7 @@ An exemplary dependency graph (starting from the simplification rules) then look .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 14-19 + :lines: 14-18 .. csv-table:: :header-rows: 1 @@ -66,7 +66,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 23-26 + :lines: 22-25 .. csv-table:: :header-rows: 1 @@ -80,7 +80,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 36-54 + :lines: 36-60 .. csv-table:: :header-rows: 1 @@ -97,7 +97,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 61-74 + :lines: 62-75 .. csv-table:: :header-rows: 1 @@ -114,7 +114,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 76-93 + :lines: 77-94 .. csv-table:: :header-rows: 1 @@ -126,7 +126,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 76,94-106 + :lines: 77,95-107 .. csv-table:: :header-rows: 1 @@ -138,7 +138,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 76,107-120 + :lines: 77,108-121 .. csv-table:: :header-rows: 1 @@ -150,7 +150,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 76,121-140 + :lines: 77,122-141 .. csv-table:: :header-rows: 1 @@ -162,7 +162,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 76,141-147 + :lines: 77,142-147 .. csv-table:: :header-rows: 1 @@ -218,7 +218,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 170-171 + :lines: 170-176 .. csv-table:: :header-rows: 1 @@ -232,7 +232,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 173-185 + :lines: 178-190 .. csv-table:: :header-rows: 1 @@ -241,7 +241,6 @@ Specifies the temporal range to build an energy system model for as arguments to .. note:: To change cost assumptions in more detail (i.e. other than ``marginal_cost`` and ``capital_cost``), consider modifying cost assumptions directly in ``data/costs.csv`` as this is not yet supported through the config file. - You can also build multiple different cost databases. Make a renamed copy of ``data/costs.csv`` (e.g. ``data/costs-optimistic.csv``) and set the variable ``COSTS=data/costs-optimistic.csv`` in the ``Snakefile``. .. _solving_cf: @@ -254,7 +253,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 187-197 + :lines: 192-202 .. csv-table:: :header-rows: 1 @@ -266,7 +265,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 187,198-214 + :lines: 192,203-219 .. csv-table:: :header-rows: 1 @@ -280,7 +279,7 @@ Specifies the temporal range to build an energy system model for as arguments to .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 216-355 + :lines: 221-299 .. csv-table:: :header-rows: 1 diff --git a/doc/contributing.rst b/doc/contributing.rst index a6321989..d57f1212 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -12,7 +12,7 @@ be it with new ideas, suggestions, by filing bug reports or contributing code to our `GitHub repository `_. * If you already have some code changes, you can submit them directly as a `pull request `_. -* If you are wondering where we would greatly appreciate your efforts, check out the ``help wanted`` tag in the `issues list `_ and initiate a discussion there. * If you start working on a feature in the code, let us know by opening an issue or a draft pull request. This helps all of us to keep an overview on what is being done and helps to avoid a situation where we are doing the same work twice in parallel. diff --git a/doc/costs.rst b/doc/costs.rst index 70bd12b8..5ced95dc 100644 --- a/doc/costs.rst +++ b/doc/costs.rst @@ -32,7 +32,7 @@ Based on the parameters above the ``marginal_cost`` and ``capital_cost`` of the .. note:: - Another great resource for `cost assumptions `_. Modifying Cost Assumptions ========================== diff --git a/doc/index.rst b/doc/index.rst index d8d9c852..02b02ce2 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -19,8 +19,8 @@ PyPSA-Eur: An Open Optimisation Model of the European Transmission System .. image:: https://img.shields.io/github/repo-size/pypsa/pypsa-eur :alt: GitHub repo size -.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3520875.svg - :target: https://doi.org/10.5281/zenodo.3520875 +.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3520874.svg + :target: https://doi.org/10.5281/zenodo.3520874 .. image:: https://badges.gitter.im/PyPSA/community.svg :target: https://gitter.im/PyPSA/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge @@ -42,6 +42,8 @@ It contains alternating current lines at and above 220 kV voltage level and all The model is suitable both for operational studies and generation and transmission expansion planning studies. The continental scope and highly resolved spatial scale enables a proper description of the long-range smoothing effects for renewable power generation and their varying resource availability. .. image:: img/base.png + :width: 50% + :align: center The restriction to freely available and open data encourages the open exchange of model data developments and eases the comparison of model results. It provides a full, automated software pipeline to assemble the load-flow-ready model from the original datasets, which enables easy replacement and improvement of the individual parts. @@ -169,16 +171,16 @@ Please use the following BibTeX: :: If you want to cite a specific PyPSA-Eur version, each release of PyPSA-Eur is stored on Zenodo with a release-specific DOI. This can be found linked from the overall PyPSA-Eur Zenodo DOI: -.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3520875.svg - :target: https://doi.org/10.5281/zenodo.3520875 +.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3520874.svg + :target: https://doi.org/10.5281/zenodo.3520874 Pre-Built Networks as a Dataset =============================== There are pre-built networks available as a dataset on Zenodo as well for every release of PyPSA-Eur. -.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3601882.svg - :target: https://doi.org/10.5281/zenodo.3601882 +.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3601881.svg + :target: https://doi.org/10.5281/zenodo.3601881 The included ``.nc`` files are PyPSA network files which can be imported with PyPSA via: diff --git a/doc/installation.rst b/doc/installation.rst index e1de3f4b..16fdf766 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -17,6 +17,7 @@ Clone the Repository First of all, clone the `PyPSA-Eur repository `_ using the version control system ``git``. The path to the directory into which the ``git repository`` is cloned, must **not** have any spaces! +If you do not have ``git`` installed, follow installation instructions `here `_. .. code:: bash @@ -24,8 +25,6 @@ The path to the directory into which the ``git repository`` is cloned, must **no /some/path/without/spaces % git clone https://github.com/PyPSA/pypsa-eur.git -.. note:: - If you do not have ``git`` installed, follow installation instructions `here `_. .. _deps: @@ -46,9 +45,8 @@ The environment can be installed and activated using .../pypsa-eur % conda activate pypsa-eur -.. note:: - Note that activation is local to the currently open shell! - After opening a new terminal window, one needs to reissue the second command! +Note that activation is local to the currently open shell! +After opening a new terminal window, one needs to reissue the second command! .. note:: If you have troubles with a slow ``conda`` installation, we recommend to install @@ -74,25 +72,23 @@ PyPSA is known to work with the free software - `Cbc `_ - `GLPK `_ (`WinGLKP `_) -and the non-free, commercial software (for which free academic licenses are available) +and the non-free, commercial software (for some of which free academic licenses are available) - `Gurobi `_ - `CPLEX `_ +- `FICO® Xpress Solver `_ -and any other solver that works with the underlying modelling framework `Pyomo `_. For installation instructions of these solvers for your operating system, follow the links above. +Commercial solvers such as Gurobi and CPLEX currently significantly outperform open-source solvers for large-scale problems. +It might be the case that you can only retrieve solutions by using a commercial solver. .. seealso:: `Getting a solver in the PyPSA documentation `_ -.. note:: - Commercial solvers such as Gurobi and CPLEX currently significantly outperform open-source solvers for large-scale problems. - It might be the case that you can only retrieve solutions by using a commercial solver. - .. note:: The rules :mod:`cluster_network` and :mod:`simplify_network` solve a quadratic optimisation problem for clustering. The open-source solvers Cbc and GlPK cannot handle this. A fallback to Ipopt is implemented in this case, but requires - also Ipopt to be installed. For an open-source solver setup install in your `conda` environment on OSX/Linux + also Ipopt to be installed. For an open-source solver setup install in your ``conda`` environment on OSX/Linux .. code:: bash diff --git a/doc/introduction.rst b/doc/introduction.rst index 9ca049e7..bc4f267f 100644 --- a/doc/introduction.rst +++ b/doc/introduction.rst @@ -64,4 +64,6 @@ Folder Structure System Requirements =================== -Building the model with the scripts in this repository uses up to 20 GB of memory. Computing optimal investment and operation scenarios requires a strong interior-point solver compatible with the modelling library `Pyomo `_ like `Gurobi `_ or `CPLEX `_ with up to 100 GB of memory. +Building the model with the scripts in this repository runs on a normal computer. +But computing optimal investment and operation scenarios requires a strong interior-point solver +like `Gurobi `_ or `CPLEX `_ with more memory. diff --git a/doc/preparation.rst b/doc/preparation.rst index b2749a41..9e986580 100644 --- a/doc/preparation.rst +++ b/doc/preparation.rst @@ -39,6 +39,7 @@ together into a detailed PyPSA network stored in ``networks/elec.nc``. preparation/retrieve preparation/build_shapes + preparation/build_load_data preparation/build_cutout preparation/build_natura_raster preparation/prepare_links_p_nom diff --git a/doc/preparation/build_load_data.rst b/doc/preparation/build_load_data.rst new file mode 100644 index 00000000..03535981 --- /dev/null +++ b/doc/preparation/build_load_data.rst @@ -0,0 +1,12 @@ +.. + SPDX-FileCopyrightText: 2020-2021 The PyPSA-Eur Authors + + SPDX-License-Identifier: CC-BY-4.0 + +.. _load_data: + +Rule ``build_load_data`` +============================= + + +.. automodule:: build_load_data diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 5d2e0f05..77134e66 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -11,43 +11,118 @@ Release Notes Upcoming Release ================ +* Fix: Value for ``co2base`` in ``config.yaml`` adjusted to 1.487e9 t CO2-eq (from 3.1e9 t CO2-eq). The new value represents emissions related to the electricity sector for EU+UK. The old value was ~2x too high and used when the emissions wildcard in ``{opts}`` was used. + +* Add option to include marginal costs of links representing fuel cells, electrolysis, and battery inverters + [`#232 `_]. + +PyPSA-Eur 0.3.0 (7th December 2020) +================================== + +**New Features** + +Using the ``{opts}`` wildcard for scenarios: + * An option is introduced which adds constraints such that each country or node produces on average a minimal share of its total consumption itself. For example ``EQ0.5c`` set in the ``{opts}`` wildcard requires each country to produce on average at least 50% of its consumption. Additionally, the option ``ATK`` requires autarky at each node and removes all means of power transmission through lines and links. ``ATKc`` only removes - cross-border transfer capacities. Moreover, line and link capacities can be capped in the ``config.yaml`` at - ``lines: s_nom_max:`` and ``links: p_nom_max`` (`#166 `_). + cross-border transfer capacities. + [`#166 `_]. -* Added an option to alter the capital cost of carriers by a factor via ``carrier+factor`` in the ``{opts}`` wildcard. This can be useful for exploring uncertain cost parameters. Example: ``solar+0.5`` reduces the capital cost of solar to 50% of original values (`#167 `_). +* Added an option to alter the capital cost (``c``) or installable potentials (``p``) of carriers by a factor via ``carrier+{c,p}factor`` in the ``{opts}`` wildcard. + This can be useful for exploring uncertain cost parameters. + Example: ``solar+c0.5`` reduces the capital cost of solar to 50% of original values + [`#167 `_, `#207 `_]. -* Add compatibility for pyomo 5.7.0 in :mod:`cluster_network` and :mod:`simplify_network`. +* Added an option to the ``{opts}`` wildcard that applies a time series segmentation algorithm based on renewables, hydro inflow and load time series + to produce a given total number of adjacent snapshots of varying lengths. + This feature is an alternative to downsampling the temporal resolution by simply averaging and + uses the `tsam `_ package + [`#186 `_]. -* Raise a warning if `tech_colors` in the config are not defined for all carriers. +More OPSD integration: -* Corrected HVDC link connections (a) between Norway and Denmark and (b) mainland Italy, Corsica (FR) and Sardinia (IT) (`#181 `_) +* Add renewable power plants from `OPSD `_ to the network for specified technologies. + This will overwrite the capacities calculated from the heuristic approach in :func:`estimate_renewable_capacities()` + [`#212 `_]. -* Added Google Cloud Platform tutorial (for Windows users). +* Electricity consumption data is now retrieved directly from the `OPSD website `_ using the rule :mod:`build_load_data`. + The user can decide whether to take the ENTSO-E power statistics data (default) or the ENTSO-E transparency data + [`#211 `_]. -* Corrected setting of exogenous emission price (in config -> cost -> emission price). This was not weighted by the efficiency and effective emission of the generators. Fixed in `#171 `_. +Other: -* Don't remove capital costs from lines and links, when imposing a line volume limit (wildcard ``lv``) or a line cost limit (``lc``). Previously, these were removed to move the expansion in direction of the limit. +* Added an option to use custom busmaps in rule :mod:`cluster_network`. To use this feature set ``enable: custom_busmap: true``. + Then, the rule looks for custom busmaps at ``data/custom_busmap_elec_s{simpl}_{clusters}.csv``, + which should have the same format as ``resources/busmap_elec_s{simpl}_{clusters}.csv``. + i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc`` + [`#193 `_]. -* Fix bug of clustering offwind-{ac,dc} sites in the option of high-resolution sites for renewables. Now, there are more sites for offwind-{ac,dc} available than network nodes. Before, they were clustered to the resolution of the network. (e.g. elec_s1024_37m.nc: 37 network nodes, 1024 sites) +* Line and link capacities can be capped in the ``config.yaml`` at ``lines: s_nom_max:`` and ``links: p_nom_max``: + [`#166 `_]. -* Use `mamba` (https://github.com/mamba-org/mamba) for faster Travis CI builds (`#196 `_) +* Added Google Cloud Platform tutorial (for Windows users) + [`#177 `_]. -* The N-1 security margin for transmission lines is now fixed to a provided value in ``config.yaml``, removing an undocumented linear interpolation between 0.5 and 0.7 in the range between 37 and 200 nodes. +**Changes** -* The mappings for clustered lines and buses produced by the ``simplify_network`` and ``cluster_network`` rules changed from Hierarchical Data Format (.h5) to Comma-Separated Values format (.csv) (`#198 `_) +* Don't remove capital costs from lines and links, when imposing a line volume limit (``lv``) or a line cost limit (``lc``). + Previously, these were removed to move the expansion in direction of the limit + [`#183 `_]. + +* The mappings for clustered lines and buses produced by the :mod:`simplify_network` and :mod:`cluster_network` rules + changed from Hierarchical Data Format (``.h5``) to Comma-Separated Values format (``.csv``) for ease of use. + [`#198 `_] + +* The N-1 security margin for transmission lines is now fixed to a provided value in ``config.yaml``, + removing an undocumented linear interpolation between 0.5 and 0.7 in the range between 37 and 200 nodes. + [`#199 `_]. + +* Modelling hydrogen and battery storage with Store and Link components is now the default, + rather than using StorageUnit components with fixed power-to-energy ratio + [`#205 `_]. + +* Use ``mamba`` (https://github.com/mamba-org/mamba) for faster Travis CI builds + [`#196 `_]. + +* Multiple smaller changes: Removed unused ``{network}`` wildcard, moved environment files to dedicated ``envs`` folder, + removed sector-coupling components from configuration files, updated documentation colors, minor refactoring and code cleaning + [`#190 `_]. + +**Bugs and Compatibility** + +* Add compatibility for pyomo 5.7.0 in :mod:`cluster_network` and :mod:`simplify_network` + [`#172 `_]. + +* Fixed a bug for storage units such that individual store and dispatch efficiencies are correctly taken account of rather than only their round-trip efficiencies. + In the cost database (``data/costs.csv``) the efficiency of battery inverters should be stated as per discharge/charge rather than per roundtrip + [`#202 `_]. + +* Corrected exogenous emission price setting (in ``config: cost: emission price:``), + which now correctly accounts for the efficiency and effective emission of the generators + [`#171 `_]. + +* Corrected HVDC link connections (a) between Norway and Denmark and (b) mainland Italy, Corsica (FR) and Sardinia (IT) + as well as for East-Western and Anglo-Scottish interconnectors + [`#181 `_, `#206 `_]. + +* Fix bug of clustering ``offwind-{ac,dc}`` generators in the option of high-resolution generators for renewables. + Now, there are more sites for ``offwind-{ac,dc}`` available than network nodes. + Before, they were clustered to the resolution of the network (``elec_s1024_37m.nc``: 37 network nodes, 1024 generators) + [`#191 `_]. + +* Raise a warning if ``tech_colors`` in the config are not defined for all carriers + [`#178 `_]. PyPSA-Eur 0.2.0 (8th June 2020) ================================== -* The optimization is now performed using the ``pyomo=False`` setting in the :func:`pypsa.lopf.network_lopf`. This speeds up the solving process significantly and consumes much less memory. The inclusion of additional constraints were adjusted to the new implementation. They are all passed to the :func:`network_lopf` function via the ``extra_functionality`` argument. The rule ``trace_solve_network`` was integrated into the rule :mod:`solve_network` and can be activated via configuration with ``solving: options: track_iterations: true``. The charging and discharging capacities of batteries modelled as store-link combination are now coupled (`#116 `_). +* The optimization is now performed using the ``pyomo=False`` setting in the :func:`pypsa.lopf.network_lopf`. This speeds up the solving process significantly and consumes much less memory. The inclusion of additional constraints were adjusted to the new implementation. They are all passed to the :func:`network_lopf` function via the ``extra_functionality`` argument. The rule ``trace_solve_network`` was integrated into the rule :mod:`solve_network` and can be activated via configuration with ``solving: options: track_iterations: true``. The charging and discharging capacities of batteries modelled as store-link combination are now coupled [`#116 `_]. -* An updated extract of the `ENTSO-E Transmission System Map `_ (including Malta) was added to the repository using the `GridKit `_ tool. This tool has been updated to retrieve up-to-date map extracts using a single `script `_. The update extract features 5322 buses, 6574 lines, 46 links. (`#118 `_). +* An updated extract of the `ENTSO-E Transmission System Map `_ (including Malta) was added to the repository using the `GridKit `_ tool. This tool has been updated to retrieve up-to-date map extracts using a single `script `_. The update extract features 5322 buses, 6574 lines, 46 links. [`#118 `_]. -* Added `FSFE REUSE `_ compliant license information. Documentation now licensed under CC-BY-4.0 (`#160 `_). +* Added `FSFE REUSE `_ compliant license information. Documentation now licensed under CC-BY-4.0 [`#160 `_]. * Added a 30 minute `video introduction `_ and a 20 minute `video tutorial `_ @@ -55,19 +130,19 @@ PyPSA-Eur 0.2.0 (8th June 2020) * Added an option to skip iterative solving usually performed to update the line impedances of expanded lines at ``solving: options: skip_iterations:``. -* ``snakemake`` rules for retrieving cutouts and the natura raster can now be disabled independently from their respective rules to build them; via ``config.*yaml`` (`#136 `_). +* ``snakemake`` rules for retrieving cutouts and the natura raster can now be disabled independently from their respective rules to build them; via ``config.*yaml`` [`#136 `_]. -* Removed the ``id`` column for custom power plants in ``data/custom_powerplants.csv`` to avoid custom power plants with conflicting ids getting attached to the wrong bus (`#131 `_). +* Removed the ``id`` column for custom power plants in ``data/custom_powerplants.csv`` to avoid custom power plants with conflicting ids getting attached to the wrong bus [`#131 `_]. -* Add option ``renewables: {carrier}: keep_all_available_areas:`` to use all availabe weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed (`#150 `_). +* Add option ``renewables: {carrier}: keep_all_available_areas:`` to use all availabe weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed [`#150 `_]. -* Added a function ``_helpers.load_network()`` which loads a network with overridden components specified in ``snakemake.config['override_components']`` (`#128 `_). +* Added a function ``_helpers.load_network()`` which loads a network with overridden components specified in ``snakemake.config['override_components']`` [`#128 `_]. -* Bugfix in :mod:`base_network` which now finds all closest links, not only the first entry (`#143 `_). +* Bugfix in :mod:`base_network` which now finds all closest links, not only the first entry [`#143 `_]. -* Bugfix in :mod:`cluster_network` which now skips recalculation of link parameters if there are no links (`#149 `_). +* Bugfix in :mod:`cluster_network` which now skips recalculation of link parameters if there are no links [`#149 `_]. -* Added information on pull requests to contribution guidelines (`#151 `_). +* Added information on pull requests to contribution guidelines [`#151 `_]. * Improved documentation on open-source solver setup and added usage warnings. @@ -78,31 +153,31 @@ PyPSA-Eur 0.1.0 (9th January 2020) This is the first release of PyPSA-Eur, a model of the European power system at the transmission network level. Recent changes include: -* Documentation on installation, workflows and configuration settings is now available online at `pypsa-eur.readthedocs.io `_ (`#65 `_). +* Documentation on installation, workflows and configuration settings is now available online at `pypsa-eur.readthedocs.io `_ [`#65 `_]. -* The ``conda`` environment files were updated and extended (`#81 `_). +* The ``conda`` environment files were updated and extended [`#81 `_]. -* The power plant database was updated with extensive filtering options via ``pandas.query`` functionality (`#84 `_ and `#94 `_). +* The power plant database was updated with extensive filtering options via ``pandas.query`` functionality [`#84 `_ and `#94 `_]. -* Continuous integration testing with `Travis CI `_ is now included for Linux, Mac and Windows (`#82 `_). +* Continuous integration testing with `Travis CI `_ is now included for Linux, Mac and Windows [`#82 `_]. -* Data dependencies were moved to `zenodo `_ and are now versioned (`#60 `_). +* Data dependencies were moved to `zenodo `_ and are now versioned [`#60 `_]. -* Data dependencies are now retrieved directly from within the snakemake workflow (`#86 `_). +* Data dependencies are now retrieved directly from within the snakemake workflow [`#86 `_]. -* Emission prices can be added to marginal costs of generators through the keyworks ``Ep`` in the ``{opts}`` wildcard (`#100 `_). +* Emission prices can be added to marginal costs of generators through the keyworks ``Ep`` in the ``{opts}`` wildcard [`#100 `_]. -* An option is introduced to add extendable nuclear power plants to the network (`#98 `_). +* An option is introduced to add extendable nuclear power plants to the network [`#98 `_]. -* Focus weights can now be specified for particular countries for the network clustering, which allows to set a proportion of the total number of clusters for particular countries (`#87 `_). +* Focus weights can now be specified for particular countries for the network clustering, which allows to set a proportion of the total number of clusters for particular countries [`#87 `_]. -* A new rule :mod:`add_extra_components` allows to add additional components to the network only after clustering. It is thereby possible to model storage units (e.g. battery and hydrogen) in more detail via a combination of ``Store``, ``Link`` and ``Bus`` elements (`#97 `_). +* A new rule :mod:`add_extra_components` allows to add additional components to the network only after clustering. It is thereby possible to model storage units (e.g. battery and hydrogen) in more detail via a combination of ``Store``, ``Link`` and ``Bus`` elements [`#97 `_]. -* Hydrogen pipelines (including cost assumptions) can now be added alongside clustered network connections in the rule :mod:`add_extra_components` . Set ``electricity: extendable_carriers: Link: [H2 pipeline]`` and ensure hydrogen storage is modelled as a ``Store``. This is a first simplified stage (`#108 `_). +* Hydrogen pipelines (including cost assumptions) can now be added alongside clustered network connections in the rule :mod:`add_extra_components` . Set ``electricity: extendable_carriers: Link: [H2 pipeline]`` and ensure hydrogen storage is modelled as a ``Store``. This is a first simplified stage [`#108 `_]. -* Logfiles for all rules of the ``snakemake`` workflow are now written in the folder ``log/`` (`#102 `_). +* Logfiles for all rules of the ``snakemake`` workflow are now written in the folder ``log/`` [`#102 `_]. -* The new function ``_helpers.mock_snakemake`` creates a ``snakemake`` object which mimics the actual ``snakemake`` object produced by workflow by parsing the ``Snakefile`` and setting all paths for inputs, outputs, and logs. This allows running all scripts within a (I)python terminal (or just by calling ``python ``) and thereby facilitates developing and debugging scripts significantly (`#107 `_). +* The new function ``_helpers.mock_snakemake`` creates a ``snakemake`` object which mimics the actual ``snakemake`` object produced by workflow by parsing the ``Snakefile`` and setting all paths for inputs, outputs, and logs. This allows running all scripts within a (I)python terminal (or just by calling ``python ``) and thereby facilitates developing and debugging scripts significantly [`#107 `_]. Release Process =============== @@ -122,10 +197,10 @@ Release Process * Tag a release on Github via ``git tag v0.x.x``, ``git push``, ``git push --tags``. Include release notes in the tag message. -* Upload code to `zenodo code repository `_ with `GNU GPL 3.0 `_ license. +* Upload code to `zenodo code repository `_ with `GNU GPL 3.0 `_ license. * Create pre-built networks for ``config.default.yaml`` by running ``snakemake -j 1 extra_components_all_networks``. -* Upload pre-built networks to `zenodo data repository `_ with `CC BY 4.0 `_ license. +* Upload pre-built networks to `zenodo data repository `_ with `CC BY 4.0 `_ license. * Send announcement on the `PyPSA and PyPSA-Eur mailing list `_. diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 53252ff8..76d9a711 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -59,35 +59,35 @@ It is also possible to allow less or more carbon-dioxide emissions. Here, we lim .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 35,37 + :lines: 36,38 PyPSA-Eur also includes a database of existing conventional powerplants. We can select which types of powerplants we like to be included with fixed capacities: .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 35,51 + :lines: 36,52 To accurately model the temporal and spatial availability of renewables such as wind and solar energy, we rely on historical weather data. It is advisable to adapt the required range of coordinates to the selection of countries. .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 53-61 + :lines: 54-62 We can also decide which weather data source should be used to calculate potentials and capacity factor time-series for each carrier. For example, we may want to use the ERA-5 dataset for solar and not the default SARAH-2 dataset. .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 63,106-107 + :lines: 64,107-108 Finally, it is possible to pick a solver. For instance, this tutorial uses the open-source solvers CBC and Ipopt and does not rely on the commercial solvers Gurobi or CPLEX (for which free academic licenses are available). .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 164,175-176 + :lines: 170,180-181 .. note:: @@ -271,7 +271,7 @@ the wildcards given in ``scenario`` in the configuration file ``config.yaml`` ar .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 13-18 + :lines: 14-18 In this example we would not only solve a 6-node model of Germany but also a 2-node model. @@ -286,12 +286,4 @@ The solved networks can be analysed just like any other PyPSA network (e.g. in J network = pypsa.Network("results/networks/elec_s_6_ec_lcopt_Co2L-24H.nc") - ... - For inspiration, read the `examples section in the PyPSA documentation `_. - -.. note:: - - There are rules for summaries and plotting available in the repository of PyPSA-Eur. - - They are currently under revision and therefore not yet documented. diff --git a/doc/wildcards.rst b/doc/wildcards.rst index 71b39ef1..227997d1 100644 --- a/doc/wildcards.rst +++ b/doc/wildcards.rst @@ -27,9 +27,6 @@ The ``{simpl}`` wildcard specifies number of buses a detailed network model should be pre-clustered to in the rule :mod:`simplify_network` (before :mod:`cluster_network`). -.. seealso:: - :mod:`simplify_network` - .. _clusters: The ``{clusters}`` wildcard @@ -45,9 +42,6 @@ If an `m` is placed behind the number of clusters (e.g. ``100m``), generators are only moved to the clustered buses but not aggregated by carrier; i.e. the clustered bus may have more than one e.g. wind generator. -.. seealso:: - :mod:`cluster_network` - .. _ll: The ``{ll}`` wildcard @@ -79,9 +73,6 @@ The wildcard, in general, consists of two parts: (c) ``c1.25`` will allow to build a transmission network that costs no more than 25 % more than the current system. -.. seealso:: - :mod:`prepare_network` - .. _opts: The ``{opts}`` wildcard @@ -98,21 +89,18 @@ It may hold multiple triggers separated by ``-``, i.e. ``Co2L-3H`` contains the :widths: 10,20,10,10 :file: configtables/opts.csv -.. seealso:: - :mod:`prepare_network`, :mod:`solve_network` - .. _country: The ``{country}`` wildcard ========================== -The rules ``make_summary`` and ``plot_summary`` (generating summaries of all or a subselection -of the solved networks) as well as ``plot_p_nom_max`` (for plotting the cumulative +The rules :mod:`make_summary` and :mod:`plot_summary` (generating summaries of all or a subselection +of the solved networks) as well as :mod:`plot_p_nom_map` (for plotting the cumulative generation potentials for renewable technologies) can be narrowed to individual countries using the ``{country}`` wildcard. -If ``country = all``, then the rule acts on the network for all countries -defined in ``config.yaml``. If otherwise ``country = DE`` or another 2-letter +If ``country=all``, then the rule acts on the network for all countries +defined in ``config.yaml``. If otherwise ``country=DE`` or another 2-letter country code, then the network is narrowed to buses of this country for the rule. For example to get a summary of the energy generated in Germany (in the solution for Europe) use: @@ -121,9 +109,6 @@ in Germany (in the solution for Europe) use: snakemake -j 1 results/summaries/elec_s_all_lall_Co2L-3H_DE -.. seealso:: - :mod:`make_summary`, :mod:`plot_summary`, :mod:`plot_p_nom_max` - .. _cutout_wc: The ``{cutout}`` wildcard @@ -133,9 +118,6 @@ The ``{cutout}`` wildcard facilitates running the rule :mod:`build_cutout` for all cutout configurations specified under ``atlite: cutouts:``. These cutouts will be stored in a folder specified by ``{cutout}``. -.. seealso:: - :mod:`build_cutout`, :ref:`atlite_cf` - .. _technology: The ``{technology}`` wildcard @@ -151,22 +133,16 @@ For instance ``{technology}`` can be used to plot regionally disaggregated poten with the rule :mod:`plot_p_nom_max` or to summarize a particular technology's full load hours in various countries with the rule :mod:`build_country_flh`. -.. seealso:: - :mod:`build_renewable_profiles`, :mod:`plot_p_nom_max`, :mod:`build_country_flh` - .. _attr: The ``{attr}`` wildcard ======================= -The ``{attr}`` wildcard specifies which attribute are used for size +The ``{attr}`` wildcard specifies which attribute is used for size representations of network components on a map plot produced by the rule -``plot_network``. While it might be extended in the future, ``{attr}`` +:mod:`plot_network`. While it might be extended in the future, ``{attr}`` currently only supports plotting of ``p_nom``. -.. seealso:: - :mod:`plot_network` - .. _ext: The ``{ext}`` wildcard @@ -181,6 +157,3 @@ formats depends on the used backend. To query the supported file types on your s import matplotlib.pyplot as plt plt.gcf().canvas.get_supported_filetypes() - -.. seealso:: - :mod:`plot_network`, :mod:`plot_summary`, :mod:`plot_p_nom_max` diff --git a/envs/environment.docs.yaml b/envs/environment.docs.yaml index fc141a02..0c937e43 100755 --- a/envs/environment.docs.yaml +++ b/envs/environment.docs.yaml @@ -6,6 +6,7 @@ name: pypsa-eur-docs channels: - conda-forge dependencies: + - python<=3.7 - pip - pypsa>=0.17.1 - atlite=0.0.3 diff --git a/envs/environment.yaml b/envs/environment.yaml index 12eeaaa5..31ba59d7 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -4,7 +4,6 @@ name: pypsa-eur channels: - - defaults - conda-forge - bioconda - http://conda.anaconda.org/gurobi @@ -19,15 +18,16 @@ dependencies: # Dependencies of the workflow itself - xlrd + - openpyxl - scikit-learn - pycountry - seaborn - - snakemake-minimal<=5.24.2 # until https://github.com/snakemake/snakemake/issues/635 closed + - snakemake-minimal - memory_profiler - yaml - pytables - lxml - - powerplantmatching>=0.4.3 + - powerplantmatching>=0.4.8 - numpy<=1.19.0 # otherwise macos fails # Second order dependencies which should really be deps of atlite @@ -51,12 +51,11 @@ dependencies: - rasterio - shapely - libgdal<=3.0.4 - - # Solvers - - gurobi:gurobi # until https://github.com/conda-forge/pypsa-feedstock/issues/4 closed + - descartes - pip: - vresutils==0.3.1 + - tsam>=1.1.0 - git+https://github.com/PyPSA/glaes.git#egg=glaes - git+https://github.com/PyPSA/geokit.git#egg=geokit - cdsapi diff --git a/scripts/_helpers.py b/scripts/_helpers.py index e0e16d20..807c439f 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -170,6 +170,7 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False): n.iterate_components(iterkeys(components), skip_empty=False), itervalues(components) ): + if c.df.empty: continue if not existing_only: p_nom += "_opt" costs[(c.list_name, 'capital')] = (c.df[p_nom] * c.df.capital_cost).groupby(c.df.carrier).sum() if p_attr is not None: diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index a5495585..8a9ee339 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -24,13 +24,13 @@ Relevant Settings conventional_carriers: co2limit: extendable_carriers: - Generator: + include_renewable_capacities_from_OPSD: estimate_renewable_capacities_from_capacity_stats: load: scaling_factor: - renewable: (keys) + renewable: hydro: carriers: hydro_max_hours: @@ -52,15 +52,8 @@ Inputs .. image:: ../img/hydrocapacities.png :scale: 34 % -- ``data/geth2015_hydro_capacities.csv``: alternative to capacities above; NOT CURRENTLY USED! -- ``data/bundle/time_series_60min_singleindex_filtered.csv``: Hourly per-country load profiles since 2010 from the `ENTSO-E statistical database `_ - - .. image:: ../img/load-box.png - :scale: 33 % - - .. image:: ../img/load-ts.png - :scale: 33 % - +- ``data/geth2015_hydro_capacities.csv``: alternative to capacities above; not currently used! +- ``resources/opsd_load.csv`` Hourly per-country load profiles. - ``resources/regions_onshore.geojson``: confer :ref:`busregions` - ``resources/nuts3_shapes.geojson``: confer :ref:`shapes` - ``resources/powerplants.csv``: confer :ref:`powerplants` @@ -98,7 +91,9 @@ import pandas as pd import numpy as np import xarray as xr import geopandas as gpd -import powerplantmatching as ppm + +import powerplantmatching as pm +from powerplantmatching.export import map_country_bus from vresutils.costdata import annuity from vresutils.load import timeseries_opsd @@ -169,13 +164,10 @@ def load_costs(Nyears=1., tech_costs=None, config=None, elec_config=None): def costs_for_storage(store, link1, link2=None, max_hours=1.): capital_cost = link1['capital_cost'] + max_hours * store['capital_cost'] - efficiency = link1['efficiency']**0.5 if link2 is not None: capital_cost += link2['capital_cost'] - efficiency *= link2['efficiency']**0.5 return pd.Series(dict(capital_cost=capital_cost, marginal_cost=0., - efficiency=efficiency, co2_emissions=0.)) if elec_config is None: @@ -213,6 +205,7 @@ def attach_load(n): regions = (gpd.read_file(snakemake.input.regions).set_index('name') .reindex(substation_lv_i)) +# ------------- TO MERGE -------------- available_years = range(2011,2016) requested_years = n.snapshots.year[[0,-1]] use_fallback = any(year not in available_years for year in requested_years) @@ -237,9 +230,15 @@ def attach_load(n): # Convert to naive UTC (has to be explicit since pandas 0.24) opsd_load.index = opsd_load.index.tz_localize(None) - nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index('index') + opsd_load = (pd.read_csv(snakemake.input.load, index_col=0, parse_dates=True) + .filter(items=snakemake.config['countries'])) - def normed(x): return x.divide(x.sum()) + scaling = snakemake.config.get('load', {}).get('scaling_factor', 1.0) + logger.info(f"Load data scaled with scalling factor {scaling}.") + opsd_load *= scaling +# ------------- TO MERGE -------------- + + nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index('index') def upsample(cntry, group): l = opsd_load[cntry] @@ -255,7 +254,8 @@ def attach_load(n): index=group.index) # relative factors 0.6 and 0.4 have been determined from a linear - # regression on the country to continent load data (refer to vresutils.load._upsampling_weights) + # regression on the country to continent load data + # (refer to vresutils.load._upsampling_weights) factors = normed(0.6 * normed(gdp_n) + 0.4 * normed(pop_n)) return pd.DataFrame(factors.values * l.values[:,np.newaxis], index=l.index, columns=factors.index) @@ -273,6 +273,11 @@ def update_transmission_costs(n, costs, length_factor=1.0, simple_hvdc_costs=Fal if n.links.empty: return dc_b = n.links.carrier == 'DC' + + # If there are no dc links, then the 'underwater_fraction' column + # may be missing. Therefore we have to return here. + if n.links.loc[dc_b].empty: return + if simple_hvdc_costs: costs = (n.links.loc[dc_b, 'length'] * length_factor * costs.at['HVDC overhead', 'capital_cost']) @@ -331,7 +336,7 @@ def attach_conventional_generators(n, costs, ppl): ppl = (ppl.query('carrier in @carriers').join(costs, on='carrier') .rename(index=lambda s: 'C' + str(s))) - logger.info('Adding {} generators with capacities\n{}' + logger.info('Adding {} generators with capacities [MW] \n{}' .format(len(ppl), ppl.groupby('carrier').p_nom.sum())) n.madd("Generator", ppl.index, @@ -360,7 +365,7 @@ def attach_hydro(n, costs, ppl): country = ppl['bus'].map(n.buses.country).rename("country") - inflow_idx = ror.index | hydro.index + inflow_idx = ror.index.union(hydro.index) if not inflow_idx.empty: dist_key = ppl.loc[inflow_idx, 'p_nom'].groupby(country).transform(normed) @@ -494,6 +499,39 @@ def attach_extendable_generators(n, costs, ppl): "Only OCGT, CCGT and nuclear are allowed at the moment.") + +def attach_OPSD_renewables(n): + + available = ['DE', 'FR', 'PL', 'CH', 'DK', 'CZ', 'SE', 'GB'] + tech_map = {'Onshore': 'onwind', 'Offshore': 'offwind', 'Solar': 'solar'} + countries = set(available) & set(n.buses.country) + techs = snakemake.config['electricity'].get('renewable_capacities_from_OPSD', []) + tech_map = {k: v for k, v in tech_map.items() if v in techs} + + if not tech_map: + return + + logger.info(f'Using OPSD renewable capacities in {", ".join(countries)} ' + f'for technologies {", ".join(tech_map.values())}.') + + df = pd.concat([pm.data.OPSD_VRE_country(c) for c in countries]) + technology_b = ~df.Technology.isin(['Onshore', 'Offshore']) + df['Fueltype'] = df.Fueltype.where(technology_b, df.Technology) + df = df.query('Fueltype in @tech_map').powerplant.convert_country_to_alpha2() + + for fueltype, carrier_like in tech_map.items(): + gens = n.generators[lambda df: df.carrier.str.contains(carrier_like)] + buses = n.buses.loc[gens.bus.unique()] + gens_per_bus = gens.groupby('bus').p_nom.count() + + caps = map_country_bus(df.query('Fueltype == @fueltype'), buses) + caps = caps.groupby(['bus']).Capacity.sum() + caps = caps / gens_per_bus.reindex(caps.index, fill_value=1) + + n.generators.p_nom.update(gens.bus.map(caps).dropna()) + + + def estimate_renewable_capacities(n, tech_map=None): if tech_map is None: tech_map = (snakemake.config['electricity'] @@ -501,16 +539,25 @@ def estimate_renewable_capacities(n, tech_map=None): if len(tech_map) == 0: return - capacities = (ppm.data.Capacity_stats().powerplant.convert_country_to_alpha2() + capacities = (pm.data.Capacity_stats().powerplant.convert_country_to_alpha2() [lambda df: df.Energy_Source_Level_2] .set_index(['Fueltype', 'Country']).sort_index()) countries = n.buses.country.unique() + if len(countries) == 0: return + + logger.info('heuristics applied to distribute renewable capacities [MW] \n{}' + .format(capacities.query('Fueltype in @tech_map.keys() and Capacity >= 0.1') + .groupby('Country').agg({'Capacity': 'sum'}))) + for ppm_fueltype, techs in tech_map.items(): tech_capacities = capacities.loc[ppm_fueltype, 'Capacity']\ .reindex(countries, fill_value=0.) - tech_i = n.generators.query('carrier in @techs').index + #tech_i = n.generators.query('carrier in @techs').index + tech_i = (n.generators.query('carrier in @techs') + [n.generators.query('carrier in @techs') + .bus.map(n.buses.country).isin(countries)].index) n.generators.loc[tech_i, 'p_nom'] = ( (n.generators_t.p_max_pu[tech_i].mean() * n.generators.loc[tech_i, 'p_nom_max']) # maximal yearly generation @@ -560,6 +607,8 @@ if __name__ == "__main__": attach_extendable_generators(n, costs, ppl) estimate_renewable_capacities(n) + attach_OPSD_renewables(n) + add_nice_carrier_names(n) n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index c3af5c68..17692f58 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -73,6 +73,9 @@ def attach_storageunits(n, costs): buses_i = n.buses.index + lookup_store = {"H2": "electrolysis", "battery": "battery inverter"} + lookup_dispatch = {"H2": "fuel cell", "battery": "battery inverter"} + for carrier in carriers: n.madd("StorageUnit", buses_i, ' ' + carrier, bus=buses_i, @@ -80,8 +83,8 @@ def attach_storageunits(n, costs): p_nom_extendable=True, capital_cost=costs.at[carrier, 'capital_cost'], marginal_cost=costs.at[carrier, 'marginal_cost'], - efficiency_store=costs.at[carrier, 'efficiency'], - efficiency_dispatch=costs.at[carrier, 'efficiency'], + efficiency_store=costs.at[lookup_store[carrier], 'efficiency'], + efficiency_dispatch=costs.at[lookup_dispatch[carrier], 'efficiency'], max_hours=max_hours[carrier], cyclic_state_of_charge=True) @@ -111,7 +114,8 @@ def attach_stores(n, costs): carrier='H2 electrolysis', p_nom_extendable=True, efficiency=costs.at["electrolysis", "efficiency"], - capital_cost=costs.at["electrolysis", "capital_cost"]) + capital_cost=costs.at["electrolysis", "capital_cost"], + marginal_cost=costs.at["electrolysis", "marginal_cost"]) n.madd("Link", h2_buses_i + " Fuel Cell", bus0=h2_buses_i, @@ -120,7 +124,8 @@ def attach_stores(n, costs): p_nom_extendable=True, efficiency=costs.at["fuel cell", "efficiency"], #NB: fixed cost is per MWel - capital_cost=costs.at["fuel cell", "capital_cost"] * costs.at["fuel cell", "efficiency"]) + capital_cost=costs.at["fuel cell", "capital_cost"] * costs.at["fuel cell", "efficiency"], + marginal_cost=costs.at["fuel cell", "marginal_cost"]) if 'battery' in carriers: b_buses_i = n.madd("Bus", buses_i + " battery", carrier="battery", **bus_sub_dict) @@ -130,23 +135,27 @@ def attach_stores(n, costs): carrier='battery', e_cyclic=True, e_nom_extendable=True, - capital_cost=costs.at['battery storage', 'capital_cost']) + capital_cost=costs.at['battery storage', 'capital_cost'], + marginal_cost=costs.at["battery", "marginal_cost"]) n.madd("Link", b_buses_i + " charger", bus0=buses_i, bus1=b_buses_i, carrier='battery charger', - efficiency=costs.at['battery inverter', 'efficiency']**0.5, + efficiency=costs.at['battery inverter', 'efficiency'], capital_cost=costs.at['battery inverter', 'capital_cost'], - p_nom_extendable=True) + p_nom_extendable=True, + marginal_cost=costs.at["battery inverter", "marginal_cost"]) n.madd("Link", b_buses_i + " discharger", bus0=b_buses_i, bus1=buses_i, carrier='battery discharger', - efficiency=costs.at['battery inverter','efficiency']**0.5, + efficiency=costs.at['battery inverter','efficiency'], capital_cost=costs.at['battery inverter', 'capital_cost'], - p_nom_extendable=True) + p_nom_extendable=True, + marginal_cost=costs.at["battery inverter", "marginal_cost"]) + def attach_hydrogen_pipelines(n, costs): diff --git a/scripts/base_network.py b/scripts/base_network.py index 0b535218..c241f270 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -205,8 +205,8 @@ def _add_links_from_tyndp(buses, links): buses = buses.loc[keep_b['Bus']] links = links.loc[keep_b['Link']] - links_tyndp["j"] = _find_closest_links(links, links_tyndp, distance_upper_bound=0.15) - # Corresponds approximately to 15km tolerances + links_tyndp["j"] = _find_closest_links(links, links_tyndp, distance_upper_bound=0.20) + # Corresponds approximately to 20km tolerances if links_tyndp["j"].notnull().any(): logger.info("TYNDP links already in the dataset (skipping): " + ", ".join(links_tyndp.loc[links_tyndp["j"].notnull(), "Name"])) @@ -552,6 +552,9 @@ def base_network(): n = pypsa.Network() n.name = 'PyPSA-Eur' + n.set_snapshots(pd.date_range(freq='h', **snakemake.config['snapshots'])) + n.snapshot_weightings[:] *= 8760. / n.snapshot_weightings.sum() + n.import_components_from_dataframe(buses, "Bus") n.import_components_from_dataframe(lines, "Line") n.import_components_from_dataframe(transformers, "Transformer") diff --git a/scripts/build_load_data.py b/scripts/build_load_data.py new file mode 100755 index 00000000..e31fa3f2 --- /dev/null +++ b/scripts/build_load_data.py @@ -0,0 +1,227 @@ +# SPDX-FileCopyrightText: : 2020 @JanFrederickUnnewehr, The PyPSA-Eur Authors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +""" + +This rule downloads the load data from `Open Power System Data Time series `_. For all countries in the network, the per country load timeseries with suffix ``_load_actual_entsoe_transparency`` are extracted from the dataset. After filling small gaps linearly and large gaps by copying time-slice of a given period, the load data is exported to a ``.csv`` file. + +Relevant Settings +----------------- + +.. code:: yaml + + snapshots: + + load: + url: + interpolate_limit: + time_shift_for_large_gaps: + manual_adjustments: + + +.. seealso:: + Documentation of the configuration file ``config.yaml`` at + :ref:`load_cf` + +Inputs +------ + + +Outputs +------- + +- ``resource/time_series_60min_singleindex_filtered.csv``: + + +""" + +import logging +logger = logging.getLogger(__name__) +from _helpers import configure_logging + +import pandas as pd +import numpy as np +import dateutil +from pandas import Timedelta as Delta + + +def load_timeseries(fn, years, countries, powerstatistics=True): + """ + Read load data from OPSD time-series package version 2020-10-06. + + Parameters + ---------- + years : None or slice() + Years for which to read load data (defaults to + slice("2018","2019")) + fn : str + File name or url location (file format .csv) + countries : listlike + Countries for which to read load data. + powerstatistics: bool + Whether the electricity consumption data of the ENTSOE power + statistics (if true) or of the ENTSOE transparency map (if false) + should be parsed. + + Returns + ------- + load : pd.DataFrame + Load time-series with UTC timestamps x ISO-2 countries + """ + logger.info(f"Retrieving load data from '{fn}'.") + + pattern = 'power_statistics' if powerstatistics else '_transparency' + pattern = f'_load_actual_entsoe_{pattern}' + rename = lambda s: s[:-len(pattern)] + date_parser = lambda x: dateutil.parser.parse(x, ignoretz=True) + return (pd.read_csv(fn, index_col=0, parse_dates=[0], date_parser=date_parser) + .filter(like=pattern) + .rename(columns=rename) + .dropna(how="all", axis=0) + .rename(columns={'GB_UKM' : 'GB'}) + .filter(items=countries) + .loc[years]) + + +def consecutive_nans(ds): + return (ds.isnull().astype(int) + .groupby(ds.notnull().astype(int).cumsum()[ds.isnull()]) + .transform('sum').fillna(0)) + + +def fill_large_gaps(ds, shift): + """ + Fill up large gaps with load data from the previous week. + + This function fills gaps ragning from 3 to 168 hours (one week). + """ + shift = Delta(shift) + nhours = shift / np.timedelta64(1, 'h') + if (consecutive_nans(ds) > nhours).any(): + logger.warning('There exist gaps larger then the time shift used for ' + 'copying time slices.') + time_shift = pd.Series(ds.values, ds.index + shift) + return ds.where(ds.notnull(), time_shift.reindex_like(ds)) + + +def nan_statistics(df): + def max_consecutive_nans(ds): + return (ds.isnull().astype(int) + .groupby(ds.notnull().astype(int).cumsum()) + .sum().max()) + consecutive = df.apply(max_consecutive_nans) + total = df.isnull().sum() + max_total_per_month = df.isnull().resample('m').sum().max() + return pd.concat([total, consecutive, max_total_per_month], + keys=['total', 'consecutive', 'max_total_per_month'], axis=1) + + +def copy_timeslice(load, cntry, start, stop, delta): + start = pd.Timestamp(start) + stop = pd.Timestamp(stop) + if start-delta in load.index and stop in load.index and cntry in load: + load.loc[start:stop, cntry] = load.loc[start-delta:stop-delta, cntry].values + + +def manual_adjustment(load, powerstatistics): + """ + Adjust gaps manual for load data from OPSD time-series package. + + 1. For the ENTSOE power statistics load data (if powerstatistics is True) + + Kosovo (KV) and Albania (AL) do not exist in the data set. Kosovo gets the + same load curve as Serbia and Albania the same as Macdedonia, both scaled + by the corresponding ratio of total energy consumptions reported by + IEA Data browser [0] for the year 2013. + + 2. For the ENTSOE transparency load data (if powerstatistics is False) + + Albania (AL) and Macedonia (MK) do not exist in the data set. Both get the + same load curve as Montenegro, scaled by the corresponding ratio of total energy + consumptions reported by IEA Data browser [0] for the year 2016. + + [0] https://www.iea.org/data-and-statistics?country=WORLD&fuel=Electricity%20and%20heat&indicator=TotElecCons + + + Parameters + ---------- + load : pd.DataFrame + Load time-series with UTC timestamps x ISO-2 countries + powerstatistics: bool + Whether argument load comprises the electricity consumption data of + the ENTSOE power statistics or of the ENTSOE transparency map + + Returns + ------- + load : pd.DataFrame + Manual adjusted and interpolated load time-series with UTC + timestamps x ISO-2 countries + """ + + if powerstatistics: + if 'MK' in load.columns: + if 'AL' not in load.columns or load.AL.isnull().values.all(): + load['AL'] = load['MK'] * (4.1 / 7.4) + if 'RS' in load.columns: + if 'KV' not in load.columns or load.KV.isnull().values.all(): + load['KV'] = load['RS'] * (4.8 / 27.) + + copy_timeslice(load, 'GR', '2015-08-11 21:00', '2015-08-15 20:00', Delta(weeks=1)) + copy_timeslice(load, 'AT', '2018-12-31 22:00', '2019-01-01 22:00', Delta(days=2)) + copy_timeslice(load, 'CH', '2010-01-19 07:00', '2010-01-19 22:00', Delta(days=1)) + copy_timeslice(load, 'CH', '2010-03-28 00:00', '2010-03-28 21:00', Delta(days=1)) + # is a WE, so take WE before + copy_timeslice(load, 'CH', '2010-10-08 13:00', '2010-10-10 21:00', Delta(weeks=1)) + copy_timeslice(load, 'CH', '2010-11-04 04:00', '2010-11-04 22:00', Delta(days=1)) + copy_timeslice(load, 'NO', '2010-12-09 11:00', '2010-12-09 18:00', Delta(days=1)) + # whole january missing + copy_timeslice(load, 'GB', '2009-12-31 23:00', '2010-01-31 23:00', Delta(days=-364)) + + else: + if 'ME' in load: + if 'AL' not in load and 'AL' in countries: + load['AL'] = load.ME * (5.7/2.9) + if 'MK' not in load and 'MK' in countries: + load['MK'] = load.ME * (6.7/2.9) + copy_timeslice(load, 'BG', '2018-10-27 21:00', '2018-10-28 22:00', Delta(weeks=1)) + + return load + + +if __name__ == "__main__": + + if 'snakemake' not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake('build_load_data') + + configure_logging(snakemake) + + config = snakemake.config + powerstatistics = config['load']['power_statistics'] + url = config['load']['url'] + interpolate_limit = config['load']['interpolate_limit'] + countries = config['countries'] + snapshots = pd.date_range(freq='h', **config['snapshots']) + years = slice(snapshots[0], snapshots[-1]) + time_shift = config['load']['time_shift_for_large_gaps'] + + load = load_timeseries(url, years, countries, powerstatistics) + + if config['load']['manual_adjustments']: + load = manual_adjustment(load, powerstatistics) + + logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.") + load = load.interpolate(method='linear', limit=interpolate_limit) + + logger.info("Filling larger gaps by copying time-slices of period " + f"'{time_shift}'.") + load = load.apply(fill_large_gaps, shift=time_shift) + + assert not load.isna().any().any(), ( + 'Load data contains nans. Adjust the parameters ' + '`time_shift_for_large_gaps` or modify the `manual_adjustment` function ' + 'for implementing the needed load data modifications.') + + load.to_csv(snakemake.output[0]) + diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index e48cb919..2f26e326 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -33,8 +33,9 @@ Inputs - ``resources/regions_onshore_elec{year}_s{simpl}.geojson``: confer :ref:`simplify` - ``resources/regions_offshore_elec{year}_s{simpl}.geojson``: confer :ref:`simplify` -- ``resources/clustermaps_elec{year}_s{simpl}.h5``: confer :ref:`simplify` - ``networks/elec{year}_s{simpl}.nc``: confer :ref:`simplify` +- ``resources/busmap_elec{year}_s{simpl}.csv``: confer :ref:`simplify` +- ``data/custom_busmap_elec{year}_s{simpl}_{clusters}.csv``: optional input Outputs ------- @@ -49,7 +50,8 @@ Outputs .. image:: ../img/regions_offshore_elec_s_X.png :scale: 33 % -- ``resources/clustermaps_elec{year}_s{simpl}_{clusters}.h5``: Mapping of buses and lines from ``networks/elec{year}_s{simpl}.nc`` to ``networks/elec{year}_s{simpl}_{clusters}.nc``; has keys ['/busmap', '/busmap_s', '/linemap', '/linemap_negative', '/linemap_positive'] +- ``resources/busmap_elec{year}_s{simpl}_{clusters}.csv``: Mapping of buses from ``networks/elec_s{simpl}.nc`` to ``networks/elec_s{simpl}_{clusters}.nc``; +- ``resources/linemap{year}_elec_s{simpl}_{clusters}.csv``: Mapping of lines from ``networks/elec_s{simpl}.nc`` to ``networks/elec_s{simpl}_{clusters}.nc``; - ``networks/elec{year}_s{simpl}_{clusters}.nc``: .. image:: ../img/elec_s_X.png @@ -255,10 +257,9 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori .apply(busmap_for_country).squeeze().rename('busmap')) -def clustering_for_n_clusters(n, n_clusters, aggregate_carriers=None, - line_length_factor=1.25, potential_mode='simple', - solver_name="cbc", algorithm="kmeans", - extended_link_costs=0, focus_weights=None): +def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carriers=None, + line_length_factor=1.25, potential_mode='simple', solver_name="cbc", + algorithm="kmeans", extended_link_costs=0, focus_weights=None): if potential_mode == 'simple': p_nom_max_strategy = np.sum @@ -267,8 +268,15 @@ def clustering_for_n_clusters(n, n_clusters, aggregate_carriers=None, else: raise AttributeError(f"potential_mode should be one of 'simple' or 'conservative' but is '{potential_mode}'") + if custom_busmap: + busmap = pd.read_csv(snakemake.input.custom_busmap, index_col=0, squeeze=True) + busmap.index = busmap.index.astype(str) + logger.info(f"Imported custom busmap from {snakemake.input.custom_busmap}") + else: + busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm) + clustering = get_clustering_from_busmap( - n, busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm), + n, busmap, bus_strategies=dict(country=_make_consense("Bus", "country")), aggregate_generators_weighted=True, aggregate_generators_carriers=aggregate_carriers, @@ -363,7 +371,8 @@ if __name__ == "__main__": return v potential_mode = consense(pd.Series([snakemake.config['renewable'][tech]['potential'] for tech in renewable_carriers])) - clustering = clustering_for_n_clusters(n, n_clusters, aggregate_carriers, + custom_busmap = snakemake.config["enable"].get("custom_busmap", False) + clustering = clustering_for_n_clusters(n, n_clusters, custom_busmap, aggregate_carriers, line_length_factor=line_length_factor, potential_mode=potential_mode, solver_name=snakemake.config['solving']['solver']['name'], diff --git a/scripts/make_summary.py b/scripts/make_summary.py index ada3fa8a..4d3e9ee5 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -71,7 +71,7 @@ opt_name = {"Store": "e", "Line" : "s", "Transformer" : "s"} def _add_indexed_rows(df, raw_index): - new_index = df.index|pd.MultiIndex.from_product(raw_index) + new_index = df.index.union(pd.MultiIndex.from_product(raw_index)) if isinstance(new_index, pd.Index): new_index = pd.MultiIndex.from_tuples(new_index) @@ -126,7 +126,7 @@ def calculate_costs(n, label, costs): marginal_costs_grouped = marginal_costs.groupby(c.df.carrier).sum() - costs = costs.reindex(costs.index|pd.MultiIndex.from_product([[c.list_name],["marginal"],marginal_costs_grouped.index])) + costs = costs.reindex(costs.index.union(pd.MultiIndex.from_product([[c.list_name],["marginal"],marginal_costs_grouped.index]))) costs.loc[idx[c.list_name,"marginal",list(marginal_costs_grouped.index)],label] = marginal_costs_grouped.values @@ -222,7 +222,7 @@ def calculate_supply(n, label, supply): #lots of sign compensation for direction and to do maximums s = (-1)**(1-int(end))*((-1)**int(end)*c.pnl["p"+end][items]).max().groupby(c.df.loc[items,'carrier']).sum() - supply = supply.reindex(supply.index|pd.MultiIndex.from_product([[i],[c.list_name],s.index])) + supply = supply.reindex(supply.index.union(pd.MultiIndex.from_product([[i],[c.list_name],s.index]))) supply.loc[idx[i,c.list_name,list(s.index)],label] = s.values return supply @@ -268,7 +268,7 @@ def calculate_supply_energy(n, label, supply_energy): s = (-1)*c.pnl["p"+end][items].sum().groupby(c.df.loc[items,'carrier']).sum() - supply_energy = supply_energy.reindex(supply_energy.index|pd.MultiIndex.from_product([[i],[c.list_name],s.index])) + supply_energy = supply_energy.reindex(supply_energy.index.union(pd.MultiIndex.from_product([[i],[c.list_name],s.index]))) supply_energy.loc[idx[i,c.list_name,list(s.index)],label] = s.values return supply_energy @@ -276,7 +276,7 @@ def calculate_supply_energy(n, label, supply_energy): def calculate_metrics(n,label,metrics): - metrics = metrics.reindex(metrics.index|pd.Index(["line_volume","line_volume_limit","line_volume_AC","line_volume_DC","line_volume_shadow","co2_shadow"])) + metrics = metrics.reindex(metrics.index.union(pd.Index(["line_volume","line_volume_limit","line_volume_AC","line_volume_DC","line_volume_shadow","co2_shadow"]))) metrics.at["line_volume_DC",label] = (n.links.length*n.links.p_nom_opt)[n.links.carrier == "DC"].sum() metrics.at["line_volume_AC",label] = (n.lines.length*n.lines.s_nom_opt).sum() @@ -298,7 +298,7 @@ def calculate_prices(n,label,prices): bus_type = pd.Series(n.buses.index.str[3:],n.buses.index).replace("","electricity") - prices = prices.reindex(prices.index|bus_type.value_counts().index) + prices = prices.reindex(prices.index.union(bus_type.value_counts().index)) logger.warning("Prices are time-averaged, not load-weighted") prices[label] = n.buses_t.marginal_price.mean().groupby(bus_type).mean() diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 84423916..e55b5de0 100755 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -164,7 +164,7 @@ def plot_map(n, ax=None, attribute='p_nom', opts={}): handler_map=make_handler_map_to_scale_circles_as_in(ax)) ax.add_artist(l2) - techs = (bus_sizes.index.levels[1]) & pd.Index(opts['vre_techs'] + opts['conv_techs'] + opts['storage_techs']) + techs = (bus_sizes.index.levels[1]).intersection(pd.Index(opts['vre_techs'] + opts['conv_techs'] + opts['storage_techs'])) handles = [] labels = [] for t in techs: diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 4aaa0d37..c8cc169c 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) def rename_techs(label): - elif "H2" in label: + if "H2" in label: label = "hydrogen storage" elif label == "solar": label = "solar PV" diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index a93f8679..6e680476 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -11,7 +11,8 @@ Prepare PyPSA network for solving according to :ref:`opts` and :ref:`ll`, such a - setting an **N-1 security margin** factor for transmission line capacities, - specifying an expansion limit on the **cost** of transmission expansion, - specifying an expansion limit on the **volume** of transmission expansion, and -- reducing the **temporal** resolution by averaging over multiple hours. +- reducing the **temporal** resolution by averaging over multiple hours + or segmenting time series into chunks of varying lengths using ``tsam``. Relevant Settings ----------------- @@ -110,7 +111,7 @@ def set_transmission_limit(n, ll_type, factor, Nyears=1): col = 'capital_cost' if ll_type == 'c' else 'length' ref = (lines_s_nom @ n.lines[col] + - n.links[links_dc_b].p_nom @ n.links[links_dc_b][col]) + n.links.loc[links_dc_b, "p_nom"] @ n.links.loc[links_dc_b, col]) costs = load_costs(Nyears, snakemake.input.tech_costs, snakemake.config['costs'], @@ -135,7 +136,7 @@ def set_transmission_limit(n, ll_type, factor, Nyears=1): def average_every_nhours(n, offset): - logger.info('Resampling the network to {}'.format(offset)) + logger.info(f"Resampling the network to {offset}") m = n.copy(with_time=False) snapshot_weightings = n.snapshot_weightings.resample(offset).sum() @@ -150,6 +151,47 @@ def average_every_nhours(n, offset): return m +def apply_time_segmentation(n, segments): + logger.info(f"Aggregating time series to {segments} segments.") + try: + import tsam.timeseriesaggregation as tsam + except: + raise ModuleNotFoundError("Optional dependency 'tsam' not found." + "Install via 'pip install tsam'") + + p_max_pu_norm = n.generators_t.p_max_pu.max() + p_max_pu = n.generators_t.p_max_pu / p_max_pu_norm + + load_norm = n.loads_t.p_set.max() + load = n.loads_t.p_set / load_norm + + inflow_norm = n.storage_units_t.inflow.max() + inflow = n.storage_units_t.inflow / inflow_norm + + raw = pd.concat([p_max_pu, load, inflow], axis=1, sort=False) + + solver_name = snakemake.config["solving"]["solver"]["name"] + + agg = tsam.TimeSeriesAggregation(raw, hoursPerPeriod=len(raw), + noTypicalPeriods=1, noSegments=int(segments), + segmentation=True, solver=solver_name) + + segmented = agg.createTypicalPeriods() + + weightings = segmented.index.get_level_values("Segment Duration") + offsets = np.insert(np.cumsum(weightings[:-1]), 0, 0) + snapshots = [n.snapshots[0] + pd.Timedelta(f"{offset}h") for offset in offsets] + + n.set_snapshots(pd.DatetimeIndex(snapshots, name='name')) + n.snapshot_weightings = pd.Series(weightings, index=snapshots, name="weightings", dtype="float64") + + segmented.index = snapshots + n.generators_t.p_max_pu = segmented[n.generators_t.p_max_pu.columns] * p_max_pu_norm + n.loads_t.p_set = segmented[n.loads_t.p_set.columns] * load_norm + n.storage_units_t.inflow = segmented[n.storage_units_t.inflow.columns] * inflow_norm + + return n + def enforce_autarky(n, only_crossborder=False): if only_crossborder: lines_rm = n.lines.loc[ @@ -162,7 +204,7 @@ def enforce_autarky(n, only_crossborder=False): ].index else: lines_rm = n.lines.index - links_rm = n.links.index + links_rm = n.links.loc[n.links.carrier=="DC"].index n.mremove("Line", lines_rm) n.mremove("Link", links_rm) @@ -191,8 +233,12 @@ if __name__ == "__main__": if m is not None: n = average_every_nhours(n, m.group(0)) break - else: - logger.info("No resampling") + + for o in opts: + m = re.match(r'^\d+seg$', o, re.IGNORECASE) + if m is not None: + n = apply_time_segmentation(n, m.group(0)[:-3]) + break for o in opts: if "Co2L" in o: @@ -208,14 +254,17 @@ if __name__ == "__main__": suptechs = map(lambda c: c.split("-", 2)[0], n.carriers.index) if oo[0].startswith(tuple(suptechs)): carrier = oo[0] - cost_factor = float(oo[1]) + # handles only p_nom_max as stores and lines have no potentials + attr_lookup = {"p": "p_nom_max", "c": "capital_cost"} + attr = attr_lookup[oo[1][0]] + factor = float(oo[1][1:]) if carrier == "AC": # lines do not have carrier - n.lines.capital_cost *= cost_factor + n.lines[attr] *= factor else: - comps = {"Generator", "Link", "StorageUnit"} + comps = {"Generator", "Link", "StorageUnit", "Store"} for c in n.iterate_components(comps): sel = c.df.carrier.str.contains(carrier) - c.df.loc[sel,"capital_cost"] *= cost_factor + c.df.loc[sel,attr] *= factor if 'Ep' in opts: add_emission_prices(n) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 780a794d..54390529 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -62,7 +62,7 @@ Outputs .. image:: ../img/regions_offshore_elec_s .png :scale: 33 % -- ``resources/clustermaps_elec{year}_s{simpl}.h5``: Mapping of buses from ``networks/elec.nc`` to ``networks/elec{year}_s{simpl}.nc``; has keys ['/busmap_s'] +- ``resources/busmap_elec{year}_s{simpl}.h5``: Mapping of buses from ``networks/elec.nc`` to ``networks/elec{year}_s{simpl}.nc``; has keys ['/busmap_s'] - ``networks/elec{year}_s{simpl}.nc``: .. image:: ../img/elec_s.png @@ -322,7 +322,7 @@ def remove_stubs(n): def cluster(n, n_clusters): - logger.info("Clustering to {} buses".format(n_clusters)) + logger.info(f"Clustering to {n_clusters} buses") renewable_carriers = pd.Index([tech for tech in n.generators.carrier.unique() @@ -336,7 +336,7 @@ def cluster(n, n_clusters): potential_mode = (consense(pd.Series([snakemake.config['renewable'][tech]['potential'] for tech in renewable_carriers])) if len(renewable_carriers) > 0 else 'conservative') - clustering = clustering_for_n_clusters(n, n_clusters, potential_mode=potential_mode, + clustering = clustering_for_n_clusters(n, n_clusters, custom_busmap=False, potential_mode=potential_mode, solver_name=snakemake.config['solving']['solver']['name']) return clustering.network, clustering.busmap diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 6e733cd8..a3e77181 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -10,10 +10,6 @@ Relevant Settings .. code:: yaml - (electricity:) - (BAU_mincapacities:) - (SAFE_reservemargin:) - solving: tmpdir: options: @@ -28,10 +24,6 @@ Relevant Settings track_iterations: solver: name: - (solveroptions): - - (plotting:) - (conv_techs:) .. seealso:: Documentation of the configuration file ``config.yaml`` at diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 2774be8a..4c6c53f4 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: CC0-1.0 -version: 0.2.0 +version: 0.3.0 tutorial: true logging: level: INFO @@ -31,6 +31,7 @@ enable: retrieve_cutout: true build_natura_raster: false retrieve_natura_raster: true + custom_busmap: false electricity: voltages: [220., 300., 380.] @@ -146,6 +147,11 @@ transformers: type: '' load: + url: https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv + power_statistics: True # only for files from <2019; set false in order to get ENTSOE transparency data + interpolate_limit: 3 # data gaps up until this size are interpolated linearly + time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from + manual_adjustments: true # false scaling_factor: 1.0 fallback_year: 2013