finalise ukraine_hackathon branch for merge

config/config.default.yaml

@@ -38,7 +38,7 @@ scenario:
   ll:
   - v1.5
   clusters:
-  - 39
+  - 37
   - 128
   - 256
   - 512
@@ -54,7 +54,7 @@ scenario:
   - 2050
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#countries
-countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MD', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK', 'UA']
+countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK']
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#snapshots
 snapshots:
@@ -90,8 +90,8 @@ co2_budget:
 electricity:
   voltages: [220., 300., 380., 500., 750.]
   gaslimit: false
-  co2limit: 9.59e+7
-  co2base: 1.918e+9
+  co2limit: 7.75e+7
+  co2base: 1.487e+9
   agg_p_nom_limits: data/agg_p_nom_minmax.csv
 
   operational_reserve:

config/config.entsoe-all.yaml

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+run:
+  name: "entsoe-all"
+  disable_progressbar: true
+  shared_resources: false
+  shared_cutouts: true
+
+scenario:
+  simpl:
+  - ''
+  ll:
+  - vopt
+  clusters:
+  - 39
+  - 128
+  - 256
+  opts:
+  - ''
+  sector_opts:
+  - ''
+  planning_horizons:
+  - ''
+
+# TODO add Turkey (TR)
+countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MD', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK', 'UA']
+
+electricity:
+  custom_powerplants: true
+  co2limit: 9.59e+7
+  co2base: 1.918e+9
+
+lines:
+  reconnect_crimea: true
+
+enable:
+  retrieve: true
+  retrieve_databundle: true
+  retrieve_sector_databundle: false
+  retrieve_cost_data: true
+  retrieve_cutout: true

config/config.ua.yaml

@@ -1,27 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 The PyPSA-Eur Authors
-#
-# SPDX-License-Identifier: CC0-1.0
-
-scenario:
-  simpl:
-  - ''
-  ll:
-  - v1.5
-  clusters:
-  - 6
-  opts:
-  - '24H'
-  sector_opts:
-  - ''
-  planning_horizons:
-  - 2050
-
-countries:
-- UA
-
-enable:
-  retrieve: true
-  retrieve_databundle: true
-  retrieve_sector_databundle: false
-  retrieve_cost_data: true
-  retrieve_cutout: false

data/EIA_hydro_generation_2000_2014.csv

@@ -1,51 +0,0 @@
-International_data,,,,,,,,,,,,,,,,,
-http://www.eia.gov/beta/international/data/browser/#/?pa=000000000000000000000000000000g&c=1028i008006gg6168g80a4k000e0ag00gg0004g800ho00g8&ct=0&ug=8&tl_id=2-A&vs=INTL.33-12-ALB-BKWH.A&cy=2014&vo=0&v=H&start=2000&end=2014,,,,,,,,,,,,,,,,,
-Wed Mar 01 2017 17:25:47 GMT+0100 (CET),,,,,,,,,,,,,,,,,
-"Source: U.S. Energy Information Administration, Manually complemented with data for Ukraine, Moldova on 2022-03-05 10:09 UTC+1 from https://www.eia.gov/international/api/series_data/csv?location32=ruvvvvvfvtvnvv1vrvvvvfvvvvvvfvvvou20evvvvvvvvvvnvvvs0008&products32=000000000000000000000000000000g&frequency=A&unit=1&start=315532800000&end=1577836800000&generated=03-05-2022_10-09-49",,,,,,,,,,,,,,,,,
-,,,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014
-,Electricity,Billion Kwh,,,,,,,,,,,,,,,
-,Generation,,,,,,,,,,,,,,,,
-,Renewables,,,,,,,,,,,,,,,,
-,Hydroelectricity,,,,,,,,,,,,,,,,
-,Albania,Billion Kwh,4.5,3.5,3.5,5.1,5.4,5.319,4.951,2.76,3.759,5.201,7.673,4.036,4.725,6.959,4.726
-,Austria,Billion Kwh,41,40,40,33,36,36.31,35.48,36.732,37.969,40.487,37.979,33.901,43.376,41.595,40.592
-,Belgium,Billion Kwh,0.5,0.4,0.4,0.2,0.3,0.285,0.355,0.385,0.406,0.325,0.309,0.194,0.353,0.376,0.271
-,Bosnia and Herzegovina,Billion Kwh,5,5.1,5.2,4.5,5.9,5.938,5.798,3.961,4.818,6.177,7.946,4.343,4.173,7.164,5.876
-,Bulgaria,Billion Kwh,2.6,1.7,2.2,3,3.1,4.294,4.196,2.845,2.796,3.435,5.006,2.888,3.194,4.039,4.559
-,Croatia,Billion Kwh,5.8,6.5,5.3,4.8,6.9,6.27,5.94,4.194,5.164,6.663,9.035,4.983,4.789,8.536,8.917
-,Cyprus,Billion Kwh,,,,,,,,,,,,,,,
-,Czech Republic,Billion Kwh,1.7,2,2.5,1.4,2,2.356,2.525,2.068,2.004,2.405,2.761,1.943,2.108,2.707,1.89
-,Denmark,Billion Kwh,0.03,0.028,0.032,0.021,0.027,0.023,0.023,0.028,0.026,0.019,0.021,0.017,0.017,0.013,0.015
-,Estonia,Billion Kwh,0.005,0.007,0.006,0.013,0.022,0.022,0.014,0.021,0.028,0.032,0.027,0.03,0.042,0.026,0.027
-,Faroe Islands,Billion Kwh,0.075262,0.075195,0.095535,0.08483,0.093443,0.097986,0.099934,0.103407,0.094921,0.091482,0.06676,0.092,0.099,0.091,0.121
-,Finland,Billion Kwh,15,13,11,9.5,15,13.646,11.379,14.035,16.941,12.559,12.793,12.321,16.69,12.71,13.263
-,France,Billion Kwh,66,74,60,59,59,50.965,55.741,57.029,63.017,56.428,62.086,44.343,58.13,70.009,62.201
-,Germany,Billion Kwh,22,23,23,19,21,19.442,19.808,20.957,20.239,18.841,20.743,17.494,21.537,22.768,19.39
-,Gibraltar,Billion Kwh,,,,,,,,,,,,,,,
-,Greece,Billion Kwh,3.7,2.1,2.8,4.7,4.6,4.967,5.806,2.565,3.279,5.32,7.385,3.971,4.359,6.283,4.431
-,Hungary,Billion Kwh,0.2,0.2,0.2,0.2,0.2,0.2,0.184,0.208,0.211,0.226,0.186,0.22,0.211,0.211,0.299
-,Iceland,Billion Kwh,6.3,6.5,6.9,7,7.1,6.949,7.22,8.31,12.303,12.156,12.466,12.382,12.214,12.734,12.744
-,Ireland,Billion Kwh,0.8,0.6,0.9,0.6,0.6,0.625,0.717,0.66,0.959,0.893,0.593,0.7,0.794,0.593,0.702
-,Italy,Billion Kwh,44,46,39,33,42,35.706,36.624,32.488,41.207,48.647,50.605,45.365,41.456,52.246,57.96
-,Kosovo,Billion Kwh,--,--,--,--,--,--,--,--,0.075,0.119,0.154,0.104,0.095,0.142,0.149
-,Latvia,Billion Kwh,2.8,2.8,2.4,2.2,3.1,3.293,2.671,2.706,3.078,3.422,3.485,2.858,3.67,2.883,1.974
-,Lithuania,Billion Kwh,0.3,0.3,0.4,0.3,0.4,0.446193,0.393,0.417,0.398,0.42,0.535,0.475,0.419,0.516,0.395
-,Luxembourg,Billion Kwh,0.1,0.1,0.098,0.078,0.1,0.093,0.11,0.116,0.131,0.105,0.107,0.061,0.098,0.118,0.107
-,Macedonia,Billion Kwh,1.2,0.6,0.7,1.4,1.5,1.477,1.634,1,0.832,1.257,2.407,1.419,1.031,1.568,1.195
-,Malta,Billion Kwh,,,,,,,,,,,,,,,
-,Montenegro,Billion Kwh,--,--,--,--,--,--,1.733,1.271,1.524,2.05,2.723,1.192,1.462,2.479,1.734
-,Netherlands,Billion Kwh,0.1,0.1,0.1,0.071,0.094,0.087,0.105,0.106,0.101,0.097,0.104,0.056,0.103,0.113,0.111
-,Norway,Billion Kwh,140,119,128,104,108,134.331,118.175,132.319,137.654,124.03,115.583,119.087,140.323,126.872,135.535
-,Poland,Billion Kwh,2.1,2.3,2.3,1.7,2.1,2.179,2.022,2.328,2.13,2.351,2.891,2.308,2.017,2.415,2.161
-,Portugal,Billion Kwh,11,14,7.7,16,9.8,4.684,10.892,9.991,6.73,8.201,15.987,11.425,5.566,13.593,15.413
-,Romania,Billion Kwh,15,15,16,13,16,20.005,18.172,15.806,17.023,15.379,19.684,14.581,11.945,14.807,18.618
-,Serbia,Billion Kwh,--,--,--,--,--,--,10.855,9.937,9.468,10.436,11.772,8.58,9.193,10.101,10.893
-,Slovakia,Billion Kwh,4.6,4.9,5.2,3.4,4.1,4.592,4.355,4.406,4,4.324,5.202,3.74,4.062,4.8,4.167
-,Slovenia,Billion Kwh,3.8,3.7,3.3,2.9,4,3.426,3.555,3.233,3.978,4.666,4.473,3.526,3.86,4.583,6.031
-,Spain,Billion Kwh,29,41,23,41,31,18.209,25.699,27.036,23.13,26.147,41.881,30.29,20.34,36.496,38.777
-,Sweden,Billion Kwh,78,78,66,53,60,72.075,61.106,65.497,68.378,65.193,65.734,65.77,78.143,60.747,63.126
-,Switzerland,Billion Kwh,36,41,35,34,33,30.914,30.649,34.898,35.676,35.366,35.7,32.066,38.214,38.076,37.656
-,Turkey,Billion Kwh,31,24,33,35,46,39.165,43.802,35.492,32.937,35.598,51.278,51.815,57.286,58.826,40.239
-,United Kingdom,Billion Kwh,5,4,4.7,3.2,4.8,4.873,4.547,5.026,5.094,5.178,3.529,5.624,5.232,4.655,5.826
-,Moldova,Billion Kwh,0.392,0.359,0.348,0.358,0.35,0.359,0.365,0.354,0.385,0.354,0.403,0.348,0.266,0.311,0.317
-,Ukraine,Billion Kwh,11.161,11.912,9.531,9.146,11.635,12.239,12.757,10.042,11.397,11.817,13.02,10.837,10.374,13.663,8.393

doc/configtables/licenses.csv

@@ -5,7 +5,7 @@
 "naturalearth/*",,,,,http://www.naturalearthdata.com/about/terms-of-use/
 "NUTS_2013 _60M_SH/*","x","x",,"x",https://ec.europa.eu/eurostat/web/gisco/geodata/reference-data/administrative-units-statistical-units
 "cantons.csv","x",,"x",,https://en.wikipedia.org/wiki/Data_codes_for_Switzerland
-"EIA_hydro_generation _2000_2014.csv","x",,,,https://www.eia.gov/about/copyrights_reuse.php
+"eia_hydro_annual_generation.csv","x",,,,https://www.eia.gov/about/copyrights_reuse.php
 "GEBCO_2014_2D.nc","x",,,,https://www.gebco.net/data_and_products/gridded_bathymetry_data/documents/gebco_2014_historic.pdf
 "hydro_capacities.csv","x",,,,
 "je-e-21.03.02.xls","x","x",,,https://www.bfs.admin.ch/bfs/en/home/fso/swiss-federal-statistical-office/terms-of-use.html

doc/configtables/lines.csv

@@ -5,6 +5,7 @@ s_nom_max,MW,"float","Global upper limit for the maximum capacity of each extend
 max_extension,MW,"float","Upper limit for the extended capacity of each extendable line."
 length_factor,--,float,"Correction factor to account for the fact that buses are *not* connected by lines through air-line distance."
 under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}","Specifies how to handle lines which are currently under construction."
+reconnect_crimea,--,"true or false","Whether to reconnect Crimea to the Ukrainian grid"
 dynamic_line_rating,,,
 -- activate,bool,"true or false","Whether to take dynamic line rating into account"
 -- cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."

doc/release_notes.rst

@@ -35,6 +35,16 @@ Upcoming Release
 
 * The ``mock_snakemake`` function can now be used with a Snakefile from a different directory using the new ``root_dir`` argument.
 
+* Merged option to extend geographical scope to Ukraine and Moldova. These
+  countries are excluded by default and is currently constrained to power-sector
+  only parts of the workflow. A special config file
+  `config/config.entsoe-all.yaml` was added as an example to run the workflow
+  with all ENTSO-E member countries (including observer members like Ukraine and
+  Moldova). Moldova can currently only be included in conjunction with Ukraine
+  due to the absence of demand data. The Crimean power system is manually
+  reconnected to the main Ukrainian grid with the configuration option
+  `reconnect_crimea`.
+
 
 **Bugs and Compatibility**
 

doc/retrieve.rst

@@ -22,11 +22,11 @@ Rule ``retrieve_databundle``
 Rule ``retrieve_cutout``
 ============================
 
-.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.6350001.svg
-   :target: https://doi.org/10.5281/zenodo.6350001
+.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.6382570.svg
+   :target: https://doi.org/10.5281/zenodo.6382570
 
 Cutouts are spatio-temporal subsets of the European weather data from the `ECMWF ERA5 <https://software.ecmwf.int/wiki/display/CKB/ERA5+data+documentation>`_ reanalysis dataset and the `CMSAF SARAH-2 <https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=SARAH_V002>`_ solar surface radiation dataset for the year 2013.
-They have been prepared by and are for use with the `atlite <https://github.com/PyPSA/atlite>`_ tool. You can either generate them yourself using the ``build_cutouts`` rule or retrieve them directly from `zenodo <https://doi.org/10.5281/zenodo.6350001>`__ through the rule ``retrieve_cutout``.
+They have been prepared by and are for use with the `atlite <https://github.com/PyPSA/atlite>`_ tool. You can either generate them yourself using the ``build_cutouts`` rule or retrieve them directly from `zenodo <https://doi.org/10.5281/zenodo.6382570>`__ through the rule ``retrieve_cutout``.
 The :ref:`tutorial` uses a smaller cutout than required for the full model (30 MB), which is also automatically downloaded.
 
 .. note::

envs/environment.yaml

@@ -14,8 +14,6 @@ dependencies:
 - pypsa==0.25.2
 - linopy
 - dask
-- jupyter
-- nbconvert
 
   # Dependencies of the workflow itself
 - xlrd

scripts/build_electricity_demand.py

@@ -306,15 +306,17 @@ if __name__ == "__main__":
 
     load = load_timeseries(snakemake.input[0], years, countries, powerstatistics)
 
-    # attach load of UA (best data only for entsoe transparency)
-    load_ua = load_timeseries(snakemake.input[0], "2018", ["UA"], False)
-    snapshot_year = str(snapshots.year.unique().item())
-    time_diff = pd.Timestamp("2018") - pd.Timestamp(snapshot_year)
-    load_ua.index -= time_diff  # hack indices (currently, UA is manually set to 2018)
-    load["UA"] = load_ua
-    # attach load of MD (no time-series available, use 2020-totals and distribute according to UA):
-    # https://www.iea.org/data-and-statistics/data-browser/?country=MOLDOVA&fuel=Energy%20consumption&indicator=TotElecCons
-    load["MD"] = 6.2e6 * (load_ua / load_ua.sum())
+    if "UA" in countries:
+        # attach load of UA (best data only for entsoe transparency)
+        load_ua = load_timeseries(snakemake.input[0], "2018", ["UA"], False)
+        snapshot_year = str(snapshots.year.unique().item())
+        time_diff = pd.Timestamp("2018") - pd.Timestamp(snapshot_year)
+        load_ua.index -= time_diff  # hack indices (currently, UA is manually set to 2018)
+        load["UA"] = load_ua
+        # attach load of MD (no time-series available, use 2020-totals and distribute according to UA):
+        # https://www.iea.org/data-and-statistics/data-browser/?country=MOLDOVA&fuel=Energy%20consumption&indicator=TotElecCons
+        if "MD" in countries:
+            load["MD"] = 6.2e6 * (load_ua / load_ua.sum())
 
     if snakemake.params.load["manual_adjustments"]:
         load = manual_adjustment(load, snakemake.input[0], powerstatistics, countries)

scripts/build_hydro_profile.py

@@ -26,7 +26,7 @@ Relevant Settings
 Inputs
 ------
 
-- ``data/bundle/EIA_hydro_generation_2000_2014.csv``: Hydroelectricity net generation per country and year (`EIA <https://www.eia.gov/beta/international/data/browser/#/?pa=000000000000000000000000000000g&c=1028i008006gg6168g80a4k000e0ag00gg0004g800ho00g8&ct=0&ug=8&tl_id=2-A&vs=INTL.33-12-ALB-BKWH.A&cy=2014&vo=0&v=H&start=2000&end=2016>`_)
+- ``data/bundle/eia_hydro_annual_generation.csv``: Hydroelectricity net generation per country and year (`EIA <https://www.eia.gov/beta/international/data/browser/#/?pa=000000000000000000000000000000g&c=1028i008006gg6168g80a4k000e0ag00gg0004g800ho00g8&ct=0&ug=8&tl_id=2-A&vs=INTL.33-12-ALB-BKWH.A&cy=2014&vo=0&v=H&start=2000&end=2016>`_)
 
     .. image:: img/hydrogeneration.png
         :scale: 33 %