Merge branch 'master' into fneum/ariadne

.gitignore

@@ -46,6 +46,7 @@ data/costs_*.csv
 
 dask-worker-space/
 publications.jrc.ec.europa.eu/
+d1gam3xoknrgr2.cloudfront.net/
 
 *.org
 

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
 
   # Sort package imports alphabetically
 - repo: https://github.com/PyCQA/isort
-  rev: 5.12.0
+  rev: 5.13.2
   hooks:
   - id: isort
     args: ["--profile", "black", "--filter-files"]
@@ -45,13 +45,13 @@ repos:
     args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]
 
 - repo: https://github.com/keewis/blackdoc
-  rev: v0.3.8
+  rev: v0.3.9
   hooks:
   - id: blackdoc
 
   # Formatting with "black" coding style
-- repo: https://github.com/psf/black
-  rev: 23.9.1
+- repo: https://github.com/psf/black-pre-commit-mirror
+  rev: 23.12.1
   hooks:
       # Format Python files
   - id: black
@@ -67,7 +67,7 @@ repos:
 
   # Do YAML formatting (before the linter checks it for misses)
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.11.0
+  rev: v2.12.0
   hooks:
   - id: pretty-format-yaml
     args: [--autofix, --indent, "2", --preserve-quotes]

README.md

@@ -61,9 +61,9 @@ The dataset consists of:
 
 - A grid model based on a modified [GridKit](https://github.com/bdw/GridKit)
   extraction of the [ENTSO-E Transmission System
-  Map](https://www.entsoe.eu/data/map/). The grid model contains 6763 lines
+  Map](https://www.entsoe.eu/data/map/). The grid model contains 7072 lines
   (alternating current lines at and above 220kV voltage level and all high
-  voltage direct current lines) and 3642 substations.
+  voltage direct current lines) and 3803 substations.
 - The open power plant database
   [powerplantmatching](https://github.com/FRESNA/powerplantmatching).
 - Electrical demand time series from the
@@ -103,6 +103,6 @@ We strongly welcome anyone interested in contributing to this project. If you ha
 # Licence
 
 The code in PyPSA-Eur is released as free software under the
-[MIT License](https://opensource.org/licenses/MIT), see `LICENSE.txt`.
+[MIT License](https://opensource.org/licenses/MIT), see [`doc/licenses.rst`](doc/licenses.rst).
 However, different licenses and terms of use may apply to the various
 input data.

Snakefile

@@ -14,7 +14,7 @@ from snakemake.utils import min_version
 min_version("7.7")
 
 
-if not exists("config/config.yaml"):
+if not exists("config/config.yaml") and exists("config/config.default.yaml"):
     copyfile("config/config.default.yaml", "config/config.yaml")
 
 

config/config.default.yaml

@@ -86,7 +86,7 @@ co2_budget:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity
 electricity:
-  voltages: [220., 300., 380.]
+  voltages: [220., 300., 380., 500., 750.]
   gaslimit: false
   co2limit: 7.75e+7
   co2base: 1.487e+9
@@ -135,14 +135,14 @@ atlite:
       # module: era5
     europe-2013-era5:
       module: era5 # in priority order
-      x: [-12., 35.]
+      x: [-12., 42.]
       y: [33., 72]
       dx: 0.3
       dy: 0.3
       time: ['2013', '2013']
     europe-2013-sarah:
       module: [sarah, era5] # in priority order
-      x: [-12., 45.]
+      x: [-12., 42.]
       y: [33., 65]
       dx: 0.2
       dy: 0.2
@@ -158,45 +158,51 @@ renewable:
     resource:
       method: wind
       turbine: Vestas_V112_3MW
+      add_cutout_windspeed: true
     capacity_per_sqkm: 3
     # correction_factor: 0.93
     corine:
       grid_codes: [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32]
       distance: 1000
       distance_grid_codes: [1, 2, 3, 4, 5, 6]
+    luisa: false
+      # grid_codes: [1111, 1121, 1122, 1123, 1130, 1210, 1221, 1222, 1230, 1241, 1242]
+      # distance: 1000
+      # distance_grid_codes: [1111, 1121, 1122, 1123, 1130, 1210, 1221, 1222, 1230, 1241, 1242]
     natura: true
     excluder_resolution: 100
-    potential: simple # or conservative
     clip_p_max_pu: 1.e-2
   offwind-ac:
     cutout: europe-2013-era5
     resource:
       method: wind
-      turbine: NREL_ReferenceTurbine_5MW_offshore
+      turbine: NREL_ReferenceTurbine_2020ATB_5.5MW
+      add_cutout_windspeed: true
     capacity_per_sqkm: 2
     correction_factor: 0.8855
     corine: [44, 255]
+    luisa: false # [0, 5230]
     natura: true
     ship_threshold: 400
     max_depth: 50
     max_shore_distance: 30000
     excluder_resolution: 200
-    potential: simple # or conservative
     clip_p_max_pu: 1.e-2
   offwind-dc:
     cutout: europe-2013-era5
     resource:
       method: wind
-      turbine: NREL_ReferenceTurbine_5MW_offshore
+      turbine: NREL_ReferenceTurbine_2020ATB_5.5MW
+      add_cutout_windspeed: true
     capacity_per_sqkm: 2
     correction_factor: 0.8855
     corine: [44, 255]
+    luisa: false # [0, 5230]
     natura: true
     ship_threshold: 400
     max_depth: 50
     min_shore_distance: 30000
     excluder_resolution: 200
-    potential: simple # or conservative
     clip_p_max_pu: 1.e-2
   solar:
     cutout: europe-2013-sarah
@@ -209,9 +215,9 @@ renewable:
     capacity_per_sqkm: 1.7
     # correction_factor: 0.854337
     corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32]
+    luisa: false # [1111, 1121, 1122, 1123, 1130, 1210, 1221, 1222, 1230, 1241, 1242, 1310, 1320, 1330, 1410, 1421, 1422, 2110, 2120, 2130, 2210, 2220, 2230, 2310, 2410, 2420, 3210, 3320, 3330]
     natura: true
     excluder_resolution: 100
-    potential: simple # or conservative
     clip_p_max_pu: 1.e-2
   hydro:
     cutout: europe-2013-era5
@@ -235,10 +241,13 @@ lines:
     220.: "Al/St 240/40 2-bundle 220.0"
     300.: "Al/St 240/40 3-bundle 300.0"
     380.: "Al/St 240/40 4-bundle 380.0"
+    500.: "Al/St 240/40 4-bundle 380.0"
+    750.: "Al/St 560/50 4-bundle 750.0"
   s_max_pu: 0.7
   s_nom_max: .inf
   max_extension: .inf
   length_factor: 1.25
+  reconnect_crimea: true
   under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
   dynamic_line_rating:
     activate: false
@@ -445,10 +454,10 @@ sector:
   solar_cf_correction: 0.788457  # =  >>> 1/1.2683
   marginal_cost_storage: 0. #1e-4
   methanation: true
-  helmeth: false
   coal_cc: false
   dac: true
   co2_vent: false
+  central_heat_vent: false
   allam_cycle: false
   hydrogen_fuel_cell: true
   hydrogen_turbine: false
@@ -475,14 +484,28 @@ sector:
   - nearshore    # within 50 km of sea
     # - offshore
   ammonia: false
-  min_part_load_fischer_tropsch: 0.9
-  min_part_load_methanolisation: 0.5
+  min_part_load_fischer_tropsch: 0.7
+  min_part_load_methanolisation: 0.3
+  min_part_load_methanation: 0.3
   use_fischer_tropsch_waste_heat: true
+  use_haber_bosch_waste_heat: true
+  use_methanolisation_waste_heat: true
+  use_methanation_waste_heat: true
   use_fuel_cell_waste_heat: true
-  use_electrolysis_waste_heat: false
+  use_electrolysis_waste_heat: true
   electricity_distribution_grid: true
   electricity_distribution_grid_cost_factor: 1.0
   electricity_grid_connection: true
+  transmission_efficiency:
+    DC:
+      efficiency_static: 0.98
+      efficiency_per_1000km: 0.977
+    H2 pipeline:
+      efficiency_per_1000km: 1 # 0.979
+      compression_per_1000km: 0.019
+    gas pipeline:
+      efficiency_per_1000km: 1 #0.977
+      compression_per_1000km: 0.01
   H2_network: true
   gas_network: false
   H2_retrofit: false
@@ -492,6 +515,7 @@ sector:
   gas_distribution_grid_cost_factor: 1.0
   biomass_spatial: false
   biomass_transport: false
+  biogas_upgrading_cc: false
   conventional_generation:
     OCGT: gas
   biomass_to_liquid: false
@@ -542,8 +566,8 @@ industry:
   MWh_NH3_per_tNH3: 5.166
   MWh_CH4_per_tNH3_SMR: 10.8
   MWh_elec_per_tNH3_SMR: 0.7
-  MWh_H2_per_tNH3_electrolysis: 6.5
-  MWh_elec_per_tNH3_electrolysis: 1.17
+  MWh_H2_per_tNH3_electrolysis: 5.93
+  MWh_elec_per_tNH3_electrolysis: 0.2473
   MWh_NH3_per_MWh_H2_cracker: 1.46 # https://github.com/euronion/trace/blob/44a5ff8401762edbef80eff9cfe5a47c8d3c8be4/data/efficiencies.csv
   NH3_process_emissions: 24.5
   petrochemical_process_emissions: 25.5
@@ -596,6 +620,7 @@ costs:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering
 clustering:
+  focus_weights: false
   simplify_network:
     to_substations: false
     algorithm: kmeans # choose from: [hac, kmeans]
@@ -624,6 +649,7 @@ solving:
     skip_iterations: true
     rolling_horizon: false
     seed: 123
+    custom_extra_functionality: "../data/custom_extra_functionality.py"
     # options that go into the optimize function
     track_iterations: false
     min_iterations: 4
@@ -721,6 +747,7 @@ plotting:
     H2: "Hydrogen Storage"
     lines: "Transmission Lines"
     ror: "Run of River"
+    load: "Load Shedding"
     ac: "AC"
     dc: "DC"
 
@@ -771,6 +798,7 @@ plotting:
     fossil gas: '#e05b09'
     natural gas: '#e05b09'
     biogas to gas: '#e36311'
+    biogas to gas CC: '#e51245'
     CCGT: '#a85522'
     CCGT marginal: '#a85522'
     allam: '#B98F76'
@@ -872,12 +900,14 @@ plotting:
     # heat demand
     Heat load: '#cc1f1f'
     heat: '#cc1f1f'
+    heat vent: '#aa3344'
     heat demand: '#cc1f1f'
     rural heat: '#ff5c5c'
     residential rural heat: '#ff7c7c'
     services rural heat: '#ff9c9c'
     central heat: '#cc1f1f'
     urban central heat: '#d15959'
+    urban central heat vent: '#a74747'
     decentral heat: '#750606'
     residential urban decentral heat: '#a33c3c'
     services urban decentral heat: '#cc1f1f'
@@ -940,7 +970,6 @@ plotting:
     Sabatier: '#9850ad'
     methanation: '#c44ce6'
     methane: '#c44ce6'
-    helmeth: '#e899ff'
     # synfuels
     Fischer-Tropsch: '#25c49a'
     liquid: '#25c49a'
@@ -986,3 +1015,4 @@ plotting:
     DC: "#8a1caf"
     DC-DC: "#8a1caf"
     DC link: "#8a1caf"
+    load: "#dd2e23"

config/config.entsoe-all.yaml

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+run:
+  name: "entsoe-all"
+  disable_progressbar: true
+  shared_resources: false
+  shared_cutouts: true
+
+scenario:
+  simpl:
+  - ''
+  ll:
+  - vopt
+  clusters:
+  - 39
+  - 128
+  - 256
+  opts:
+  - ''
+  sector_opts:
+  - ''
+  planning_horizons:
+  - ''
+
+# TODO add Turkey (TR)
+countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MD', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK', 'UA']
+
+electricity:
+  custom_powerplants: true
+  co2limit: 9.59e+7
+  co2base: 1.918e+9
+
+lines:
+  reconnect_crimea: true
+
+enable:
+  retrieve: true
+  retrieve_databundle: true
+  retrieve_sector_databundle: false
+  retrieve_cost_data: true
+  retrieve_cutout: true

config/test/config.myopic.yaml

@@ -30,6 +30,9 @@ snapshots:
   start: "2013-03-01"
   end: "2013-03-08"
 
+sector:
+  central_heat_vent: true
+
 electricity:
   co2limit: 100.e+6
 

config/test/config.perfect.yaml

@@ -44,6 +44,7 @@ electricity:
 sector:
   min_part_load_fischer_tropsch: 0
   min_part_load_methanolisation: 0
+
 atlite:
   default_cutout: be-03-2013-era5
   cutouts:

data/GDP_PPP_30arcsec_v3_mapped_default.csv

@@ -0,0 +1,151 @@
+name,GDP_PPP,country
+3140,632728.0438507323,MD
+3139,806541.9318093687,MD
+3142,1392454.6690911907,MD
+3152,897871.2903553953,MD
+3246,645554.8588933202,MD
+7049,1150156.4449477682,MD
+1924,162285.16792916053,UA
+1970,751970.6071848695,UA
+2974,368873.75840156944,UA
+2977,294847.85539198935,UA
+2979,197988.13680768458,UA
+2980,301371.2491126519,UA
+3031,56925.21878805953,UA
+3032,139395.18279351242,UA
+3033,145377.8061037629,UA
+3035,52282.83655208812,UA
+3036,497950.25890516065,UA
+3037,1183293.1987702171,UA
+3038,255005.98207636533,UA
+3039,224711.50098325178,UA
+3040,342959.943226467,UA
+3044,69119.31486955672,UA
+3045,246273.65986119965,UA
+3047,146742.08407299497,UA
+3049,107265.7028733467,UA
+3050,1126147.985259493,UA
+3051,69833.56303043803,UA
+3052,67230.88206577855,UA
+3053,27019.224685201345,UA
+3054,260571.47337292184,UA
+3055,88760.94152915622,UA
+3056,101368.26196568517,UA
+3058,55752.92329667119,UA
+3059,89024.37880630122,UA
+3062,358411.291265149,UA
+3064,75081.64142862396,UA
+3065,158101.42949135564,UA
+3066,83763.89576442329,UA
+3068,173474.51218344545,UA
+3069,60327.01572375589,UA
+3070,18073.687271955278,UA
+3071,249069.43314695224,UA
+3072,220707.35700825177,UA
+3073,61342.30137462664,UA
+3074,254235.98867635374,UA
+3077,769558.9832370486,UA
+3078,132674.2315809836,UA
+3079,1388517.1478032232,UA
+3080,1861003.8718246964,UA
+3082,140123.73854745473,UA
+3083,834887.5595419679,UA
+3084,1910795.5590558557,UA
+3086,93828.36549170096,UA
+3088,347197.65113392205,UA
+3089,3754718.141734592,UA
+3090,521912.69768585655,UA
+3093,232818.05269714879,UA
+3095,435376.20361377904,UA
+3099,345596.5288937008,UA
+3100,175689.10947424968,UA
+3105,538438.9311459162,UA
+3107,88096.86032871014,UA
+3108,79847.68447063807,UA
+3109,348504.73449373,UA
+3144,71657.0165675802,UA
+3146,80342.05037424155,UA
+3158,74465.12922576343,UA
+3164,3102112.2672631275,UA
+3165,65215.04081671433,UA
+3166,413924.2225725632,UA
+3167,135060.0056434935,UA
+3168,54980.442979330146,UA
+3170,29584.879122227037,UA
+3171,142780.68163047134,UA
+3172,40436.63814695243,UA
+3173,1253342.1790126422,UA
+3174,173842.03139155387,UA
+3176,65699.76352408895,UA
+3177,143591.75419817626,UA
+3178,56434.04525832523,UA
+3179,389996.1670051216,UA
+3180,138452.84503524794,UA
+3181,67402.59500436619,UA
+3184,51204.293695376415,UA
+3185,46867.82356528432,UA
+3186,103892.35612417295,UA
+3187,193668.91476930346,UA
+3189,54584.176457692694,UA
+3190,219077.64942830536,UA
+3197,88516.52699983507,UA
+3198,298166.8272673622,UA
+3199,61334.952541812374,UA
+3229,175692.61136747137,UA
+3230,106722.62773321665,UA
+3236,61542.06264321315,UA
+3241,83752.90489164277,UA
+4301,48419.52825967164,UA
+4305,147759.74280349456,UA
+4306,53156.905740992224,UA
+4315,218025.78516351627,UA
+4317,155240.40554731718,UA
+4318,1342144.2459407183,UA
+4319,91669.1449633853,UA
+4321,85852.49282415409,UA
+4347,67938.7698430624,UA
+4357,20064.979012172935,UA
+4360,47840.51245168512,UA
+4361,55580.924388032574,UA
+4362,165753.82588729708,UA
+4363,46390.2448142152,UA
+4365,96265.47592938849,UA
+4366,272003.25510057947,UA
+4367,80878.50229245829,UA
+4370,330072.35444044066,UA
+4371,7707066.181975477,UA
+4373,2019766.7891575783,UA
+4374,985354.331818515,UA
+4377,230805.08833664874,UA
+4382,125670.67125287943,UA
+4383,46914.065511740075,UA
+4384,48020.804310510954,UA
+4385,55612.34707641123,UA
+4387,74558.3475791577,UA
+4388,245243.33449409154,UA
+4389,95696.56767732685,UA
+4391,251085.7523045193,UA
+4401,66375.82996856027,UA
+4403,111954.41038437477,UA
+4405,46911.68560148837,UA
+4408,150782.51691456966,UA
+4409,112776.7399582134,UA
+4410,153076.56860965435,UA
+4412,192629.31238456024,UA
+4413,181295.3120834606,UA
+4414,995694.9413199169,UA
+4416,157640.7868989174,UA
+4418,77580.20674809469,UA
+4420,122320.99275223716,UA
+4424,184891.10924920067,UA
+4425,84486.75974340564,UA
+4431,50485.84380961137,UA
+4435,231040.45446464577,UA
+4436,81222.18707585508,UA
+4438,114819.76472988473,UA
+4439,76839.1052178896,UA
+4440,135337.0313562152,UA
+4441,49159.485269198034,UA
+7031,42001.73757065917,UA
+7059,159790.48382874,UA
+7063,39599.10564971086,UA

data/custom_extra_functionality.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2023- The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+
+def custom_extra_functionality(n, snapshots):
+    """
+    Add custom extra functionality constraints.
+    """
+    pass

data/custom_powerplants.csv

@@ -1 +1,37 @@
-Name,Fueltype,Technology,Set,Country,Capacity,Efficiency,Duration,Volume_Mm3,DamHeight_m,YearCommissioned,Retrofit,lat,lon,projectID,YearDecommissioning
+,Name,Fueltype,Technology,Set,Country,Capacity,Efficiency,Duration,Volume_Mm3,DamHeight_m,StorageCapacity_MWh,DateIn,DateRetrofit,DateMothball,DateOut,lat,lon,EIC,projectID
+1266,Khmelnitskiy,Nuclear,,PP,UA,1901.8916595755832,,0.0,0.0,0.0,0.0,1988.0,2005.0,,,50.3023,26.6466,[nan],"{'GEO': ['GEO3842'], 'GPD': ['WRI1005111'], 'CARMA': ['CARMA22000']}"
+1268,Kaniv,Hydro,Reservoir,PP,UA,452.1656050955414,,0.0,0.0,0.0,0.0,1972.0,2003.0,,,49.76653,31.47165,[nan],"{'GEO': ['GEO43017'], 'GPD': ['WRI1005122'], 'CARMA': ['CARMA21140']}"
+1269,Kahovska kakhovka,Hydro,Reservoir,PP,UA,352.45222929936307,,0.0,0.0,0.0,0.0,1955.0,1956.0,,,46.77858,33.36965,[nan],"{'GEO': ['GEO43018'], 'GPD': ['WRI1005118'], 'CARMA': ['CARMA20855']}"
+1347,Kharkiv,Natural Gas,Steam Turbine,CHP,UA,494.94274967602314,,0.0,0.0,0.0,0.0,1979.0,1980.0,,,49.9719,36107,[nan],"{'GEO': ['GEO43027'], 'GPD': ['WRI1005126'], 'CARMA': ['CARMA21972']}"
+1348,Kremenchuk,Hydro,Reservoir,PP,UA,617.0382165605096,,0.0,0.0,0.0,0.0,1959.0,1960.0,,,49.07759,33.2505,[nan],"{'GEO': ['GEO43019'], 'GPD': ['WRI1005121'], 'CARMA': ['CARMA23072']}"
+1377,Krivorozhskaya,Hard Coal,Steam Turbine,PP,UA,2600.0164509342876,,0.0,0.0,0.0,0.0,1965.0,1992.0,,,47.5432,33.6583,[nan],"{'GEO': ['GEO42989'], 'GPD': ['WRI1005100'], 'CARMA': ['CARMA23176']}"
+1407,Zmiyevskaya zmiivskaya,Hard Coal,Steam Turbine,PP,UA,2028.3816283884514,,0.0,0.0,0.0,0.0,1960.0,2005.0,,,49.5852,36.5231,[nan],"{'GEO': ['GEO42999'], 'GPD': ['WRI1005103'], 'CARMA': ['CARMA51042']}"
+1408,Pridneprovskaya,Hard Coal,Steam Turbine,CHP,UA,1627.3152609570984,,0.0,0.0,0.0,0.0,1959.0,1966.0,,,48.4051,35.1131,[nan],"{'GEO': ['GEO42990'], 'GPD': ['WRI1005102'], 'CARMA': ['CARMA35874']}"
+1409,Kurakhovskaya,Hard Coal,Steam Turbine,PP,UA,1371.0015824607397,,0.0,0.0,0.0,0.0,1972.0,2003.0,,,47.9944,37.24022,[nan],"{'GEO': ['GEO42994'], 'GPD': ['WRI1005104'], 'CARMA': ['CARMA23339']}"
+1410,Dobrotvorsky,Hard Coal,Steam Turbine,PP,UA,553.1949895604868,,0.0,0.0,0.0,0.0,1960.0,1964.0,,,50.2133,24375,[nan],"{'GEO': ['GEO42992'], 'GPD': ['WRI1005096'], 'CARMA': ['CARMA10971']}"
+1422,Zuyevskaya,Hard Coal,Steam Turbine,PP,UA,1147.87960333801,,0.0,0.0,0.0,0.0,1982.0,2007.0,,,48.0331,38.28615,[nan],"{'GEO': ['GEO42995'], 'GPD': ['WRI1005106'], 'CARMA': ['CARMA51083']}"
+1423,Zaporozhye,Nuclear,,PP,UA,5705.67497872675,,0.0,0.0,0.0,0.0,1985.0,1996.0,,,47.5119,34.5863,[nan],"{'GEO': ['GEO6207'], 'GPD': ['WRI1005114'], 'CARMA': ['CARMA50875']}"
+1424,Trypilska,Hard Coal,Steam Turbine,PP,UA,1659.5849686814602,,0.0,0.0,0.0,0.0,1969.0,1972.0,,,50.1344,30.7468,[nan],"{'GEO': ['GEO43000'], 'GPD': ['WRI1005099'], 'CARMA': ['CARMA46410']}"
+1425,Tashlyk,Hydro,Pumped Storage,Store,UA,285.55968954109585,,0.0,0.0,0.0,0.0,2006.0,2007.0,,,47.7968,31.1811,[nan],"{'GEO': ['GEO43025'], 'GPD': ['WRI1005117'], 'CARMA': ['CARMA44696']}"
+1426,Starobeshivska,Hard Coal,Steam Turbine,PP,UA,1636.5351774497733,,0.0,0.0,0.0,0.0,1961.0,1967.0,,,47.7997,38.00612,[nan],"{'GEO': ['GEO43003'], 'GPD': ['WRI1005105'], 'CARMA': ['CARMA43083']}"
+1427,South,Nuclear,,PP,UA,2852.837489363375,,0.0,0.0,0.0,0.0,1983.0,1989.0,,,47812,31.22,[nan],"{'GEO': ['GEO5475'], 'GPD': ['WRI1005113'], 'CARMA': ['CARMA42555']}"
+1428,Rovno rivne,Nuclear,,PP,UA,2695.931427448389,,0.0,0.0,0.0,0.0,1981.0,2006.0,,,51.3245,25.89744,[nan],"{'GEO': ['GEO5174'], 'GPD': ['WRI1005112'], 'CARMA': ['CARMA38114']}"
+1429,Ladyzhinska,Hard Coal,Steam Turbine,PP,UA,1659.5849686814602,,0.0,0.0,0.0,0.0,1970.0,1971.0,,,48706,29.2202,[nan],"{'GEO': ['GEO42993'], 'GPD': ['WRI1005098'], 'CARMA': ['CARMA24024']}"
+1430,Kiev,Hydro,Pumped Storage,PP,UA,635.8694635681177,,0.0,0.0,0.0,0.0,1964.0,1972.0,,,50.5998,30501,"[nan, nan]","{'GEO': ['GEO43024', 'GEO43023'], 'GPD': ['WRI1005123', 'WRI1005124'], 'CARMA': ['CARMA23516', 'CARMA23517']}"
+2450,Cet chisinau,Natural Gas,,PP,MD,306.0,,0.0,0.0,0.0,0.0,,,,,47.027550000000005,28.8801,"[nan, nan]","{'GPD': ['WRI1002985', 'WRI1002984'], 'CARMA': ['CARMA8450', 'CARMA8451']}"
+2460,Hydropower che costesti,Hydro,,PP,MD,16.0,,0.0,0.0,0.0,0.0,1978.0,,,,47.8381,27.2246,[nan],"{'GPD': ['WRI1002987'], 'CARMA': ['CARMA9496']}"
+2465,Moldavskaya gres,Hard Coal,,PP,MD,2520.0,,0.0,0.0,0.0,0.0,,,,,46.6292,29.9407,[nan],"{'GPD': ['WRI1002989'], 'CARMA': ['CARMA28979']}"
+2466,Hydropower dubasari,Hydro,,PP,MD,48.0,,0.0,0.0,0.0,0.0,,,,,47.2778,29123,[nan],"{'GPD': ['WRI1002988'], 'CARMA': ['CARMA11384']}"
+2676,Cet nord balti,Natural Gas,,PP,MD,24.0,,0.0,0.0,0.0,0.0,,,,,47.7492,27.8938,[nan],"{'GPD': ['WRI1002986'], 'CARMA': ['CARMA3071']}"
+2699,Dniprodzerzhynsk,Hydro,Reservoir,PP,UA,360.3503184713376,,0.0,0.0,0.0,0.0,1963.0,1964.0,,,48.5485,34.541015,[nan],"{'GEO': ['GEO43020'], 'GPD': ['WRI1005119']}"
+2707,Burshtynska tes,Hard Coal,Steam Turbine,PP,UA,2212.779958241947,,0.0,0.0,0.0,0.0,1965.0,1984.0,,,49.21038,24.66654,[nan],"{'GEO': ['GEO42991'], 'GPD': ['WRI1005097']}"
+2708,Danipro dnieper,Hydro,Reservoir,PP,UA,1484.8407643312103,,0.0,0.0,0.0,0.0,1932.0,1947.0,,,47.86944,35.08611,[nan],"{'GEO': ['GEO43016'], 'GPD': ['WRI1005120']}"
+2709,Dniester,Hydro,Pumped Storage,Store,UA,612.7241020616891,,0.0,0.0,0.0,0.0,2009.0,2011.0,,,48.51361,27.47333,[nan],"{'GEO': ['GEO43022'], 'GPD': ['WRI1005116', 'WRI1005115']}"
+2710,Kiev,Natural Gas,Steam Turbine,CHP,UA,458.2803237740955,,0.0,0.0,0.0,0.0,1982.0,1984.0,,,50532,30.6625,[nan],"{'GEO': ['GEO42998'], 'GPD': ['WRI1005125']}"
+2712,Luganskaya,Hard Coal,Steam Turbine,PP,UA,1060.2903966575996,,0.0,0.0,0.0,0.0,1962.0,1969.0,,,48.74781,39.2624,[nan],"{'GEO': ['GEO42996'], 'GPD': ['WRI1005110']}"
+2713,Slavyanskaya,Hard Coal,Steam Turbine,PP,UA,737.5933194139823,,0.0,0.0,0.0,0.0,1971.0,1971.0,,,48872,37.76567,[nan],"{'GEO': ['GEO43002'], 'GPD': ['WRI1005109']}"
+2714,Vuhlehirska uglegorskaya,Hard Coal,Steam Turbine,PP,UA,3319.1699373629203,,0.0,0.0,0.0,0.0,1972.0,1977.0,,,48.4633,38.20328,[nan],"{'GEO': ['GEO43001'], 'GPD': ['WRI1005107']}"
+2715,Zaporiska,Hard Coal,Steam Turbine,PP,UA,3319.1699373629203,,0.0,0.0,0.0,0.0,1972.0,1977.0,,,47.5089,34.6253,[nan],"{'GEO': ['GEO42988'], 'GPD': ['WRI1005101']}"
+3678,Mironovskaya,Hard Coal,,PP,UA,815.0,,0.0,0.0,0.0,0.0,,,,,48.3407,38.4049,[nan],"{'GPD': ['WRI1005108'], 'CARMA': ['CARMA28679']}"
+3679,Kramatorskaya,Hard Coal,,PP,UA,120.0,,0.0,0.0,0.0,0.0,1974.0,,,,48.7477,37.5723,[nan],"{'GPD': ['WRI1075856'], 'CARMA': ['CARMA54560']}"
+3680,Chernihiv,Hard Coal,,PP,UA,200.0,,0.0,0.0,0.0,0.0,1968.0,,,,51455,31.2602,[nan],"{'GPD': ['WRI1075853'], 'CARMA': ['CARMA8190']}"

data/eia_hydro_annual_generation.csv

@@ -1,50 +1,53 @@
-https://www.eia.gov/international/data/world/electricity/electricity-generation?pd=2&p=000000000000000000000000000000g&u=1&f=A&v=mapbubble&a=-&i=none&vo=value&t=R&g=000000000000002&l=73-1028i008017kg6368g80a4k000e0ag00gg0004g8g0ho00g000400008&s=315532800000&e=1577836800000&ev=false&
-Report generated on: 03-28-2022 11:20:48
-"API","","1980","1981","1982","1983","1984","1985","1986","1987","1988","1989","1990","1991","1992","1993","1994","1995","1996","1997","1998","1999","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2011","2012","2013","2014","2015","2016","2017","2018","2019","2020"
-"","hydroelectricity net generation (billion kWh)","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","",""
-"INTL.33-12-EURO-BKWH.A","    Europe","458.018","464.155","459.881","473.685","481.241","476.739","459.535","491.085","534.517","465.365","474.466","475.47","509.041","526.448","531.815","543.743","529.114164","543.845616","562.441501","569.308453","591.206662","587.371195","541.542535","506.19703","544.536443","545.176179","537.335934","540.934407","567.557921","564.244482","619.96477","543.05273","600.46622","631.86431","619.59229","615.53013","629.98906","562.59258","619.31106","610.62616","670.925"
-"INTL.33-12-ALB-BKWH.A","        Albania","2.919","3.018","3.093","3.167","3.241","3.315","3.365","3.979","3.713","3.846","2.82","3.483","3.187","3.281","3.733","4.162","5.669","4.978","4.872","5.231","4.548","3.519","3.477","5.117","5.411","5.319","4.951","2.76","3.759","5.201","7.49133","4.09068","4.67775","6.88941","4.67676","5.83605","7.70418","4.47975","8.46648","5.15394","5.281"
-"INTL.33-12-AUT-BKWH.A","        Austria","28.501","30.008","29.893","29.577","28.384","30.288","30.496","25.401","35.151","34.641","31.179","31.112","34.483","36.336","35.349","36.696","33.874","35.744","36.792","40.292","41.418","40.05","39.825","32.883","36.394","36.31","35.48","36.732","37.969","40.487","36.466","32.511","41.862","40.138","39.001","35.255","37.954","36.462","35.73","40.43655","45.344"
-"INTL.33-12-BEL-BKWH.A","        Belgium","0.274","0.377","0.325","0.331","0.348","0.282","0.339","0.425","0.354","0.3","0.263","0.226","0.338","0.252","0.342","0.335","0.237","0.30195","0.38511","0.338","0.455","0.437","0.356","0.245","0.314","0.285","0.355","0.385","0.406","0.325","0.298","0.193","0.353","0.376","0.289","0.314","0.367","0.268","0.311","0.108","1.29"
-"INTL.33-12-BIH-BKWH.A","        Bosnia and Herzegovina","--","--","--","--","--","--","--","--","--","--","--","--","3.374","2.343","3.424","3.607","5.104","4.608","4.511","5.477","5.043","5.129","5.215","4.456","5.919","5.938","5.798","3.961","4.818","6.177","7.946","4.343","4.173","7.164","5.876","5.495","5.585","3.7521","6.35382","6.02019","6.1"
-"INTL.33-12-BGR-BKWH.A","        Bulgaria","3.674","3.58","3.018","3.318","3.226","2.214","2.302","2.512","2.569","2.662","1.859","2.417","2.042","1.923","1.453","2.291","2.89","2.726","3.066","2.725","2.646","1.72","2.172","2.999","3.136","4.294","4.196","2.845","2.796","3.435","4.98168","2.84328","3.14622","3.99564","4.55598","5.59845","3.8412","2.79972","5.09553","3.34917","3.37"
-"INTL.33-12-HRV-BKWH.A","        Croatia","--","--","--","--","--","--","--","--","--","--","--","--","4.298","4.302","4.881","5.212","7.156","5.234","5.403","6.524","5.794","6.482","5.311","4.827","6.888","6.27","5.94","4.194","5.164","6.663","9.035","4.983","4.789","8.536","8.917","6.327","6.784","5.255","7.62399","5.87268","3.4"
-"INTL.33-12-CYP-BKWH.A","        Cyprus","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0"
-"INTL.33-12-CZE-BKWH.A","        Czech Republic","--","--","--","--","--","--","--","--","--","--","--","--","--","1.355","1.445","1.982","1.949","1.68201","1.382","1.664","1.7404","2.033","2.467","1.369","1.999","2.356","2.525","2.068","2.004","2.405","2.775","1.95","2.107","2.704","1.909","1.779","1.983","1.852","1.615","1.98792","3.4"
-"INTL.33-12-DNK-BKWH.A","        Denmark","0.03","0.031","0.028","0.036","0.028","0.027","0.029","0.029","0.032","0.027","0.027","0.026","0.028","0.027","0.033","0.03","0.019","0.019","0.02673","0.031","0.03","0.028","0.032","0.021","0.027","0.023","0.023","0.028","0.026","0.019","0.021","0.017","0.017","0.013","0.015","0.018","0.019","0.018","0.015","0.01584","0.02"
-"INTL.33-12-EST-BKWH.A","        Estonia","--","--","--","--","--","--","--","--","--","--","--","--","0.001","0.001","0.003","0.002","0.002","0.003","0.004","0.004","0.005","0.007","0.006","0.013","0.022","0.022","0.014","0.021","0.028","0.032","0.027","0.03","0.042","0.026","0.027","0.027","0.035","0.026","0.015","0.01881","0.04"
-"INTL.33-12-FRO-BKWH.A","        Faroe Islands","0.049","0.049","0.049","0.049","0.049","0.049","0.049","0.049","0.062","0.071","0.074","0.074","0.083","0.073","0.075","0.075","0.069564","0.075066","0.076501","0.069453","0.075262","0.075195","0.095535","0.08483","0.093443","0.097986","0.099934","0.103407","0.094921","0.091482","0.06676","0.092","0.099","0.091","0.121","0.132","0.105","0.11","0.107","0.102","0.11"
-"INTL.33-12-FIN-BKWH.A","        Finland","10.115","13.518","12.958","13.445","13.115","12.211","12.266","13.658","13.229","12.9","10.75","13.065","14.956","13.341","11.669","12.796","11.742","12.11958","14.9","12.652","14.513","13.073","10.668","9.495","14.919","13.646","11.379","14.035","16.941","12.559","12.743","12.278","16.667","12.672","13.24","16.584","15.634","14.61","13.137","12.31461","15.56"
-"INTL.33-12-CSK-BKWH.A","        Former Czechoslovakia","4.8","4.2","3.7","3.9","3.2","4.3","4","4.853","4.355","4.229","3.919","3.119","3.602","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--"
-"INTL.33-12-SCG-BKWH.A","        Former Serbia and Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","11.23","10.395","11.016","12.071","14.266","12.636","12.763","13.243","11.88","12.326","11.633","9.752","11.01","11.912","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--"
-"INTL.33-12-YUG-BKWH.A","        Former Yugoslavia","27.868","25.044","23.295","21.623","25.645","24.363","27.474","25.98","25.612","23.256","19.601","18.929","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--"
-"INTL.33-12-FRA-BKWH.A","        France","68.253","70.358","68.6","67.515","64.01","60.248","60.953","68.623","73.952","45.744","52.796","56.277","68.313","64.3","78.057","72.196","64.43","63.151","61.479","71.832","66.466","73.888","59.992","58.567","59.276","50.965","55.741","57.029","63.017","56.428","61.945","45.184","59.099","71.042","62.993","54.876","60.094","49.389","64.485","56.98242","64.84"
-"INTL.33-12-DEU-BKWH.A","        Germany","--","--","--","--","--","--","--","--","--","--","--","14.742","17.223","17.699","19.731","21.562","21.737","17.18343","17.044","19.451","21.515","22.506","22.893","19.071","20.866","19.442","19.808","20.957","20.239","18.841","20.678","17.323","21.331","22.66","19.31","18.664","20.214","19.985","17.815","19.86039","24.75"
-"INTL.33-12-DDR-BKWH.A","        Germany, East","1.658","1.718","1.748","1.683","1.748","1.758","1.767","1.726","1.719","1.551","1.389","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--"
-"INTL.33-12-DEUW-BKWH.A","        Germany, West","17.125","17.889","17.694","16.713","16.434","15.354","16.526","18.36","18.128","16.482","15.769","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--"
-"INTL.33-12-GIB-BKWH.A","        Gibraltar","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0"
-"INTL.33-12-GRC-BKWH.A","        Greece","3.396","3.398","3.551","2.331","2.852","2.792","3.222","2.768","2.354","1.888","1.751","3.068","2.181","2.26","2.573","3.494","4.305","3.84318","3.68","4.546","3.656","2.076","2.772","4.718","4.625","4.967","5.806","2.565","3.279","5.32","7.431","3.998","4.387","6.337","4.464","5.782","5.543","3.962","5.035","3.9798","3.43"
-"INTL.33-12-HUN-BKWH.A","        Hungary","0.111","0.166","0.158","0.153","0.179","0.153","0.152","0.167","0.167","0.156","0.176","0.192","0.156","0.164","0.159","0.161","0.205","0.21384","0.15345","0.179","0.176","0.184","0.192","0.169","0.203","0.2","0.184","0.208","0.211","0.226","0.184","0.216","0.206","0.208","0.294","0.227","0.253","0.214","0.216","0.21681","0.24"
-"INTL.33-12-ISL-BKWH.A","        Iceland","3.053","3.085","3.407","3.588","3.738","3.667","3.846","3.918","4.169","4.217","4.162","4.162","4.267","4.421","4.47","4.635","4.724","5.15493","5.565","5.987","6.292","6.512","6.907","7.017","7.063","6.949","7.22","8.31","12.303","12.156","12.51","12.382","12.214","12.747","12.554","13.541","13.092","13.892","13.679","13.32441","12.46"
-"INTL.33-12-IRL-BKWH.A","        Ireland","0.833","0.855","0.792","0.776","0.68","0.824","0.91","0.673","0.862","0.684","0.69","0.738","0.809","0.757","0.911","0.706","0.715","0.67122","0.907","0.838","0.838","0.59","0.903","0.592","0.624","0.625","0.717","0.66","0.959","0.893","0.593","0.699","0.795","0.593","0.701","0.798","0.674","0.685","0.687","0.87813","1.21"
-"INTL.33-12-ITA-BKWH.A","        Italy","44.997","42.782","41.216","40.96","41.923","40.616","40.626","39.05","40.205","33.647","31.31","41.817","41.778","41.011","44.212","37.404","41.617","41.18697","40.808","44.911","43.763","46.343","39.125","33.303","41.915","35.706","36.624","32.488","41.207","48.647","50.506","45.36477","41.45625","52.24626","57.95955","45.08163","42.00768","35.83701","48.29913","45.31824","47.72"
-"INTL.33-12-XKS-BKWH.A","        Kosovo","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","0.075","0.119","0.154","0.104","0.095","0.142","0.149","0.139","0.243","0.177","0.27027","0.2079","0.26"
-"INTL.33-12-LVA-BKWH.A","        Latvia","--","--","--","--","--","--","--","--","--","--","--","--","2.498","2.846","3.272","2.908","1.841","2.922","2.99","2.729","2.791","2.805","2.438","2.243","3.078","3.293","2.671","2.706","3.078","3.422","3.488","2.857","3.677","2.838","1.953","1.841","2.523","4.356","2.417","2.08692","2.59"
-"INTL.33-12-LTU-BKWH.A","        Lithuania","--","--","--","--","--","--","--","--","--","--","--","--","0.308","0.389","0.447","0.369","0.323","0.291","0.413","0.409","0.336","0.322","0.35","0.323","0.417","0.446193","0.393","0.417","0.398","0.42","0.535","0.475","0.419","0.516","0.395","0.346","0.45","0.597","0.427","0.34254","1.06"
-"INTL.33-12-LUX-BKWH.A","        Luxembourg","0.086","0.095","0.084","0.083","0.088","0.071","0.084","0.101","0.097","0.072","0.07","0.083","0.069","0.066","0.117","0.087","0.059","0.082","0.114","0.084","0.119","0.117","0.098","0.078","0.103","0.093","0.11","0.116","0.131","0.105","0.104","0.061","0.095","0.114","0.104","0.095","0.111","0.082","0.089","0.10593","1.09"
-"INTL.33-12-MLT-BKWH.A","        Malta","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0"
-"INTL.33-12-MNE-BKWH.A","        Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","1.733","1.271","1.524","2.05","2.723","1.192","1.462","2.479","1.734","1.476","1.825","1.014","2.09187","1.78","1.8"
-"INTL.33-12-NLD-BKWH.A","        Netherlands","0","0","0","0","0","0.003","0.003","0.001","0.002","0.037","0.119","0.079","0.119","0.091","0.1","0.087","0.079","0.09108","0.111","0.089","0.141","0.116","0.109","0.071","0.094","0.087","0.105","0.106","0.101","0.097","0.105","0.057","0.104","0.114","0.112","0.093","0.1","0.061","0.072","0.07326","0.05"
-"INTL.33-12-MKD-BKWH.A","        North Macedonia","--","--","--","--","--","--","--","--","--","--","--","--","0.817","0.517","0.696","0.793","0.842","0.891","1.072","1.375","1.158","0.62","0.749","1.36","1.467","1.477","1.634","1","0.832","1.257","2.407","1.419","1.031","1.568","1.195","1.846","1.878","1.099","1.773","1.15236","1.24"
-"INTL.33-12-NOR-BKWH.A","        Norway","82.717","91.876","91.507","104.704","104.895","101.464","95.321","102.341","107.919","117.369","119.933","109.032","115.505","118.024","110.398","120.315","102.823","108.677","114.546","120.237","140.4","119.258","128.078","104.425","107.693","134.331","118.175","132.319","137.654","124.03","116.257","119.78","141.189","127.551","134.844","136.662","142.244","141.651","138.202","123.66288","141.69"
-"INTL.33-12-POL-BKWH.A","        Poland","2.326","2.116","1.528","1.658","1.394","1.833","1.534","1.644","1.775","1.593","1.403","1.411","1.492","1.473","1.716","1.868","1.912","1.941","2.286","2.133","2.085","2.302","2.256","1.654","2.06","2.179","2.022","2.328","2.13","2.351","2.9","2.313","2.02","2.421","2.165","1.814","2.117","2.552","1.949","1.93842","2.93"
-"INTL.33-12-PRT-BKWH.A","        Portugal","7.873","4.934","6.82","7.897","9.609","10.512","8.364","9.005","12.037","5.72","9.065","8.952","4.599","8.453","10.551","8.26","14.613","12.97395","12.853","7.213","11.21","13.894","7.722","15.566","9.77","4.684","10.892","9.991","6.73","8.201","15.954","11.423","5.589","13.652","15.471","8.615","15.608","5.79","12.316","8.6526","13.96"
-"INTL.33-12-ROU-BKWH.A","        Romania","12.506","12.605","11.731","9.934","11.208","11.772","10.688","11.084","13.479","12.497","10.87","14.107","11.583","12.64","12.916","16.526","15.597","17.334","18.69","18.107","14.63","14.774","15.886","13.126","16.348","20.005","18.172","15.806","17.023","15.379","19.684","14.581","11.945","14.807","18.618","16.467","17.848","14.349","17.48736","15.65289","15.53"
-"INTL.33-12-SRB-BKWH.A","        Serbia","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","10.855","9.937","9.468","10.436","11.772","8.58","9.193","10.101","10.893","9.979","10.684","9.061","10.53261","10.07028","9.66"
-"INTL.33-12-SVK-BKWH.A","        Slovakia","--","--","--","--","--","--","--","--","--","--","--","--","--","3.432","4.311","4.831","4.185","4.023","4.224","4.429","4.569","4.878","5.215","3.4452","4.059","4.592","4.355","4.406","4","4.324","5.184","3.211","3.687","4.329","3.762","3.701","4.302","4.321","3.506","4.27383","4.67"
-"INTL.33-12-SVN-BKWH.A","        Slovenia","--","--","--","--","--","--","--","--","--","--","--","--","3.379","2.974","3.348","3.187","3.616","3.046","3.4","3.684","3.771","3.741","3.265","2.916","4.033","3.426","3.555","3.233","3.978","4.666","4.452","3.506","3.841","4.562","6.011","3.75","4.443","3.814","4.643","4.43421","5.24"
-"INTL.33-12-ESP-BKWH.A","        Spain","29.16","21.64","25.99","26.696","31.088","30.895","26.105","27.016","34.76","19.046","25.16","27.01","18.731","24.133","27.898","22.881","39.404","34.43","33.665","22.634","29.274","40.617","22.691","40.643","31.359","18.209","25.699","27.036","23.13","26.147","41.576","30.07","20.192","36.45","38.815","27.656","35.77","18.007","33.743","24.23025","33.34"
-"INTL.33-12-SWE-BKWH.A","        Sweden","58.133","59.006","54.369","62.801","67.106","70.095","60.134","70.95","69.016","70.911","71.778","62.603","73.588","73.905","58.508","67.421","51.2226","68.365","74.25","70.974","77.798","78.269","65.696","53.005","59.522","72.075","61.106","65.497","68.378","65.193","66.279","66.047","78.333","60.81","63.227","74.734","61.645","64.651","61.79","64.46583","71.6"
-"INTL.33-12-CHE-BKWH.A","        Switzerland","32.481","35.13","35.974","35.069","29.871","31.731","32.576","34.328","35.437","29.477","29.497","31.756","32.373","35.416","38.678","34.817","28.458","33.70257","33.136","39.604","36.466","40.895","34.862","34.471","33.411","30.914","30.649","34.898","35.676","35.366","35.704","32.069","38.218","38.08","37.659","37.879","34.281","33.754","34.637","37.6596","40.62"
-"INTL.33-12-TUR-BKWH.A","        Turkey","11.159","12.308","13.81","11.13","13.19","11.822","11.637","18.314","28.447","17.61","22.917","22.456","26.302","33.611","30.28","35.186","40.07","39.41784","41.80671","34.33","30.57","23.77","33.346","34.977","45.623","39.165","43.802","35.492","32.937","35.598","51.423","51.155","56.669","58.225","39.75","65.856","66.686","57.824","59.49","87.99714","77.39"
-"INTL.33-12-GBR-BKWH.A","        United Kingdom","3.921","4.369","4.543","4.548","3.992","4.08","4.767","4.13","4.915","4.732","5.119","4.534","5.329","4.237","5.043","4.79","3.359","4.127","5.067","5.283","5.035","4.015","4.74","3.195","4.795","4.873","4.547","5.026","5.094","5.178","3.566","5.655","5.286","4.667","5.832","6.246","5.342","5.836","5.189","5.89941","7.64"
+https://www.eia.gov/international/data/world/electricity/electricity-generation?pd=2&p=000000000000000000000000000000g&u=1&f=A&v=mapbubble&a=-&i=none&vo=value&t=R&g=000000000000002&l=73-1028i008017kg6368g80a4k000e0ag00gg0004g8g0ho00g000400008&l=72-00000000000000000000000000080000000000000000000g&s=315532800000&e=1609459200000&ev=false&,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Report generated on: 01-06-2023 21:17:46,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+API,,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021
+,hydroelectricity net generation (billion kWh),,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+INTL.33-12-EURO-BKWH.A,    Europe,"458,018","464,155","459,881","473,685","481,241","476,739","459,535","491,085","534,517","465,365","474,466","475,47","509,041","526,448","531,815","543,743","529,114164","543,845616","562,491501","566,861453","588,644662","584,806195","539,051405","503,7067","542,112443","542,974669","535,006084","538,449707","565,143111","561,761402","617,547148","540,926277","598,055253","629,44709","617,111295","613,079848","627,720566217","560,362524","616,5081462","606,5997419","644,1106599","628,1390143"
+INTL.33-12-ALB-BKWH.A,        Albania,"2,919","3,018","3,093","3,167","3,241","3,315","3,365","3,979","3,713","3,846","2,82","3,483","3,187","3,281","3,733","4,162","5,669","4,978","4,872","5,231","4,548","3,519","3,477","5,117","5,411","5,319","4,951","2,76","3,759","5,201","7,49133","4,09068","4,67775","6,88941","4,67676","5,83605","7,70418","4,47975","8,46648","5,15394","5,281","8,891943"
+INTL.33-12-AUT-BKWH.A,        Austria,"28,501","30,008","29,893","29,577","28,384","30,288","30,496","25,401","35,151","34,641","31,179","31,112","34,483","36,336","35,349","36,696","33,874","35,744","36,792","40,292","41,418","40,05","39,825","32,883","36,394","36,31","35,48","36,732","37,969","40,487","36,466","32,511","41,862","40,138","39,001","35,255","37,954","36,462","35,73","40,43655","41,9356096","38,75133"
+INTL.33-12-BEL-BKWH.A,        Belgium,"0,274","0,377","0,325","0,331","0,348","0,282","0,339","0,425","0,354","0,3","0,263","0,226","0,338","0,252","0,342","0,335","0,237","0,30195","0,38511","0,338","0,455","0,437","0,356","0,245","0,314","0,285","0,355","0,385","0,406","0,325","0,298","0,193","0,353","0,376","0,289","0,314","0,367","0,268","0,3135","0,302","0,2669","0,3933"
+INTL.33-12-BIH-BKWH.A,        Bosnia and Herzegovina,--,--,--,--,--,--,--,--,--,--,--,--,"3,374","2,343","3,424","3,607","5,104","4,608","4,511","5,477","5,043","5,129","5,215","4,456","5,919","5,938","5,798","3,961","4,818","6,177","7,946","4,343","4,173","7,164","5,876","5,495","5,585","3,7521","6,35382","6,02019","4,58","6,722"
+INTL.33-12-BGR-BKWH.A,        Bulgaria,"3,674","3,58","3,018","3,318","3,226","2,214","2,302","2,512","2,569","2,662","1,859","2,417","2,042","1,923","1,453","2,291","2,89","2,726","3,066","2,725","2,646","1,72","2,172","2,999","3,136","4,294","4,196","2,845","2,796","3,435","4,98168","2,84328","3,14622","3,99564","4,55598","5,59845","3,8412","2,79972","5,09553","2,929499","2,820398","4,819205"
+INTL.33-12-HRV-BKWH.A,        Croatia,--,--,--,--,--,--,--,--,--,--,--,--,"4,298","4,302","4,881","5,212","7,156","5,234","5,403","6,524","5,794","6,482","5,311","4,827","6,888","6,27","5,94","4,194","5,164","6,663","9,035","4,983","4,789","8,536","8,917","6,327","6,784","5,255","7,62399","5,87268","5,6624","7,1277"
+INTL.33-12-CYP-BKWH.A,        Cyprus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+INTL.33-12-CZE-BKWH.A,        Czechia,--,--,--,--,--,--,--,--,--,--,--,--,--,"1,355","1,445","1,982","1,949","1,68201","1,382","1,664","1,7404","2,033","2,467","1,369","1,999","2,356","2,525","2,068","2,004","2,405","2,775","1,95","2,107","2,704","1,909","1,779","1,983","1,852","1,615","1,98792","2,143884","2,40852"
+INTL.33-12-DNK-BKWH.A,        Denmark,"0,03","0,031","0,028","0,036","0,028","0,027","0,029","0,029","0,032","0,027","0,027","0,026","0,028","0,027","0,033","0,03","0,019","0,019","0,02673","0,031","0,03","0,028","0,032","0,021","0,027","0,023","0,023","0,028","0,026","0,019","0,021","0,017","0,017","0,013","0,015","0,01803","0,01927","0,017871","0,0148621","0,0172171","0,017064","0,016295"
+INTL.33-12-EST-BKWH.A,        Estonia,--,--,--,--,--,--,--,--,--,--,--,--,"0,001","0,001","0,003","0,002","0,002","0,003","0,004","0,004","0,005","0,007","0,006","0,013","0,022","0,022","0,014","0,021","0,028","0,032","0,027","0,029999","0,042","0,026","0,027","0,027","0,035","0,025999","0,0150003","0,0189999","0,03","0,0248"
+INTL.33-12-FRO-BKWH.A,        Faroe Islands,"0,049","0,049","0,049","0,049","0,049","0,049","0,049","0,049","0,062","0,071","0,074","0,074","0,083","0,073","0,075","0,075","0,069564","0,075066","0,076501","0,069453","0,075262","0,075195","0,095535","0,08483","0,093443","0,097986","0,099934","0,103407","0,094921","0,091482","0,06676","0,092","0,099","0,091","0,121","0,132","0,105","0,11","0,107","0,102","0,11","0,11"
+INTL.33-12-FIN-BKWH.A,        Finland,"10,115","13,518","12,958","13,445","13,115","12,211","12,266","13,658","13,229","12,9","10,75","13,065","14,956","13,341","11,669","12,796","11,742","12,11958","14,9","12,652","14,513","13,073","10,668","9,495","14,919","13,646","11,379","14,035","16,941","12,559","12,743","12,278001","16,666998","12,672","13,240001","16,583999","15,634127","14,609473","13,1369998","12,2454823","15,883","15,766"
+INTL.33-12-CSK-BKWH.A,        Former Czechoslovakia,"4,8","4,2","3,7","3,9","3,2","4,3",4,"4,853","4,355","4,229","3,919","3,119","3,602",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--
+INTL.33-12-SCG-BKWH.A,        Former Serbia and Montenegro,--,--,--,--,--,--,--,--,--,--,--,--,"11,23","10,395","11,016","12,071","14,266","12,636","12,763","13,243","11,88","12,326","11,633","9,752","11,01","11,912",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--
+INTL.33-12-YUG-BKWH.A,        Former Yugoslavia,"27,868","25,044","23,295","21,623","25,645","24,363","27,474","25,98","25,612","23,256","19,601","18,929",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--
+INTL.33-12-FRA-BKWH.A,        France,"68,253","70,358","68,6","67,515","64,01","60,248","60,953","68,623","73,952","45,744","52,796","56,277","68,313","64,3","78,057","72,196","64,43","63,151","61,479","71,832","66,466","73,888","59,992","58,567","59,276","50,965","55,741","57,029","63,017","56,428","61,945","45,184","59,099","71,042","62,993","54,876","60,094","49,389","64,485","56,913891","62,06191","58,856657"
+INTL.33-12-DEU-BKWH.A,        Germany,--,--,--,--,--,--,--,--,--,--,--,"14,742","17,223","17,699","19,731","21,562","21,737","17,18343","17,044","19,451","21,515","22,506","22,893","19,071","20,866","19,442","19,808","20,957","20,239","18,841","20,678","17,323","21,331","22,66","19,31","18,664","20,214","19,985","17,694","19,731","18,322","19,252"
+INTL.33-12-DDR-BKWH.A,"        Germany, East","1,658","1,718","1,748","1,683","1,748","1,758","1,767","1,726","1,719","1,551","1,389",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--
+INTL.33-12-DEUW-BKWH.A,"        Germany, West","17,125","17,889","17,694","16,713","16,434","15,354","16,526","18,36","18,128","16,482","15,769",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--
+INTL.33-12-GIB-BKWH.A,        Gibraltar,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+INTL.33-12-GRC-BKWH.A,        Greece,"3,396","3,398","3,551","2,331","2,852","2,792","3,222","2,768","2,354","1,888","1,751","3,068","2,181","2,26","2,573","3,494","4,305","3,84318","3,68","4,546","3,656","2,076","2,772","4,718","4,625","4,967","5,806","2,565","3,279","5,32","7,431","3,998","4,387","6,337","4,464","5,782","5,543","3,962","5,035","3,9798","3,343687","5,909225"
+INTL.33-12-HUN-BKWH.A,        Hungary,"0,111","0,166","0,158","0,153","0,179","0,153","0,152","0,167","0,167","0,156","0,176","0,192","0,156","0,164","0,159","0,161","0,205","0,21384","0,15345","0,179","0,176","0,184","0,192","0,169","0,203","0,2","0,184","0,208","0,211","0,226","0,184","0,215999","0,205999","0,207999","0,294001","0,226719","0,253308","0,213999","0,216","0,2129999","0,238","0,202379"
+INTL.33-12-ISL-BKWH.A,        Iceland,"3,053","3,085","3,407","3,588","3,738","3,667","3,846","3,918","4,169","4,217","4,162","4,162","4,267","4,421","4,47","4,635","4,724","5,15493","5,565","5,987","6,292","6,512","6,907","7,017","7,063","6,949","7,22","8,31","12,303","12,156","12,509999","12,381999","12,213999","12,747001","12,554","13,541","13,091609","13,891929","13,679377","13,32911","12,9196201","13,5746171"
+INTL.33-12-IRL-BKWH.A,        Ireland,"0,833","0,855","0,792","0,776","0,68","0,824","0,91","0,673","0,862","0,684","0,69","0,738","0,809","0,757","0,911","0,706","0,715","0,67122","0,907","0,838","0,838","0,59","0,903","0,592","0,624","0,625","0,717","0,66","0,959","0,893","0,593","0,699","0,795","0,593","0,701","0,798","0,674","0,685","0,687","0,87813","0,932656","0,750122"
+INTL.33-12-ITA-BKWH.A,        Italy,"44,997","42,782","41,216","40,96","41,923","40,616","40,626","39,05","40,205","33,647","31,31","41,817","41,778","41,011","44,212","37,404","41,617","41,18697","40,808","44,911","43,763","46,343","39,125","33,303","41,915","35,706","36,624","32,488","41,207","48,647","50,506","45,36477","41,45625","52,24626","57,95955","45,08163","42,00768","35,83701","48,29913","45,31824","47,551784","44,739"
+INTL.33-12-XKS-BKWH.A,        Kosovo,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,"0,075","0,119","0,154","0,104","0,095","0,142","0,149","0,139","0,243","0,177","0,27027","0,2079","0,262826","0,300635"
+INTL.33-12-LVA-BKWH.A,        Latvia,--,--,--,--,--,--,--,--,--,--,--,--,"2,498","2,846","3,272","2,908","1,841","2,922","2,99","2,729","2,791","2,805","2,438","2,243","3,078","3,293","2,671","2,706","3,078","3,422","3,487998","2,8568","3,677","2,838","1,953","1,841","2,522819","4,355513","2,4170639","2,0958919","2,5840101","2,6889293"
+INTL.33-12-LTU-BKWH.A,        Lithuania,--,--,--,--,--,--,--,--,--,--,--,--,"0,308","0,389","0,447","0,369","0,323","0,291","0,413","0,409","0,336","0,322","0,35","0,323","0,417","0,446193","0,393","0,417","0,398","0,42","0,535","0,475","0,419","0,516","0,395","0,346","0,45","0,597","0,427","0,34254","0,3006","0,3837"
+INTL.33-12-LUX-BKWH.A,        Luxembourg,"0,086","0,095","0,084","0,083","0,088","0,071","0,084","0,101","0,097","0,072","0,07","0,083","0,069","0,066","0,117","0,087","0,059","0,082","0,114","0,084","0,119","0,117","0,098","0,078","0,103","0,093","0,11","0,116","0,131","0,105","0,104","0,061","0,095","0,114","0,104","0,095","0,111","0,082","0,089","0,10593","0,091602","0,1068"
+INTL.33-12-MLT-BKWH.A,        Malta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+INTL.33-12-MNE-BKWH.A,        Montenegro,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,"1,733","1,271","1,524","2,05","2,723","1,192","1,462","2,479","1,734","1,476","1,825","1,014","1,693443","1,262781","0,867637","1,212652"
+INTL.33-12-NLD-BKWH.A,        Netherlands,0,0,0,0,0,"0,003","0,003","0,001","0,002","0,037","0,119","0,079","0,119","0,091","0,1","0,087","0,079","0,09108","0,111","0,089","0,141","0,116","0,109","0,071","0,094","0,087","0,105","0,106","0,101","0,097","0,105","0,057","0,104389","0,11431","0,112202","0,0927","0,100078","0,060759","0,0723481","0,074182","0,0462851","0,0838927"
+INTL.33-12-MKD-BKWH.A,        North Macedonia,--,--,--,--,--,--,--,--,--,--,--,--,"0,817","0,517","0,696","0,793","0,842","0,891","1,072","1,375","1,158","0,62","0,749","1,36","1,467","1,477","1,634",1,"0,832","1,257","2,407","1,419","1,031","1,568","1,195","1,846","1,878","1,099","1,773","1,15236","1,277144","1,451623"
+INTL.33-12-NOR-BKWH.A,        Norway,"82,717","91,876","91,507","104,704","104,895","101,464","95,321","102,341","107,919","117,369","119,933","109,032","115,505","118,024","110,398","120,315","102,823","108,677","114,546","120,237","140,4","119,258","128,078","104,425","107,693","134,331","118,175","132,319","137,654","124,03","116,257","119,78","141,189","127,551","134,844","136,662","142,244","141,651","138,202","123,66288","141,69",144
+INTL.33-12-POL-BKWH.A,        Poland,"2,326","2,116","1,528","1,658","1,394","1,833","1,534","1,644","1,775","1,593","1,403","1,411","1,492","1,473","1,716","1,868","1,912","1,941","2,286","2,133","2,085","2,302","2,256","1,654","2,06","2,179","2,022","2,328","2,13","2,351","2,9","2,313","2,02","2,421","2,165","1,814","2,117","2,552","1,949","1,93842","2,118337","2,339192"
+INTL.33-12-PRT-BKWH.A,        Portugal,"7,873","4,934","6,82","7,897","9,609","10,512","8,364","9,005","12,037","5,72","9,065","8,952","4,599","8,453","10,551","8,26","14,613","12,97395","12,853","7,213","11,21","13,894","7,722","15,566","9,77","4,684","10,892","9,991","6,73","8,201","15,954","11,423","5,589","13,652","15,471","8,615","15,608","5,79","12,316","8,6526","12,082581","11,846464"
+INTL.33-12-ROU-BKWH.A,        Romania,"12,506","12,605","11,731","9,934","11,208","11,772","10,688","11,084","13,479","12,497","10,87","14,107","11,583","12,64","12,916","16,526","15,597","17,334","18,69","18,107","14,63","14,774","15,886","13,126","16,348","20,005","18,172","15,806","17,023","15,379","19,684","14,581","11,945","14,807","18,618","16,467","17,848","14,349","17,48736","15,580622","15,381243","17,376933"
+INTL.33-12-SRB-BKWH.A,        Serbia,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,"10,855","9,937","9,468","10,436","11,772","8,58","9,193","10,101","10,893","9,979","10,684","9,061","10,53261","9,457175","9,034496","11,284232"
+INTL.33-12-SVK-BKWH.A,        Slovakia,--,--,--,--,--,--,--,--,--,--,--,--,--,"3,432","4,311","4,831","4,185","4,023","4,224","4,429","4,569","4,878","5,215","3,4452","4,059","4,592","4,355","4,406",4,"4,324","5,184","3,211","3,687","4,329","3,762","3,701","4,302","4,321","3,506","4,27383","4,517","4,17"
+INTL.33-12-SVN-BKWH.A,        Slovenia,--,--,--,--,--,--,--,--,--,--,--,--,"3,379","2,974","3,348","3,187","3,616","3,046","3,4","3,684","3,771","3,741","3,265","2,916","4,033","3,426","3,555","3,233","3,978","4,666","4,452","3,506","3,841","4,562","6,011","3,75","4,443","3,814","4,643","4,43421","4,93406","4,711944"
+INTL.33-12-ESP-BKWH.A,        Spain,"29,16","21,64","25,99","26,696","31,088","30,895","26,105","27,016","34,76","19,046","25,16","27,01","18,731","24,133","27,898","22,881","39,404","34,43","33,665","22,634","29,274","40,617","22,691","40,643","31,359","18,209","25,699","27,036","23,13","26,147","41,576","30,07","20,192","36,45","38,815","27,656","35,77","18,007","33,743","24,23025","30,507","29,626"
+INTL.33-12-SWE-BKWH.A,        Sweden,"58,133","59,006","54,369","62,801","67,106","70,095","60,134","70,95","69,016","70,911","71,778","62,603","73,588","73,905","58,508","67,421","51,2226","68,365","74,25","70,974","77,798","78,269","65,696","53,005","59,522","72,075","61,106","65,497","68,378","65,193","66,279","66,047","78,333","60,81","63,227","74,734","61,645","64,651","61,79","64,46583","71,6","71,086"
+INTL.33-12-CHE-BKWH.A,        Switzerland,"32,481","35,13","35,974","35,069","29,871","31,731","32,576","34,328","35,437","29,477","29,497","31,756","32,373","35,416","38,678","34,817","28,458","33,70257","33,136","37,104","33,854","38,29","32,323","31,948","30,938","28,664","28,273","32,362","33,214","32,833","33,261","29,906","35,783","35,628","35,122","35,378","31,984","31,47968","32,095881","35,156989","37,867647","36,964485"
+INTL.33-12-TUR-BKWH.A,        Turkey,"11,159","12,308","13,81","11,13","13,19","11,822","11,637","18,314","28,447","17,61","22,917","22,456","26,302","33,611","30,28","35,186","40,07","39,41784","41,80671","34,33","30,57","23,77","33,346","34,977","45,623","39,165","43,802","35,492","32,937","35,598","51,423001","51,154999","56,668998","58,225","39,750001","65,856","66,685883","57,823851","59,490211","88,2094218","78,094369","55,1755392"
+INTL.33-12-GBR-BKWH.A,        United Kingdom,"3,921","4,369","4,543","4,548","3,992","4,08","4,767","4,13","4,915","4,732","5,119","4,534","5,329","4,237","5,043","4,79","3,359","4,127","5,117","5,336","5,085","4,055","4,78787","3,22767","4,844","4,92149","4,59315","5,0773","5,14119","5,22792","3,59138","5,69175","5,30965","4,70147","5,8878","6,29727","5,370412217","5,88187","5,44327","5,84628","6,75391","5,0149"
+,    Eurasia,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+INTL.33-12-MDA-BKWH.A,        Moldova,--,--,--,--,--,--,--,--,--,--,--,--,"0,255","0,371","0,275","0,321","0,362","0,378","0,387","0,363","0,392","0,359","0,348","0,358","0,35","0,359","0,365","0,354","0,385","0,354","0,403","0,348","0,266","0,311","0,317","0,265","0,228","0,282","0,27324","0,29799","0,276","0,316"
+INTL.33-12-UKR-BKWH.A,        Ukraine,--,--,--,--,--,--,--,--,--,--,--,--,"7,725","10,929","11,997","9,853","8,546","9,757","15,756","14,177","11,161","11,912","9,531","9,146","11,635","12,239","12,757","10,042","11,397","11,817","13,02","10,837","10,374","13,663","8,393","5,343","7,594","8,856","10,32372","6,5083","7,5638","10,3326"

doc/configtables/clustering.csv

@@ -1,4 +1,5 @@
 ,Unit,Values,Description
+focus_weights,,,Optionally specify the focus weights for the clustering of countries. For instance: `DE: 0.8` will distribute 80% of all nodes to Germany and 20% to the rest of the countries.
 simplify_network,,,
 -- to_substations,bool,"{'true','false'}","Aggregates all nodes without power injection (positive or negative, i.e. demand or generation) to electrically closest ones"
 -- algorithm,str,"One of {‘kmeans’, ‘hac’, ‘modularity‘}",

doc/configtables/licenses.csv

@@ -5,7 +5,7 @@
 "naturalearth/*",,,,,http://www.naturalearthdata.com/about/terms-of-use/
 "NUTS_2013 _60M_SH/*","x","x",,"x",https://ec.europa.eu/eurostat/web/gisco/geodata/reference-data/administrative-units-statistical-units
 "cantons.csv","x",,"x",,https://en.wikipedia.org/wiki/Data_codes_for_Switzerland
-"EIA_hydro_generation _2000_2014.csv","x",,,,https://www.eia.gov/about/copyrights_reuse.php
+"eia_hydro_annual_generation.csv","x",,,,https://www.eia.gov/about/copyrights_reuse.php
 "GEBCO_2014_2D.nc","x",,,,https://www.gebco.net/data_and_products/gridded_bathymetry_data/documents/gebco_2014_historic.pdf
 "hydro_capacities.csv","x",,,,
 "je-e-21.03.02.xls","x","x",,,https://www.bfs.admin.ch/bfs/en/home/fso/swiss-federal-statistical-office/terms-of-use.html

doc/configtables/lines.csv

@@ -5,6 +5,7 @@ s_nom_max,MW,"float","Global upper limit for the maximum capacity of each extend
 max_extension,MW,"float","Upper limit for the extended capacity of each extendable line."
 length_factor,--,float,"Correction factor to account for the fact that buses are *not* connected by lines through air-line distance."
 under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}","Specifies how to handle lines which are currently under construction."
+reconnect_crimea,--,"true or false","Whether to reconnect Crimea to the Ukrainian grid"
 dynamic_line_rating,,,
 -- activate,bool,"true or false","Whether to take dynamic line rating into account"
 -- cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."

doc/configtables/offwind-ac.csv

@@ -7,10 +7,10 @@ capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of wind turbine place
 correction_factor,--,float,"Correction factor for capacity factor time series."
 excluder_resolution,m,float,"Resolution on which to perform geographical elibility analysis."
 corine,--,"Any *realistic* subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes which are generally eligible for AC-connected offshore wind turbine placement."
+luisa,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes which are generally eligible for AC-connected offshore wind turbine placement."
 natura,bool,"{true, false}","Switch to exclude `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas. Area is excluded if ``true``."
 ship_threshold,--,float,"Ship density threshold from which areas are excluded."
 max_depth,m,float,"Maximum sea water depth at which wind turbines can be build. Maritime areas with deeper waters are excluded in the process of calculating the AC-connected offshore wind potential."
 min_shore_distance,m,float,"Minimum distance to the shore below which wind turbines cannot be build. Such areas close to the shore are excluded in the process of calculating the AC-connected offshore wind potential."
 max_shore_distance,m,float,"Maximum distance to the shore above which wind turbines cannot be build. Such areas close to the shore are excluded in the process of calculating the AC-connected offshore wind potential."
-potential,--,"One of {'simple', 'conservative'}","Method to compute the maximal installable potential for a node; confer :ref:`renewableprofiles`"
 clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero."

doc/configtables/offwind-dc.csv

@@ -7,10 +7,10 @@ capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of wind turbine place
 correction_factor,--,float,"Correction factor for capacity factor time series."
 excluder_resolution,m,float,"Resolution on which to perform geographical elibility analysis."
 corine,--,"Any *realistic* subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes which are generally eligible for AC-connected offshore wind turbine placement."
+luisa,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes which are generally eligible for DC-connected offshore wind turbine placement."
 natura,bool,"{true, false}","Switch to exclude `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas. Area is excluded if ``true``."
 ship_threshold,--,float,"Ship density threshold from which areas are excluded."
 max_depth,m,float,"Maximum sea water depth at which wind turbines can be build. Maritime areas with deeper waters are excluded in the process of calculating the AC-connected offshore wind potential."
 min_shore_distance,m,float,"Minimum distance to the shore below which wind turbines cannot be build."
 max_shore_distance,m,float,"Maximum distance to the shore above which wind turbines cannot be build."
-potential,--,"One of {'simple', 'conservative'}","Method to compute the maximal installable potential for a node; confer :ref:`renewableprofiles`"
 clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero."

doc/configtables/onwind.csv

@@ -8,8 +8,11 @@ corine,,,
 -- grid_codes,--,"Any subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes which are generally eligible for wind turbine placement."
 -- distance,m,float,"Distance to keep from areas specified in ``distance_grid_codes``"
 -- distance_grid_codes,--,"Any subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes to which wind turbines must maintain a distance specified in the setting ``distance``."
+luisa,,,
+-- grid_codes,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes which are generally eligible for wind turbine placement."
+-- distance,m,float,"Distance to keep from areas specified in ``distance_grid_codes``"
+-- distance_grid_codes,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes to which wind turbines must maintain a distance specified in the setting ``distance``."
 natura,bool,"{true, false}","Switch to exclude `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas. Area is excluded if ``true``."
-potential,--,"One of {'simple', 'conservative'}","Method to compute the maximal installable potential for a node; confer :ref:`renewableprofiles`"
 clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero."
 correction_factor,--,float,"Correction factor for capacity factor time series."
 excluder_resolution,m,float,"Resolution on which to perform geographical elibility analysis."

doc/configtables/sector.csv

@@ -71,7 +71,6 @@ solar_thermal,--,"{true, false}",Add option for using solar thermal to generate
 solar_cf_correction,--,float,The correction factor for the value provided by the solar thermal profile calculations
 marginal_cost_storage,currency/MWh ,float,The marginal cost of discharging batteries in distributed grids
 methanation,--,"{true, false}",Add option for transforming hydrogen and CO2 into methane using methanation.
-helmeth,--,"{true, false}",Add option for transforming power into gas using HELMETH (Integrated High-Temperature ELectrolysis and METHanation for Effective Power to Gas Conversion)
 coal_cc,--,"{true, false}",Add option for coal CHPs with carbon capture
 dac,--,"{true, false}",Add option for Direct Air Capture (DAC)
 co2_vent,--,"{true, false}",Add option for vent out CO2 from storages to the atmosphere.
@@ -108,6 +107,11 @@ electricity_distribution _grid,--,"{true, false}",Add a simplified representatio
 electricity_distribution _grid_cost_factor,,,Multiplies the investment cost of the electricity distribution grid
 ,,,
 electricity_grid _connection,--,"{true, false}",Add the cost of electricity grid connection for onshore wind and solar
+transmission_efficiency,,,Section to specify transmission losses or compression energy demands of bidirectional links. Splits them into two capacity-linked unidirectional links.
+-- {carrier},--,str,The carrier of the link.
+-- -- efficiency_static,p.u.,float,Length-independent transmission efficiency.
+-- -- efficiency_per_1000km,p.u. per 1000 km,float,Length-dependent transmission efficiency ($\eta^{\text{length}}$)
+-- -- compression_per_1000km,p.u. per 1000 km,float,Length-dependent electricity demand for compression ($\eta \cdot \text{length}$) implemented as multi-link to local electricity bus.
 H2_network,--,"{true, false}",Add option for new hydrogen pipelines
 gas_network,--,"{true, false}","Add existing natural gas infrastructure, incl. LNG terminals, production and entry-points. The existing gas network is added with a lossless transport model. A length-weighted `k-edge augmentation algorithm   <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_   can be run to add new candidate gas pipelines such that all regions of the model can be connected to the gas network. When activated, all the gas demands are regionally disaggregated as well."
 H2_retrofit,--,"{true, false}",Add option for retrofiting existing pipelines to transport hydrogen.
@@ -118,6 +122,7 @@ gas_distribution_grid _cost_factor,,,Multiplier for the investment cost of the g
 ,,,
 biomass_spatial,--,"{true, false}",Add option for resolving biomass demand regionally
 biomass_transport,--,"{true, false}",Add option for transporting solid biomass between nodes
+biogas_upgrading_cc,--,"{true, false}",Add option to capture CO2 from biomass upgrading
 conventional_generation,,,Add a more detailed description of conventional carriers. Any power generation requires the consumption of fuel from nodes representing that fuel.
 biomass_to_liquid,--,"{true, false}",Add option for transforming solid biomass into liquid fuel with the same properties as oil
 biosng,--,"{true, false}",Add option for transforming solid biomass into synthesis gas with the same properties as natural gas

doc/configtables/solar.csv

@@ -9,7 +9,7 @@ resource,,,
 capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of solar panel placement."
 correction_factor,--,float,"A correction factor for the capacity factor (availability) time series."
 corine,--,"Any subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes which are generally eligible for solar panel placement."
+luisa,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes which are generally eligible for solar panel placement."
 natura,bool,"{true, false}","Switch to exclude `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas. Area is excluded if ``true``."
-potential,--,"One of {'simple', 'conservative'}","Method to compute the maximal installable potential for a node; confer :ref:`renewableprofiles`"
 clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero."
 excluder_resolution,m,float,"Resolution on which to perform geographical elibility analysis."

doc/configtables/solving.csv

@@ -6,6 +6,7 @@ options,,,
 -- skip_iterations,bool,"{'true','false'}","Skip iterating, do not update impedances of branches. Defaults to true."
 -- rolling_horizon,bool,"{'true','false'}","Whether to optimize the network in a rolling horizon manner, where the snapshot range is split into slices of size `horizon` which are solved consecutively."
 -- seed,--,int,Random seed for increased deterministic behaviour.
+-- custom_extra_functionality,--,str,Path to a Python file with custom extra functionality code to be injected into the solving rules of the workflow relative to ``rules`` directory.
 -- track_iterations,bool,"{'true','false'}",Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration)
 -- min_iterations,--,int,Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.
 -- max_iterations,--,int,Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.

doc/index.rst

@@ -116,7 +116,7 @@ of the individual parts.
     topics we are working on. Please feel free to help or make suggestions.
 
 This project is currently maintained by the `Department of Digital
-Transformation in Energy Systems <https:/www.ensys.tu-berlin.de>`_ at the
+Transformation in Energy Systems <https://www.tu.berlin/en/ensys>`_ at the
 `Technische Universität Berlin <https://www.tu.berlin>`_. Previous versions were
 developed within the `IAI <http://www.iai.kit.edu>`_ at the `Karlsruhe Institute
 of Technology (KIT) <http://www.kit.edu/english/index.php>`_ which was funded by
@@ -185,7 +185,7 @@ For sector-coupling studies: ::
         pages = "1--25"
         year = "2023",
         eprint = "2207.05816",
-        doi = "10.1016/j.joule.2022.04.016",
+        doi = "10.1016/j.joule.2023.06.016",
     }
 
 For sector-coupling studies with pathway optimisation: ::

doc/release_notes.rst

@@ -10,6 +10,16 @@ Release Notes
 Upcoming Release
 ================
 
+* Add option to specify losses for bidirectional links, e.g. pipelines or HVDC
+  links, in configuration file under ``sector: transmission_efficiency:``. Users
+  can specify static or length-dependent values as well as a length-dependent
+  electricity demand for compression, which is implemented as a multi-link to
+  the local electricity buses. The bidirectional links will then be split into
+  two unidirectional links with linked capacities.
+
+* Pin ``snakemake`` version to below 8.0.0, as the new version is not yet
+  supported by ``pypsa-eur``.
+
 * Updated Global Energy Monitor LNG terminal data to March 2023 version.
 
 * For industry distribution, use EPRTR as fallback if ETS data is not available.
@@ -31,13 +41,76 @@ Upcoming Release
 
 * Rule ``retrieve_irena`` get updated values for renewables capacities.
 
+* Rule ``retrieve_wdpa`` updated to not only check for current and previous, but also potentially next months dataset availability.
+
 * Split configuration to enable SMR and SMR CC.
 
+* Bugfix: The unit of the capital cost of Haber-Bosch plants was corrected.
 
+* The configuration setting for country focus weights when clustering the
+  network has been moved from ``focus_weights:`` to ``clustering:
+  focus_weights:``. Backwards compatibility to old config files is maintained.
+
+* Extend options for waste usage from Haber-Bosch, methanolisation and methanation.
+
+* Use electrolysis waste heat by default.
+
+* Add new ``sector_opts`` wildcard option "nowasteheat" to disable all waste heat usage.
+
+* Set minimum part loads for PtX processes to 30% for methanolisation and methanation, and to 70% for Fischer-Tropsch synthesis.
+
+* Add VOM as marginal cost to PtX processes.
+
+* Add pelletizing costs for biomass boilers.
+
+* The ``mock_snakemake`` function can now be used with a Snakefile from a different directory using the new ``root_dir`` argument.
+
+* Switch to using hydrogen and electricity inputs for Haber-Bosch from https://github.com/PyPSA/technology-data.
+
+* Add option to capture CO2 contained in biogas when upgrading (``sector: biogas_to_gas_cc``).
+
+* Merged option to extend geographical scope to Ukraine and Moldova. These
+  countries are excluded by default and is currently constrained to power-sector
+  only parts of the workflow. A special config file
+  `config/config.entsoe-all.yaml` was added as an example to run the workflow
+  with all ENTSO-E member countries (including observer members like Ukraine and
+  Moldova). Moldova can currently only be included in conjunction with Ukraine
+  due to the absence of demand data. The Crimean power system is manually
+  reconnected to the main Ukrainian grid with the configuration option
+  `reconnect_crimea`.
+
+* Add option to reference an additional source file where users can specify
+  custom ``extra_functionality`` constraints in the configuration file. The
+  default setting points to an empty hull at
+  ``data/custom_extra_functionality.py``.
+
+* Validate downloads from Zenodo using MD5 checksums. This identifies corrupted
+  or incomplete downloads.
+
+* Add locations, capacities and costs of existing gas storage using Global
+  Energy Monitor's `Europe Gas Tracker
+  <https://globalenergymonitor.org/projects/europe-gas-tracker>`_.
+
+* Remove HELMETH option.
+
+* Print Irreducible Infeasible Subset (IIS) if model is infeasible. Only for
+  solvers with IIS support.
+
+* Add option to use `LUISA Base Map
+  <https://publications.jrc.ec.europa.eu/repository/handle/JRC124621>`_ 50m land
+  coverage dataset for land eligibility analysis in
+  :mod:`build_renewable_profiles`. Settings are analogous to the CORINE dataset
+  but with the key ``luisa:`` in the configuration file. To leverage the
+  dataset's full advantages, set the excluder resolution to 50m
+  (``excluder_resolution: 50``). For land category codes, see `Annex 1 of the
+  technical documentation
+  <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_.
 
 **Bugs and Compatibility**
 
 * A bug preventing custom powerplants specified in ``data/custom_powerplants.csv`` was fixed. (https://github.com/PyPSA/pypsa-eur/pull/732)
+* Fix nodal fraction in ``add_existing_year`` when using distributed generators
+* Fix typo in buses definition for oil boilers in ``add_industry`` in ``prepare_sector_network``
 
 
 PyPSA-Eur 0.8.1 (27th July 2023)
@@ -163,6 +236,8 @@ PyPSA-Eur 0.8.1 (27th July 2023)
   (https://github.com/PyPSA/pypsa-eur/pull/672)
 
 
+* Addressed deprecation warnings for ``pandas=2.0``. ``pandas=2.0`` is now minimum requirement.
+
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================
 

doc/retrieve.rst

@@ -22,11 +22,11 @@ Rule ``retrieve_databundle``
 Rule ``retrieve_cutout``
 ============================
 
-.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517949.svg
-   :target: https://doi.org/10.5281/zenodo.3517949
+.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.6382570.svg
+   :target: https://doi.org/10.5281/zenodo.6382570
 
 Cutouts are spatio-temporal subsets of the European weather data from the `ECMWF ERA5 <https://software.ecmwf.int/wiki/display/CKB/ERA5+data+documentation>`_ reanalysis dataset and the `CMSAF SARAH-2 <https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=SARAH_V002>`_ solar surface radiation dataset for the year 2013.
-They have been prepared by and are for use with the `atlite <https://github.com/PyPSA/atlite>`_ tool. You can either generate them yourself using the ``build_cutouts`` rule or retrieve them directly from `zenodo <https://doi.org/10.5281/zenodo.3517949>`__ through the rule ``retrieve_cutout``.
+They have been prepared by and are for use with the `atlite <https://github.com/PyPSA/atlite>`_ tool. You can either generate them yourself using the ``build_cutouts`` rule or retrieve them directly from `zenodo <https://doi.org/10.5281/zenodo.6382570>`__ through the rule ``retrieve_cutout``.
 The :ref:`tutorial` uses a smaller cutout than required for the full model (30 MB), which is also automatically downloaded.
 
 .. note::
@@ -91,7 +91,7 @@ None.
 
 **Outputs**
 
-- ``data/load_raw.csv``
+- ``resources/load_raw.csv``
 
 
 Rule ``retrieve_cost_data``

doc/tutorial.rst

@@ -25,7 +25,7 @@ full model, which allows the user to explore most of its functionalities on a
 local machine. The tutorial will cover examples on how to configure and
 customise the PyPSA-Eur model and run the ``snakemake`` workflow step by step
 from network creation to the solved network. The configuration for the tutorial
-is located at ``test/config.electricity.yaml``. It includes parts deviating from
+is located at ``config/test/config.electricity.yaml``. It includes parts deviating from
 the default config file ``config/config.default.yaml``. To run the tutorial with this
 configuration, execute
 
@@ -96,7 +96,7 @@ open-source solver GLPK.
    :start-at: solver:
    :end-before: plotting:
 
-Note, that ``test/config.electricity.yaml`` only includes changes relative to
+Note, that ``config/test/config.electricity.yaml`` only includes changes relative to
 the default configuration. There are many more configuration options, which are
 documented at :ref:`config`.
 

envs/environment.yaml

@@ -11,6 +11,8 @@ dependencies:
 - pip
 
 - atlite>=0.2.9
+- pypsa>=0.26.1
+- linopy
 - dask
 
   # Dependencies of the workflow itself
@@ -18,23 +20,24 @@ dependencies:
 - openpyxl!=3.1.1
 - pycountry
 - seaborn
-- snakemake-minimal>=7.7.0
+  # snakemake 8 introduced a number of breaking changes which the workflow has yet to be made compatible with
+- snakemake-minimal>=7.7.0,<8.0.0
 - memory_profiler
 - yaml
 - pytables
 - lxml
 - powerplantmatching>=0.5.5
 - numpy
-- pandas>=1.4
+- pandas>=2.1
 - geopandas>=0.11.0
-- xarray<=2023.8.0
+- xarray>=2023.11.0
 - rioxarray
 - netcdf4
 - networkx
 - scipy
 - shapely>=2.0
 - pyomo
-- matplotlib<3.6
+- matplotlib
 - proj
 - fiona
 - country_converter
@@ -44,6 +47,7 @@ dependencies:
 - tabula-py
 - pyxlsb
 - graphviz
+- ipopt
 
   # Keep in conda environment when calling ipython
 - ipython
@@ -56,4 +60,3 @@ dependencies:
 
 - pip:
   - tsam>=2.3.1
-  - pypsa>=0.25.2

rules/build_electricity.smk

@@ -24,7 +24,7 @@ rule build_electricity_demand:
         countries=config["countries"],
         load=config["load"],
     input:
-        ancient("data/load_raw.csv"),
+        ancient(RESOURCES + "load_raw.csv"),
     output:
         RESOURCES + "load.csv",
     log:
@@ -206,10 +206,61 @@ rule build_ship_raster:
         "../scripts/build_ship_raster.py"
 
 
+rule determine_availability_matrix_MD_UA:
+    input:
+        copernicus="data/Copernicus_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif",
+        wdpa="data/WDPA.gpkg",
+        wdpa_marine="data/WDPA_WDOECM_marine.gpkg",
+        gebco=lambda w: (
+            "data/bundle/GEBCO_2014_2D.nc"
+            if "max_depth" in config["renewable"][w.technology].keys()
+            else []
+        ),
+        ship_density=lambda w: (
+            RESOURCES + "shipdensity_raster.tif"
+            if "ship_threshold" in config["renewable"][w.technology].keys()
+            else []
+        ),
+        country_shapes=RESOURCES + "country_shapes.geojson",
+        offshore_shapes=RESOURCES + "offshore_shapes.geojson",
+        regions=lambda w: (
+            RESOURCES + "regions_onshore.geojson"
+            if w.technology in ("onwind", "solar")
+            else RESOURCES + "regions_offshore.geojson"
+        ),
+        cutout=lambda w: "cutouts/"
+        + CDIR
+        + config["renewable"][w.technology]["cutout"]
+        + ".nc",
+    output:
+        availability_matrix=RESOURCES + "availability_matrix_MD-UA_{technology}.nc",
+        availability_map=RESOURCES + "availability_matrix_MD-UA_{technology}.png",
+    log:
+        LOGS + "determine_availability_matrix_MD_UA_{technology}.log",
+    threads: ATLITE_NPROCESSES
+    resources:
+        mem_mb=ATLITE_NPROCESSES * 5000,
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/determine_availability_matrix_MD_UA.py"
+
+
+# Optional input when having Ukraine (UA) or Moldova (MD) in the countries list
+if {"UA", "MD"}.intersection(set(config["countries"])):
+    opt = {
+        "availability_matrix_MD_UA": RESOURCES
+        + "availability_matrix_MD-UA_{technology}.nc"
+    }
+else:
+    opt = {}
+
+
 rule build_renewable_profiles:
     params:
         renewable=config["renewable"],
     input:
+        **opt,
         base_network=RESOURCES + "networks/base.nc",
         corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"),
         natura=lambda w: (
@@ -217,6 +268,11 @@ rule build_renewable_profiles:
             if config["renewable"][w.technology]["natura"]
             else []
         ),
+        luisa=lambda w: (
+            "data/LUISA_basemap_020321_50m.tif"
+            if config["renewable"][w.technology].get("luisa")
+            else []
+        ),
         gebco=ancient(
             lambda w: (
                 "data/bundle/GEBCO_2014_2D.nc"
@@ -226,7 +282,7 @@ rule build_renewable_profiles:
         ),
         ship_density=lambda w: (
             RESOURCES + "shipdensity_raster.tif"
-            if "ship_threshold" in config["renewable"][w.technology].keys()
+            if config["renewable"][w.technology].get("ship_threshold", False)
             else []
         ),
         country_shapes=RESOURCES + "country_shapes.geojson",
@@ -355,6 +411,7 @@ rule add_electricity:
         else [],
         load=RESOURCES + "load.csv",
         nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
+        ua_md_gdp="data/GDP_PPP_30arcsec_v3_mapped_default.csv",
     output:
         RESOURCES + "networks/elec.nc",
     log:
@@ -374,7 +431,9 @@ rule simplify_network:
     params:
         simplify_network=config["clustering"]["simplify_network"],
         aggregation_strategies=config["clustering"].get("aggregation_strategies", {}),
-        focus_weights=config.get("focus_weights", None),
+        focus_weights=config["clustering"].get(
+            "focus_weights", config.get("focus_weights")
+        ),
         renewable_carriers=config["electricity"]["renewable_carriers"],
         max_hours=config["electricity"]["max_hours"],
         length_factor=config["lines"]["length_factor"],
@@ -409,7 +468,9 @@ rule cluster_network:
         cluster_network=config["clustering"]["cluster_network"],
         aggregation_strategies=config["clustering"].get("aggregation_strategies", {}),
         custom_busmap=config["enable"].get("custom_busmap", False),
-        focus_weights=config.get("focus_weights", None),
+        focus_weights=config["clustering"].get(
+            "focus_weights", config.get("focus_weights")
+        ),
         renewable_carriers=config["electricity"]["renewable_carriers"],
         conventional_carriers=config["electricity"].get("conventional_carriers", []),
         max_hours=config["electricity"]["max_hours"],

rules/build_sector.smk

@@ -85,12 +85,12 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
 
     rule build_gas_input_locations:
         input:
-            lng=HTTP.remote(
+            gem=HTTP.remote(
                 "https://globalenergymonitor.org/wp-content/uploads/2023/07/Europe-Gas-Tracker-2023-03-v3.xlsx",
                 keep_local=True,
             ),
             entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson",
-            production="data/gas_network/scigrid-gas/data/IGGIELGN_Productions.geojson",
+            storage="data/gas_network/scigrid-gas/data/IGGIELGN_Storages.geojson",
             regions_onshore=RESOURCES
             + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
             regions_offshore=RESOURCES
@@ -269,7 +269,7 @@ rule build_biomass_potentials:
         biomass=config["biomass"],
     input:
         enspreso_biomass=HTTP.remote(
-            "https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx",
+            "https://zenodo.org/records/10356004/files/ENSPRESO_BIOMASS.xlsx",
             keep_local=True,
         ),
         nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",  # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21

rules/common.smk

@@ -2,6 +2,11 @@
 #
 # SPDX-License-Identifier: MIT
 
+import os, sys
+
+sys.path.insert(0, os.path.abspath("scripts"))
+from _helpers import validate_checksum
+
 
 def memory(w):
     factor = 3.0
@@ -23,6 +28,13 @@ def memory(w):
         return int(factor * (10000 + 195 * int(w.clusters)))
 
 
+def input_custom_extra_functionality(w):
+    path = config["solving"]["options"].get("custom_extra_functionality", False)
+    if path:
+        return workflow.source_path(path)
+    return []
+
+
 # Check if the workflow has access to the internet by trying to access the HEAD of specified url
 def has_internet_access(url="www.zenodo.org") -> bool:
     import http.client as http_client

rules/postprocess.smk

@@ -5,7 +5,6 @@
 
 localrules:
     copy_config,
-    copy_conda_env,
 
 
 if config["foresight"] != "perfect":

rules/retrieve.smk

@@ -2,6 +2,9 @@
 #
 # SPDX-License-Identifier: MIT
 
+import requests
+from datetime import datetime, timedelta
+
 if config["enable"].get("retrieve", "auto") == "auto":
     config["enable"]["retrieve"] = has_internet_access()
 
@@ -74,6 +77,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True
         retries: 2
         run:
             move(input[0], output[0])
+            validate_checksum(output[0], input[0])
 
 
 if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", True):
@@ -110,7 +114,7 @@ if config["enable"]["retrieve"] and config["enable"].get(
                 static=True,
             ),
         output:
-            protected(RESOURCES + "natura.tiff"),
+            RESOURCES + "natura.tiff",
         log:
             LOGS + "retrieve_natura_raster.log",
         resources:
@@ -118,6 +122,7 @@ if config["enable"]["retrieve"] and config["enable"].get(
         retries: 2
         run:
             move(input[0], output[0])
+            validate_checksum(output[0], input[0])
 
 
 if config["enable"]["retrieve"] and config["enable"].get(
@@ -164,6 +169,7 @@ if config["enable"]["retrieve"] and (
         "IGGIELGN_LNGs.geojson",
         "IGGIELGN_BorderPoints.geojson",
         "IGGIELGN_Productions.geojson",
+        "IGGIELGN_Storages.geojson",
         "IGGIELGN_PipeSegments.geojson",
     ]
 
@@ -195,7 +201,7 @@ if config["enable"]["retrieve"]:
                 static=True,
             ),
         output:
-            "data/load_raw.csv",
+            RESOURCES + "load_raw.csv",
         log:
             LOGS + "retrieve_electricity_demand.log",
         resources:
@@ -223,6 +229,122 @@ if config["enable"]["retrieve"]:
         retries: 2
         run:
             move(input[0], output[0])
+            validate_checksum(output[0], input[0])
+
+
+if config["enable"]["retrieve"]:
+
+    # Downloading Copernicus Global Land Cover for land cover and land use:
+    # Website: https://land.copernicus.eu/global/products/lc
+    rule download_copernicus_land_cover:
+        input:
+            HTTP.remote(
+                "zenodo.org/record/3939050/files/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif",
+                static=True,
+            ),
+        output:
+            "data/Copernicus_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif",
+        run:
+            move(input[0], output[0])
+            validate_checksum(output[0], input[0])
+
+
+if config["enable"]["retrieve"]:
+
+    # Downloading LUISA Base Map for land cover and land use:
+    # Website: https://ec.europa.eu/jrc/en/luisa
+    rule retrieve_luisa_land_cover:
+        input:
+            HTTP.remote(
+                "jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/LUISA/EUROPE/Basemaps/LandUse/2018/LATEST/LUISA_basemap_020321_50m.tif",
+                static=True,
+            ),
+        output:
+            "data/LUISA_basemap_020321_50m.tif",
+        run:
+            move(input[0], output[0])
+
+
+if config["enable"]["retrieve"]:
+    # Some logic to find the correct file URL
+    # Sometimes files are released delayed or ahead of schedule, check which file is currently available
+
+    def check_file_exists(url):
+        response = requests.head(url)
+        return response.status_code == 200
+
+    # Basic pattern where WDPA files can be found
+    url_pattern = (
+        "https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{bYYYY}_Public_shp.zip"
+    )
+
+    # 3-letter month + 4 digit year for current/previous/next month to test
+    current_monthyear = datetime.now().strftime("%b%Y")
+    prev_monthyear = (datetime.now() - timedelta(30)).strftime("%b%Y")
+    next_monthyear = (datetime.now() + timedelta(30)).strftime("%b%Y")
+
+    # Test prioritised: current month -> previous -> next
+    for bYYYY in [current_monthyear, prev_monthyear, next_monthyear]:
+        if check_file_exists(url := url_pattern.format(bYYYY=bYYYY)):
+            break
+        else:
+            # If None of the three URLs are working
+            url = False
+
+    assert (
+        url
+    ), f"No WDPA files found at {url_pattern} for bY='{current_monthyear}, {prev_monthyear}, or {next_monthyear}'"
+
+    # Downloading protected area database from WDPA
+    # extract the main zip and then merge the contained 3 zipped shapefiles
+    # Website: https://www.protectedplanet.net/en/thematic-areas/wdpa
+    rule download_wdpa:
+        input:
+            HTTP.remote(
+                url,
+                static=True,
+                keep_local=True,
+            ),
+        params:
+            zip="data/WDPA_shp.zip",
+            folder=directory("data/WDPA"),
+        output:
+            gpkg=protected("data/WDPA.gpkg"),
+        run:
+            shell("cp {input} {params.zip}")
+            shell("unzip -o {params.zip} -d {params.folder}")
+            for i in range(3):
+                # vsizip is special driver for directly working with zipped shapefiles in ogr2ogr
+                layer_path = (
+                    f"/vsizip/{params.folder}/WDPA_{bYYYY}_Public_shp_{i}.zip"
+                )
+                print(f"Adding layer {i+1} of 3 to combined output file.")
+                shell("ogr2ogr -f gpkg -update -append {output.gpkg} {layer_path}")
+
+    rule download_wdpa_marine:
+        # Downloading Marine protected area database from WDPA
+        # extract the main zip and then merge the contained 3 zipped shapefiles
+        # Website: https://www.protectedplanet.net/en/thematic-areas/marine-protected-areas
+        input:
+            HTTP.remote(
+                f"d1gam3xoknrgr2.cloudfront.net/current/WDPA_WDOECM_{bYYYY}_Public_marine_shp.zip",
+                static=True,
+                keep_local=True,
+            ),
+        params:
+            zip="data/WDPA_WDOECM_marine.zip",
+            folder=directory("data/WDPA_WDOECM_marine"),
+        output:
+            gpkg=protected("data/WDPA_WDOECM_marine.gpkg"),
+        run:
+            shell("cp {input} {params.zip}")
+            shell("unzip -o {params.zip} -d {params.folder}")
+            for i in range(3):
+                # vsizip is special driver for directly working with zipped shapefiles in ogr2ogr
+                layer_path = f"/vsizip/{params.folder}/WDPA_WDOECM_{bYYYY}_Public_marine_shp_{i}.zip"
+                print(f"Adding layer {i+1} of 3 to combined output file.")
+                shell("ogr2ogr -f gpkg -update -append {output.gpkg} {layer_path}")
+
 
 
 if config["enable"]["retrieve"]:

rules/solve_electricity.smk

@@ -11,6 +11,7 @@ rule solve_network:
         co2_sequestration_potential=config["sector"].get(
             "co2_sequestration_potential", 200
         ),
+        custom_extra_functionality=input_custom_extra_functionality,
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
         config=RESULTS + "config.yaml",

rules/solve_myopic.smk

@@ -88,13 +88,12 @@ rule solve_sector_network_myopic:
         co2_sequestration_potential=config["sector"].get(
             "co2_sequestration_potential", 200
         ),
-        countries=config["countries"],
+        custom_extra_functionality=input_custom_extra_functionality,
     input:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         costs="data/costs_{planning_horizons}.csv",
         config=RESULTS + "config.yaml",
-        co2_totals_name=RESOURCES + "co2_totals.csv",
     output:
         RESULTS
         + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",

rules/solve_overnight.smk

@@ -11,6 +11,7 @@ rule solve_sector_network:
         co2_sequestration_potential=config["sector"].get(
             "co2_sequestration_potential", 200
         ),
+        custom_extra_functionality=input_custom_extra_functionality,
     input:
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",

rules/solve_perfect.smk

@@ -118,6 +118,7 @@ rule solve_sector_network_perfect:
         co2_sequestration_potential=config["sector"].get(
             "co2_sequestration_potential", 200
         ),
+        custom_extra_functionality=input_custom_extra_functionality,
     input:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",

scripts/_helpers.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: MIT
 
 import contextlib
+import hashlib
 import logging
 import os
 import urllib
@@ -11,6 +12,7 @@ from pathlib import Path
 
 import pandas as pd
 import pytz
+import requests
 import yaml
 from pypsa.components import component_attrs, components
 from pypsa.descriptors import Dict
@@ -191,7 +193,7 @@ def progress_retrieve(url, file, disable=False):
             urllib.request.urlretrieve(url, file, reporthook=update_to)
 
 
-def mock_snakemake(rulename, configfiles=[], **wildcards):
+def mock_snakemake(rulename, root_dir=None, configfiles=[], **wildcards):
     """
     This function is expected to be executed from the 'scripts'-directory of '
     the snakemake project. It returns a snakemake.script.Snakemake object,
@@ -203,6 +205,8 @@ def mock_snakemake(rulename, configfiles=[], **wildcards):
     ----------
     rulename: str
         name of the rule for which the snakemake object should be generated
+    root_dir: str/path-like
+        path to the root directory of the snakemake project
     configfiles: list, str
         list of configfiles to be used to update the config
     **wildcards:
@@ -217,7 +221,10 @@ def mock_snakemake(rulename, configfiles=[], **wildcards):
     from snakemake.script import Snakemake
 
     script_dir = Path(__file__).parent.resolve()
-    root_dir = script_dir.parent
+    if root_dir is None:
+        root_dir = script_dir.parent
+    else:
+        root_dir = Path(root_dir).resolve()
 
     user_in_script_dir = Path.cwd().resolve() == script_dir
     if user_in_script_dir:
@@ -313,3 +320,63 @@ def update_config_with_sector_opts(config, sector_opts):
         if o.startswith("CF+"):
             l = o.split("+")[1:]
             update_config(config, parse(l))
+
+
+def get_checksum_from_zenodo(file_url):
+    parts = file_url.split("/")
+    record_id = parts[parts.index("record") + 1]
+    filename = parts[-1]
+
+    response = requests.get(f"https://zenodo.org/api/records/{record_id}", timeout=30)
+    response.raise_for_status()
+    data = response.json()
+
+    for file in data["files"]:
+        if file["key"] == filename:
+            return file["checksum"]
+    return None
+
+
+def validate_checksum(file_path, zenodo_url=None, checksum=None):
+    """
+    Validate file checksum against provided or Zenodo-retrieved checksum.
+    Calculates the hash of a file using 64KB chunks. Compares it against a
+    given checksum or one from a Zenodo URL.
+
+    Parameters
+    ----------
+    file_path : str
+        Path to the file for checksum validation.
+    zenodo_url : str, optional
+        URL of the file on Zenodo to fetch the checksum.
+    checksum : str, optional
+        Checksum (format 'hash_type:checksum_value') for validation.
+
+    Raises
+    ------
+    AssertionError
+        If the checksum does not match, or if neither `checksum` nor `zenodo_url` is provided.
+
+
+    Examples
+    --------
+    >>> validate_checksum("/path/to/file", checksum="md5:abc123...")
+    >>> validate_checksum(
+    ...     "/path/to/file",
+    ...     zenodo_url="https://zenodo.org/record/12345/files/example.txt",
+    ... )
+
+    If the checksum is invalid, an AssertionError will be raised.
+    """
+    assert checksum or zenodo_url, "Either checksum or zenodo_url must be provided"
+    if zenodo_url:
+        checksum = get_checksum_from_zenodo(zenodo_url)
+    hash_type, checksum = checksum.split(":")
+    hasher = hashlib.new(hash_type)
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(65536), b""):  # 64kb chunks
+            hasher.update(chunk)
+    calculated_checksum = hasher.hexdigest()
+    assert (
+        calculated_checksum == checksum
+    ), "Checksum is invalid. This may be due to an incomplete download. Delete the file and re-execute the rule."

scripts/add_brownfield.py

@@ -124,31 +124,25 @@ def disable_grid_expansion_if_LV_limit_hit(n):
     if not "lv_limit" in n.global_constraints.index:
         return
 
-    # calculate minimum LV
-    attr = "nom_min"
-    dc = n.links.index[n.links.carrier == "DC"]
-    tot = (n.lines["s_" + attr] * n.lines["length"]).sum() + (
-        n.links.loc[dc, "p_" + attr] * n.links.loc[dc, "length"]
+    total_expansion = (
+        n.lines.eval("s_nom_min * length").sum()
+        + n.links.query("carrier == 'DC'").eval("p_nom_min * length").sum()
     ).sum()
 
-    diff = n.global_constraints.at["lv_limit", "constant"] - tot
+    lv_limit = n.global_constraints.at["lv_limit", "constant"]
 
     # allow small numerical differences
-    limit = 1
-
-    if diff < limit:
+    if lv_limit - total_expansion < 1:
         logger.info(
-            f"LV is already reached (gap {diff}), disabling expansion and LV limit"
+            f"LV is already reached (gap {diff} MWkm), disabling expansion and LV limit"
         )
-        expandable_acs = n.lines.index[n.lines.s_nom_extendable]
-        n.lines.loc[expandable_acs, "s_nom_extendable"] = False
-        n.lines.loc[expandable_acs, "s_nom"] = n.lines.loc[expandable_acs, "s_nom_min"]
+        extendable_acs = n.lines.query("s_nom_extendable").index
+        n.lines.loc[extendable_acs, "s_nom_extendable"] = False
+        n.lines.loc[extendable_acs, "s_nom"] = n.lines.loc[extendable_acs, "s_nom_min"]
 
-        expandable_dcs = n.links.index[
-            n.links.p_nom_extendable & (n.links.carrier == "DC")
-        ]
-        n.links.loc[expandable_dcs, "p_nom_extendable"] = False
-        n.links.loc[expandable_dcs, "p_nom"] = n.links.loc[expandable_dcs, "p_nom_min"]
+        extendable_dcs = n.links.query("carrier == 'DC' and p_nom_extendable").index
+        n.links.loc[extendable_dcs, "p_nom_extendable"] = False
+        n.links.loc[extendable_dcs, "p_nom"] = n.links.loc[extendable_dcs, "p_nom_min"]
 
         n.global_constraints.drop("lv_limit", inplace=True)
 

scripts/add_electricity.py

@@ -84,6 +84,7 @@ It further adds extendable ``generators`` with **zero** capacity for
 
 import logging
 from itertools import product
+from typing import Dict, List
 
 import geopandas as gpd
 import numpy as np
@@ -255,6 +256,7 @@ def load_powerplants(ppl_fn):
         "bioenergy": "biomass",
         "ccgt, thermal": "CCGT",
         "hard coal": "coal",
+        "natural gas": "OCGT",
     }
     return (
         pd.read_csv(ppl_fn, index_col=0, dtype={"bus": "str"})
@@ -279,11 +281,13 @@ def shapes_to_shapes(orig, dest):
     return transfer
 
 
-def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0):
+def attach_load(n, regions, load, nuts3_shapes, ua_md_gdp, countries, scaling=1.0):
     substation_lv_i = n.buses.index[n.buses["substation_lv"]]
     regions = gpd.read_file(regions).set_index("name").reindex(substation_lv_i)
     opsd_load = pd.read_csv(load, index_col=0, parse_dates=True).filter(items=countries)
 
+    ua_md_gdp = pd.read_csv(ua_md_gdp, dtype={"name": "str"}).set_index("name")
+
     logger.info(f"Load data scaled with scalling factor {scaling}.")
     opsd_load *= scaling
 
@@ -291,6 +295,7 @@ def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0):
 
     def upsample(cntry, group):
         l = opsd_load[cntry]
+
         if len(group) == 1:
             return pd.DataFrame({group.index[0]: l})
         nuts3_cntry = nuts3.loc[nuts3.country == cntry]
@@ -305,6 +310,9 @@ def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0):
         # relative factors 0.6 and 0.4 have been determined from a linear
         # regression on the country to continent load data
         factors = normed(0.6 * normed(gdp_n) + 0.4 * normed(pop_n))
+        if cntry in ["UA", "MD"]:
+            # overwrite factor because nuts3 provides no data for UA+MD
+            factors = normed(ua_md_gdp.loc[group.index, "GDP_PPP"].squeeze())
         return pd.DataFrame(
             factors.values * l.values[:, np.newaxis],
             index=l.index,
@@ -710,7 +718,17 @@ def attach_extendable_generators(n, costs, ppl, carriers):
             )
 
 
-def attach_OPSD_renewables(n, tech_map):
+def attach_OPSD_renewables(n: pypsa.Network, tech_map: Dict[str, List[str]]) -> None:
+    """
+    Attach renewable capacities from the OPSD dataset to the network.
+
+    Args:
+    - n: The PyPSA network to attach the capacities to.
+    - tech_map: A dictionary mapping fuel types to carrier names.
+
+    Returns:
+    - None
+    """
     tech_string = ", ".join(sum(tech_map.values(), []))
     logger.info(f"Using OPSD renewable capacities for carriers {tech_string}.")
 
@@ -735,7 +753,26 @@ def attach_OPSD_renewables(n, tech_map):
         n.generators.p_nom_min.update(gens.bus.map(caps).dropna())
 
 
-def estimate_renewable_capacities(n, year, tech_map, expansion_limit, countries):
+def estimate_renewable_capacities(
+    n: pypsa.Network, year: int, tech_map: dict, expansion_limit: bool, countries: list
+) -> None:
+    """
+    Estimate a different between renewable capacities in the network and
+    reported country totals from IRENASTAT dataset. Distribute the difference
+    with a heuristic.
+
+    Heuristic: n.generators_t.p_max_pu.mean() * n.generators.p_nom_max
+
+    Args:
+    - n: The PyPSA network.
+    - year: The year of optimisation.
+    - tech_map: A dictionary mapping fuel types to carrier names.
+    - expansion_limit: Boolean value from config file
+    - countries: A list of country codes to estimate capacities for.
+
+    Returns:
+    - None
+    """
     if not len(countries) or not len(tech_map):
         return
 
@@ -752,7 +789,10 @@ def estimate_renewable_capacities(n, year, tech_map, expansion_limit, countries)
 
     for ppm_technology, techs in tech_map.items():
         tech_i = n.generators.query("carrier in @techs").index
-        stats = capacities.loc[ppm_technology].reindex(countries, fill_value=0.0)
+        if ppm_technology in capacities.index.get_level_values("Technology"):
+            stats = capacities.loc[ppm_technology].reindex(countries, fill_value=0.0)
+        else:
+            stats = pd.Series(0.0, index=countries)
         country = n.generators.bus[tech_i].map(n.buses.country)
         existent = n.generators.p_nom[tech_i].groupby(country).sum()
         missing = stats - existent
@@ -825,6 +865,7 @@ if __name__ == "__main__":
         snakemake.input.regions,
         snakemake.input.load,
         snakemake.input.nuts3_shapes,
+        snakemake.input.ua_md_gdp,
         params.countries,
         params.scaling_factor,
     )

scripts/add_existing_baseyear.py

@@ -88,7 +88,9 @@ def add_existing_renewables(df_agg):
             ]
             cfs = n.generators_t.p_max_pu[gens].mean()
             cfs_key = cfs / cfs.sum()
-            nodal_fraction.loc[n.generators.loc[gens, "bus"]] = cfs_key.values
+            nodal_fraction.loc[n.generators.loc[gens, "bus"]] = cfs_key.groupby(
+                n.generators.loc[gens, "bus"]
+            ).sum()
 
         nodal_df = df.loc[n.buses.loc[elec_buses, "country"]]
         nodal_df.index = elec_buses

scripts/base_network.py

@@ -366,6 +366,25 @@ def _apply_parameter_corrections(n, parameter_corrections):
                 df.loc[inds, attr] = r[inds].astype(df[attr].dtype)
 
 
+def _reconnect_crimea(lines):
+    logger.info("Reconnecting Crimea to the Ukrainian grid.")
+    lines_to_crimea = pd.DataFrame(
+        {
+            "bus0": ["3065", "3181", "3181"],
+            "bus1": ["3057", "3055", "3057"],
+            "v_nom": [300, 300, 300],
+            "num_parallel": [1, 1, 1],
+            "length": [140, 120, 140],
+            "carrier": ["AC", "AC", "AC"],
+            "underground": [False, False, False],
+            "under_construction": [False, False, False],
+        },
+        index=["Melitopol", "Liubymivka left", "Luibymivka right"],
+    )
+
+    return pd.concat([lines, lines_to_crimea])
+
+
 def _set_electrical_parameters_lines(lines, config):
     v_noms = config["electricity"]["voltages"]
     linetypes = config["lines"]["types"]
@@ -450,12 +469,12 @@ def _remove_dangling_branches(branches, buses):
     )
 
 
-def _remove_unconnected_components(network):
+def _remove_unconnected_components(network, threshold=6):
     _, labels = csgraph.connected_components(network.adjacency_matrix(), directed=False)
     component = pd.Series(labels, index=network.buses.index)
 
     component_sizes = component.value_counts()
-    components_to_remove = component_sizes.iloc[1:]
+    components_to_remove = component_sizes.loc[component_sizes < threshold]
 
     logger.info(
         f"Removing {len(components_to_remove)} unconnected network components with less than {components_to_remove.max()} buses. In total {components_to_remove.sum()} buses."
@@ -541,7 +560,7 @@ def _set_countries_and_substations(n, config, country_shapes, offshore_shapes):
         ~buses["under_construction"]
     )
 
-    c_nan_b = buses.country.isnull()
+    c_nan_b = buses.country == "na"
     if c_nan_b.sum() > 0:
         c_tag = _get_country(buses.loc[c_nan_b])
         c_tag.loc[~c_tag.isin(countries)] = np.nan
@@ -699,6 +718,9 @@ def base_network(
     lines = _load_lines_from_eg(buses, eg_lines)
     transformers = _load_transformers_from_eg(buses, eg_transformers)
 
+    if config["lines"].get("reconnect_crimea", True) and "UA" in config["countries"]:
+        lines = _reconnect_crimea(lines)
+
     lines = _set_electrical_parameters_lines(lines, config)
     transformers = _set_electrical_parameters_transformers(transformers, config)
     links = _set_electrical_parameters_links(links, config, links_p_nom)

scripts/build_biomass_potentials.py

@@ -134,7 +134,7 @@ def disaggregate_nuts0(bio):
     # get population in nuts2
     pop_nuts2 = pop.loc[pop.index.str.len() == 4]
     by_country = pop_nuts2.total.groupby(pop_nuts2.ct).sum()
-    pop_nuts2["fraction"] = pop_nuts2.total / pop_nuts2.ct.map(by_country)
+    pop_nuts2.loc[:, "fraction"] = pop_nuts2.total / pop_nuts2.ct.map(by_country)
 
     # distribute nuts0 data to nuts2 by population
     bio_nodal = bio.loc[pop_nuts2.ct]
@@ -263,7 +263,7 @@ if __name__ == "__main__":
     df.to_csv(snakemake.output.biomass_potentials_all)
 
     grouper = {v: k for k, vv in params["classes"].items() for v in vv}
-    df = df.groupby(grouper, axis=1).sum()
+    df = df.T.groupby(grouper).sum().T
 
     df *= 1e6  # TWh/a to MWh/a
     df.index.name = "MWh/a"

scripts/build_clustered_population_layouts.py

@@ -25,10 +25,7 @@ if __name__ == "__main__":
     cutout = atlite.Cutout(snakemake.input.cutout)
 
     clustered_regions = (
-        gpd.read_file(snakemake.input.regions_onshore)
-        .set_index("name")
-        .buffer(0)
-        .squeeze()
+        gpd.read_file(snakemake.input.regions_onshore).set_index("name").buffer(0)
     )
 
     I = cutout.indicatormatrix(clustered_regions)

scripts/build_electricity_demand.py

@@ -31,7 +31,7 @@ Relevant Settings
 Inputs
 ------
 
-- ``data/load_raw.csv``:
+- ``resources/load_raw.csv``:
 
 Outputs
 -------
@@ -81,7 +81,7 @@ def load_timeseries(fn, years, countries, powerstatistics=True):
         return s[: -len(pattern)]
 
     return (
-        pd.read_csv(fn, index_col=0, parse_dates=[0])
+        pd.read_csv(fn, index_col=0, parse_dates=[0], date_format="%Y-%m-%dT%H:%M:%SZ")
         .tz_localize(None)
         .filter(like=pattern)
         .rename(columns=rename)
@@ -155,7 +155,7 @@ def copy_timeslice(load, cntry, start, stop, delta, fn_load=None):
             ].values
 
 
-def manual_adjustment(load, fn_load, powerstatistics):
+def manual_adjustment(load, fn_load, powerstatistics, countries):
     """
     Adjust gaps manual for load data from OPSD time-series package.
 
@@ -278,6 +278,14 @@ def manual_adjustment(load, fn_load, powerstatistics):
             load, "LU", "2019-02-05 20:00", "2019-02-06 19:00", Delta(weeks=-1)
         )
 
+    if "UA" in countries:
+        copy_timeslice(
+            load, "UA", "2013-01-25 14:00", "2013-01-28 21:00", Delta(weeks=1)
+        )
+        copy_timeslice(
+            load, "UA", "2013-10-28 03:00", "2013-10-28 20:00", Delta(weeks=1)
+        )
+
     return load
 
 
@@ -298,8 +306,22 @@ if __name__ == "__main__":
 
     load = load_timeseries(snakemake.input[0], years, countries, powerstatistics)
 
+    if "UA" in countries:
+        # attach load of UA (best data only for entsoe transparency)
+        load_ua = load_timeseries(snakemake.input[0], "2018", ["UA"], False)
+        snapshot_year = str(snapshots.year.unique().item())
+        time_diff = pd.Timestamp("2018") - pd.Timestamp(snapshot_year)
+        load_ua.index -= (
+            time_diff  # hack indices (currently, UA is manually set to 2018)
+        )
+        load["UA"] = load_ua
+        # attach load of MD (no time-series available, use 2020-totals and distribute according to UA):
+        # https://www.iea.org/data-and-statistics/data-browser/?country=MOLDOVA&fuel=Energy%20consumption&indicator=TotElecCons
+        if "MD" in countries:
+            load["MD"] = 6.2e6 * (load_ua / load_ua.sum())
+
     if snakemake.params.load["manual_adjustments"]:
-        load = manual_adjustment(load, snakemake.input[0], powerstatistics)
+        load = manual_adjustment(load, snakemake.input[0], powerstatistics, countries)
 
     if load.empty:
         logger.warning("Build electricity demand time series is empty.")

scripts/build_energy_totals.py

@@ -189,12 +189,12 @@ def idees_per_country(ct, year, base_dir):
     ct_totals["total residential water"] = df.at["Water heating"]
 
     assert df.index[23] == "Electricity"
-    ct_totals["electricity residential water"] = df[23]
+    ct_totals["electricity residential water"] = df.iloc[23]
 
     ct_totals["total residential cooking"] = df["Cooking"]
 
     assert df.index[30] == "Electricity"
-    ct_totals["electricity residential cooking"] = df[30]
+    ct_totals["electricity residential cooking"] = df.iloc[30]
 
     df = pd.read_excel(fn_residential, "RES_summary", index_col=0)[year]
 
@@ -202,13 +202,13 @@ def idees_per_country(ct, year, base_dir):
     ct_totals["total residential"] = df[row]
 
     assert df.index[47] == "Electricity"
-    ct_totals["electricity residential"] = df[47]
+    ct_totals["electricity residential"] = df.iloc[47]
 
     assert df.index[46] == "Derived heat"
-    ct_totals["derived heat residential"] = df[46]
+    ct_totals["derived heat residential"] = df.iloc[46]
 
     assert df.index[50] == "Thermal uses"
-    ct_totals["thermal uses residential"] = df[50]
+    ct_totals["thermal uses residential"] = df.iloc[50]
 
     # services
 
@@ -222,12 +222,12 @@ def idees_per_country(ct, year, base_dir):
     ct_totals["total services water"] = df["Hot water"]
 
     assert df.index[24] == "Electricity"
-    ct_totals["electricity services water"] = df[24]
+    ct_totals["electricity services water"] = df.iloc[24]
 
     ct_totals["total services cooking"] = df["Catering"]
 
     assert df.index[31] == "Electricity"
-    ct_totals["electricity services cooking"] = df[31]
+    ct_totals["electricity services cooking"] = df.iloc[31]
 
     df = pd.read_excel(fn_tertiary, "SER_summary", index_col=0)[year]
 
@@ -235,13 +235,13 @@ def idees_per_country(ct, year, base_dir):
     ct_totals["total services"] = df[row]
 
     assert df.index[50] == "Electricity"
-    ct_totals["electricity services"] = df[50]
+    ct_totals["electricity services"] = df.iloc[50]
 
     assert df.index[49] == "Derived heat"
-    ct_totals["derived heat services"] = df[49]
+    ct_totals["derived heat services"] = df.iloc[49]
 
     assert df.index[53] == "Thermal uses"
-    ct_totals["thermal uses services"] = df[53]
+    ct_totals["thermal uses services"] = df.iloc[53]
 
     # agriculture, forestry and fishing
 
@@ -282,28 +282,28 @@ def idees_per_country(ct, year, base_dir):
     ct_totals["total two-wheel"] = df["Powered 2-wheelers (Gasoline)"]
 
     assert df.index[19] == "Passenger cars"
-    ct_totals["total passenger cars"] = df[19]
+    ct_totals["total passenger cars"] = df.iloc[19]
 
     assert df.index[30] == "Battery electric vehicles"
-    ct_totals["electricity passenger cars"] = df[30]
+    ct_totals["electricity passenger cars"] = df.iloc[30]
 
     assert df.index[31] == "Motor coaches, buses and trolley buses"
-    ct_totals["total other road passenger"] = df[31]
+    ct_totals["total other road passenger"] = df.iloc[31]
 
     assert df.index[39] == "Battery electric vehicles"
-    ct_totals["electricity other road passenger"] = df[39]
+    ct_totals["electricity other road passenger"] = df.iloc[39]
 
     assert df.index[41] == "Light duty vehicles"
-    ct_totals["total light duty road freight"] = df[41]
+    ct_totals["total light duty road freight"] = df.iloc[41]
 
     assert df.index[49] == "Battery electric vehicles"
-    ct_totals["electricity light duty road freight"] = df[49]
+    ct_totals["electricity light duty road freight"] = df.iloc[49]
 
     row = "Heavy duty vehicles (Diesel oil incl. biofuels)"
     ct_totals["total heavy duty road freight"] = df[row]
 
     assert df.index[61] == "Passenger cars"
-    ct_totals["passenger car efficiency"] = df[61]
+    ct_totals["passenger car efficiency"] = df.iloc[61]
 
     df = pd.read_excel(fn_transport, "TrRail_ene", index_col=0)[year]
 
@@ -312,39 +312,39 @@ def idees_per_country(ct, year, base_dir):
     ct_totals["electricity rail"] = df["Electricity"]
 
     assert df.index[15] == "Passenger transport"
-    ct_totals["total rail passenger"] = df[15]
+    ct_totals["total rail passenger"] = df.iloc[15]
 
     assert df.index[16] == "Metro and tram, urban light rail"
     assert df.index[19] == "Electric"
     assert df.index[20] == "High speed passenger trains"
-    ct_totals["electricity rail passenger"] = df[[16, 19, 20]].sum()
+    ct_totals["electricity rail passenger"] = df.iloc[[16, 19, 20]].sum()
 
     assert df.index[21] == "Freight transport"
-    ct_totals["total rail freight"] = df[21]
+    ct_totals["total rail freight"] = df.iloc[21]
 
     assert df.index[23] == "Electric"
-    ct_totals["electricity rail freight"] = df[23]
+    ct_totals["electricity rail freight"] = df.iloc[23]
 
     df = pd.read_excel(fn_transport, "TrAvia_ene", index_col=0)[year]
 
     assert df.index[6] == "Passenger transport"
-    ct_totals["total aviation passenger"] = df[6]
+    ct_totals["total aviation passenger"] = df.iloc[6]
 
     assert df.index[10] == "Freight transport"
-    ct_totals["total aviation freight"] = df[10]
+    ct_totals["total aviation freight"] = df.iloc[10]
 
     assert df.index[7] == "Domestic"
-    ct_totals["total domestic aviation passenger"] = df[7]
+    ct_totals["total domestic aviation passenger"] = df.iloc[7]
 
     assert df.index[8] == "International - Intra-EU"
     assert df.index[9] == "International - Extra-EU"
-    ct_totals["total international aviation passenger"] = df[[8, 9]].sum()
+    ct_totals["total international aviation passenger"] = df.iloc[[8, 9]].sum()
 
     assert df.index[11] == "Domestic and International - Intra-EU"
-    ct_totals["total domestic aviation freight"] = df[11]
+    ct_totals["total domestic aviation freight"] = df.iloc[11]
 
     assert df.index[12] == "International - Extra-EU"
-    ct_totals["total international aviation freight"] = df[12]
+    ct_totals["total international aviation freight"] = df.iloc[12]
 
     ct_totals["total domestic aviation"] = (
         ct_totals["total domestic aviation freight"]
@@ -364,7 +364,7 @@ def idees_per_country(ct, year, base_dir):
     df = pd.read_excel(fn_transport, "TrRoad_act", index_col=0)[year]
 
     assert df.index[85] == "Passenger cars"
-    ct_totals["passenger cars"] = df[85]
+    ct_totals["passenger cars"] = df.iloc[85]
 
     return pd.Series(ct_totals, name=ct)
 

scripts/build_gas_input_locations.py

@@ -23,11 +23,10 @@ def read_scigrid_gas(fn):
     return df
 
 
-def build_gem_lng_data(lng_fn):
-    df = pd.read_excel(lng_fn[0], sheet_name="LNG terminals - data")
+def build_gem_lng_data(fn):
+    df = pd.read_excel(fn[0], sheet_name="LNG terminals - data")
     df = df.set_index("ComboID")
 
-    remove_status = ["Cancelled"]
     remove_country = ["Cyprus", "Turkey"]
     remove_terminal = ["Puerto de la Luz LNG Terminal", "Gran Canaria LNG Terminal"]
 
@@ -42,9 +41,50 @@ def build_gem_lng_data(lng_fn):
     return gpd.GeoDataFrame(df, geometry=geometry, crs="EPSG:4326")
 
 
-def build_gas_input_locations(lng_fn, entry_fn, prod_fn, countries):
+def build_gem_prod_data(fn):
+    df = pd.read_excel(fn[0], sheet_name="Gas extraction - main")
+    df = df.set_index("GEM Unit ID")
+
+    remove_country = ["Cyprus", "Türkiye"]
+    remove_fuel_type = ["oil"]
+
+    df = df.query(
+        "Status != 'shut in' \
+              & 'Fuel type' != 'oil' \
+              & Country != @remove_country \
+              & ~Latitude.isna() \
+              & ~Longitude.isna()"
+    ).copy()
+
+    p = pd.read_excel(fn[0], sheet_name="Gas extraction - production")
+    p = p.set_index("GEM Unit ID")
+    p = p[p["Fuel description"] == "gas"]
+
+    capacities = pd.DataFrame(index=df.index)
+    for key in ["production", "production design capacity", "reserves"]:
+        cap = (
+            p.loc[p["Production/reserves"] == key, "Quantity (converted)"]
+            .groupby("GEM Unit ID")
+            .sum()
+            .reindex(df.index)
+        )
+        # assume capacity such that 3% of reserves can be extracted per year (25% quantile)
+        annualization_factor = 0.03 if key == "reserves" else 1.0
+        capacities[key] = cap * annualization_factor
+
+    df["mcm_per_year"] = (
+        capacities["production"]
+        .combine_first(capacities["production design capacity"])
+        .combine_first(capacities["reserves"])
+    )
+
+    geometry = gpd.points_from_xy(df["Longitude"], df["Latitude"])
+    return gpd.GeoDataFrame(df, geometry=geometry, crs="EPSG:4326")
+
+
+def build_gas_input_locations(gem_fn, entry_fn, sto_fn, countries):
     # LNG terminals
-    lng = build_gem_lng_data(lng_fn)
+    lng = build_gem_lng_data(gem_fn)
 
     # Entry points from outside the model scope
     entry = read_scigrid_gas(entry_fn)
@@ -55,25 +95,30 @@ def build_gas_input_locations(lng_fn, entry_fn, prod_fn, countries):
         | (entry.from_country == "NO")  # malformed datapoint  # entries from NO to GB
     ]
 
+    sto = read_scigrid_gas(sto_fn)
+    remove_country = ["RU", "UA", "TR", "BY"]
+    sto = sto.query("country_code != @remove_country")
+
     # production sites inside the model scope
-    prod = read_scigrid_gas(prod_fn)
-    prod = prod.loc[
-        (prod.geometry.y > 35) & (prod.geometry.x < 30) & (prod.country_code != "DE")
-    ]
+    prod = build_gem_prod_data(gem_fn)
 
     mcm_per_day_to_mw = 437.5  # MCM/day to MWh/h
+    mcm_per_year_to_mw = 1.199  #  MCM/year to MWh/h
     mtpa_to_mw = 1649.224  # mtpa to MWh/h
-    lng["p_nom"] = lng["CapacityInMtpa"] * mtpa_to_mw
-    entry["p_nom"] = entry["max_cap_from_to_M_m3_per_d"] * mcm_per_day_to_mw
-    prod["p_nom"] = prod["max_supply_M_m3_per_d"] * mcm_per_day_to_mw
+    mcm_to_gwh = 11.36  # MCM to GWh
+    lng["capacity"] = lng["CapacityInMtpa"] * mtpa_to_mw
+    entry["capacity"] = entry["max_cap_from_to_M_m3_per_d"] * mcm_per_day_to_mw
+    prod["capacity"] = prod["mcm_per_year"] * mcm_per_year_to_mw
+    sto["capacity"] = sto["max_cushionGas_M_m3"] * mcm_to_gwh
 
     lng["type"] = "lng"
     entry["type"] = "pipeline"
     prod["type"] = "production"
+    sto["type"] = "storage"
 
-    sel = ["geometry", "p_nom", "type"]
+    sel = ["geometry", "capacity", "type"]
 
-    return pd.concat([prod[sel], entry[sel], lng[sel]], ignore_index=True)
+    return pd.concat([prod[sel], entry[sel], lng[sel], sto[sel]], ignore_index=True)
 
 
 if __name__ == "__main__":
@@ -83,7 +128,7 @@ if __name__ == "__main__":
         snakemake = mock_snakemake(
             "build_gas_input_locations",
             simpl="",
-            clusters="37",
+            clusters="128",
         )
 
     logging.basicConfig(level=snakemake.config["logging"]["level"])
@@ -104,9 +149,9 @@ if __name__ == "__main__":
     countries = regions.index.str[:2].unique().str.replace("GB", "UK")
 
     gas_input_locations = build_gas_input_locations(
-        snakemake.input.lng,
+        snakemake.input.gem,
         snakemake.input.entry,
-        snakemake.input.production,
+        snakemake.input.storage,
         countries,
     )
 
@@ -116,9 +161,13 @@ if __name__ == "__main__":
 
     gas_input_nodes.to_file(snakemake.output.gas_input_nodes, driver="GeoJSON")
 
+    ensure_columns = ["lng", "pipeline", "production", "storage"]
     gas_input_nodes_s = (
-        gas_input_nodes.groupby(["bus", "type"])["p_nom"].sum().unstack()
+        gas_input_nodes.groupby(["bus", "type"])["capacity"]
+        .sum()
+        .unstack()
+        .reindex(columns=ensure_columns)
     )
-    gas_input_nodes_s.columns.name = "p_nom"
+    gas_input_nodes_s.columns.name = "capacity"
 
     gas_input_nodes_s.to_csv(snakemake.output.gas_input_nodes_simplified)

scripts/build_heat_demand.py

@@ -31,10 +31,7 @@ if __name__ == "__main__":
     cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time)
 
     clustered_regions = (
-        gpd.read_file(snakemake.input.regions_onshore)
-        .set_index("name")
-        .buffer(0)
-        .squeeze()
+        gpd.read_file(snakemake.input.regions_onshore).set_index("name").buffer(0)
     )
 
     I = cutout.indicatormatrix(clustered_regions)

scripts/build_hydro_profile.py

@@ -26,7 +26,7 @@ Relevant Settings
 Inputs
 ------
 
-- ``data/bundle/EIA_hydro_generation_2000_2014.csv``: Hydroelectricity net generation per country and year (`EIA <https://www.eia.gov/beta/international/data/browser/#/?pa=000000000000000000000000000000g&c=1028i008006gg6168g80a4k000e0ag00gg0004g800ho00g8&ct=0&ug=8&tl_id=2-A&vs=INTL.33-12-ALB-BKWH.A&cy=2014&vo=0&v=H&start=2000&end=2016>`_)
+- ``data/bundle/eia_hydro_annual_generation.csv``: Hydroelectricity net generation per country and year (`EIA <https://www.eia.gov/beta/international/data/browser/#/?pa=000000000000000000000000000000g&c=1028i008006gg6168g80a4k000e0ag00gg0004g800ho00g8&ct=0&ug=8&tl_id=2-A&vs=INTL.33-12-ALB-BKWH.A&cy=2014&vo=0&v=H&start=2000&end=2016>`_)
 
     .. image:: img/hydrogeneration.png
         :scale: 33 %
@@ -72,12 +72,14 @@ cc = coco.CountryConverter()
 
 def get_eia_annual_hydro_generation(fn, countries):
     # in billion kWh/a = TWh/a
-    df = pd.read_csv(fn, skiprows=2, index_col=1, na_values=[" ", "--"]).iloc[1:, 1:]
+    df = pd.read_csv(
+        fn, skiprows=2, index_col=1, na_values=[" ", "--"], decimal=","
+    ).iloc[1:, 1:]
     df.index = df.index.str.strip()
 
     former_countries = {
         "Former Czechoslovakia": dict(
-            countries=["Czech Republic", "Slovakia"], start=1980, end=1992
+            countries=["Czechia", "Slovakia"], start=1980, end=1992
         ),
         "Former Serbia and Montenegro": dict(
             countries=["Serbia", "Montenegro"], start=1992, end=2005

scripts/build_line_rating.py

@@ -119,7 +119,7 @@ def calculate_line_rating(n, cutout):
             .apply(lambda x: int(re.findall(r"(\d+)-bundle", x)[0]))
         )
         # Set default number of bundles per line
-        relevant_lines["n_bundle"].fillna(1, inplace=True)
+        relevant_lines["n_bundle"] = relevant_lines["n_bundle"].fillna(1)
         R *= relevant_lines["n_bundle"]
         R = calculate_resistance(T=353, R_ref=R)
     Imax = cutout.line_rating(shapes, R, D=0.0218, Ts=353, epsilon=0.8, alpha=0.8)

scripts/build_renewable_profiles.py

@@ -26,20 +26,9 @@ Relevant settings
 
     renewable:
         {technology}:
-            cutout:
-            corine:
-            grid_codes:
-            distance:
-            natura:
-            max_depth:
-            max_shore_distance:
-            min_shore_distance:
-            capacity_per_sqkm:
-            correction_factor:
-            potential:
-            min_p_max_pu:
-            clip_p_max_pu:
-            resource:
+            cutout: corine: luisa: grid_codes: distance: natura: max_depth:
+            max_shore_distance: min_shore_distance: capacity_per_sqkm:
+            correction_factor: min_p_max_pu: clip_p_max_pu: resource:
 
 .. seealso::
     Documentation of the configuration file ``config/config.yaml`` at
@@ -48,21 +37,37 @@ Relevant settings
 Inputs
 ------
 
-- ``data/bundle/corine/g250_clc06_V18_5.tif``: `CORINE Land Cover (CLC) <https://land.copernicus.eu/pan-european/corine-land-cover>`_ inventory on `44 classes <https://wiki.openstreetmap.org/wiki/Corine_Land_Cover#Tagging>`_ of land use (e.g. forests, arable land, industrial, urban areas).
+- ``data/bundle/corine/g250_clc06_V18_5.tif``: `CORINE Land Cover (CLC)
+  <https://land.copernicus.eu/pan-european/corine-land-cover>`_ inventory on `44
+  classes <https://wiki.openstreetmap.org/wiki/Corine_Land_Cover#Tagging>`_ of
+  land use (e.g. forests, arable land, industrial, urban areas) at 100m
+  resolution.
 
     .. image:: img/corine.png
         :scale: 33 %
 
-- ``data/bundle/GEBCO_2014_2D.nc``: A `bathymetric <https://en.wikipedia.org/wiki/Bathymetry>`_ data set with a global terrain model for ocean and land at 15 arc-second intervals by the `General Bathymetric Chart of the Oceans (GEBCO) <https://www.gebco.net/data_and_products/gridded_bathymetry_data/>`_.
+- ``data/LUISA_basemap_020321_50m.tif``: `LUISA Base Map
+  <https://publications.jrc.ec.europa.eu/repository/handle/JRC124621>`_ land
+  coverage dataset at 50m resolution similar to CORINE. For codes in relation to
+  CORINE land cover, see `Annex 1 of the technical documentation
+  <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_.
+
+- ``data/bundle/GEBCO_2014_2D.nc``: A `bathymetric
+  <https://en.wikipedia.org/wiki/Bathymetry>`_ data set with a global terrain
+  model for ocean and land at 15 arc-second intervals by the `General
+  Bathymetric Chart of the Oceans (GEBCO)
+  <https://www.gebco.net/data_and_products/gridded_bathymetry_data/>`_.
 
     .. image:: img/gebco_2019_grid_image.jpg
         :scale: 50 %
 
-    **Source:** `GEBCO <https://www.gebco.net/data_and_products/images/gebco_2019_grid_image.jpg>`_
+    **Source:** `GEBCO
+    <https://www.gebco.net/data_and_products/images/gebco_2019_grid_image.jpg>`_
 
 - ``resources/natura.tiff``: confer :ref:`natura`
 - ``resources/offshore_shapes.geojson``: confer :ref:`shapes`
-- ``resources/regions_onshore.geojson``: (if not offshore wind), confer :ref:`busregions`
+- ``resources/regions_onshore.geojson``: (if not offshore wind), confer
+  :ref:`busregions`
 - ``resources/regions_offshore.geojson``: (if offshore wind), :ref:`busregions`
 - ``"cutouts/" + params["renewable"][{technology}]['cutout']``: :ref:`cutout`
 - ``networks/base.nc``: :ref:`base`
@@ -128,25 +133,26 @@ Description
 This script functions at two main spatial resolutions: the resolution of the
 network nodes and their `Voronoi cells
 <https://en.wikipedia.org/wiki/Voronoi_diagram>`_, and the resolution of the
-cutout grid cells for the weather data. Typically the weather data grid is
-finer than the network nodes, so we have to work out the distribution of
-generators across the grid cells within each Voronoi cell. This is done by
-taking account of a combination of the available land at each grid cell and the
-capacity factor there.
+cutout grid cells for the weather data. Typically the weather data grid is finer
+than the network nodes, so we have to work out the distribution of generators
+across the grid cells within each Voronoi cell. This is done by taking account
+of a combination of the available land at each grid cell and the capacity factor
+there.
 
 First the script computes how much of the technology can be installed at each
-cutout grid cell and each node using the `GLAES
-<https://github.com/FZJ-IEK3-VSA/glaes>`_ library. This uses the CORINE land use data,
-Natura2000 nature reserves and GEBCO bathymetry data.
+cutout grid cell and each node using the `atlite
+<https://github.com/pypsa/atlite>`_ library. This uses the CORINE land use data,
+LUISA land use data, Natura2000 nature reserves, GEBCO bathymetry data, and
+shipping lanes.
 
 .. image:: img/eligibility.png
     :scale: 50 %
     :align: center
 
-To compute the layout of generators in each node's Voronoi cell, the
-installable potential in each grid cell is multiplied with the capacity factor
-at each grid cell. This is done since we assume more generators are installed
-at cells with a higher capacity factor.
+To compute the layout of generators in each node's Voronoi cell, the installable
+potential in each grid cell is multiplied with the capacity factor at each grid
+cell. This is done since we assume more generators are installed at cells with a
+higher capacity factor.
 
 .. image:: img/offwinddc-gridcell.png
     :scale: 50 %
@@ -164,20 +170,14 @@ at cells with a higher capacity factor.
     :scale: 50 %
     :align: center
 
-This layout is then used to compute the generation availability time series
-from the weather data cutout from ``atlite``.
+This layout is then used to compute the generation availability time series from
+the weather data cutout from ``atlite``.
 
-Two methods are available to compute the maximal installable potential for the
-node (`p_nom_max`): ``simple`` and ``conservative``:
-
-- ``simple`` adds up the installable potentials of the individual grid cells.
-  If the model comes close to this limit, then the time series may slightly
-  overestimate production since it is assumed the geographical distribution is
-  proportional to capacity factor.
-
-- ``conservative`` assertains the nodal limit by increasing capacities
-  proportional to the layout until the limit of an individual grid cell is
-  reached.
+The maximal installable potential for the node (`p_nom_max`) is computed by
+adding up the installable potentials of the individual grid cells. If the model
+comes close to this limit, then the time series may slightly overestimate
+production since it is assumed the geographical distribution is proportional to
+capacity factor.
 """
 import functools
 import logging
@@ -210,10 +210,6 @@ if __name__ == "__main__":
     resource = params["resource"]  # pv panel params / wind turbine params
     correction_factor = params.get("correction_factor", 1.0)
     capacity_per_sqkm = params["capacity_per_sqkm"]
-    p_nom_max_meth = params.get("potential", "conservative")
-
-    if isinstance(params.get("corine", {}), list):
-        params["corine"] = {"grid_codes": params["corine"]}
 
     if correction_factor != 1.0:
         logger.info(f"correction_factor is set as {correction_factor}")
@@ -240,18 +236,31 @@ if __name__ == "__main__":
     if params["natura"]:
         excluder.add_raster(snakemake.input.natura, nodata=0, allow_no_overlap=True)
 
-    corine = params.get("corine", {})
-    if "grid_codes" in corine:
-        codes = corine["grid_codes"]
-        excluder.add_raster(snakemake.input.corine, codes=codes, invert=True, crs=3035)
-    if corine.get("distance", 0.0) > 0.0:
-        codes = corine["distance_grid_codes"]
-        buffer = corine["distance"]
-        excluder.add_raster(
-            snakemake.input.corine, codes=codes, buffer=buffer, crs=3035
-        )
+    for dataset in ["corine", "luisa"]:
+        kwargs = {"nodata": 0} if dataset == "luisa" else {}
+        settings = params.get(dataset, {})
+        if not settings:
+            continue
+        if dataset == "luisa" and res > 50:
+            logger.info(
+                "LUISA data is available at 50m resolution, "
+                f"but coarser {res}m resolution is used."
+            )
+        if isinstance(settings, list):
+            settings = {"grid_codes": settings}
+        if "grid_codes" in settings:
+            codes = settings["grid_codes"]
+            excluder.add_raster(
+                snakemake.input[dataset], codes=codes, invert=True, crs=3035, **kwargs
+            )
+        if settings.get("distance", 0.0) > 0.0:
+            codes = settings["distance_grid_codes"]
+            buffer = settings["distance"]
+            excluder.add_raster(
+                snakemake.input[dataset], codes=codes, buffer=buffer, crs=3035, **kwargs
+            )
 
-    if "ship_threshold" in params:
+    if params.get("ship_threshold"):
         shipping_threshold = (
             params["ship_threshold"] * 8760 * 6
         )  # approximation because 6 years of data which is hourly collected
@@ -277,15 +286,22 @@ if __name__ == "__main__":
             snakemake.input.country_shapes, buffer=buffer, invert=True
         )
 
+    logger.info("Calculate landuse availability...")
+    start = time.time()
+
     kwargs = dict(nprocesses=nprocesses, disable_progressbar=noprogress)
-    if noprogress:
-        logger.info("Calculate landuse availabilities...")
-        start = time.time()
-        availability = cutout.availabilitymatrix(regions, excluder, **kwargs)
-        duration = time.time() - start
-        logger.info(f"Completed availability calculation ({duration:2.2f}s)")
-    else:
-        availability = cutout.availabilitymatrix(regions, excluder, **kwargs)
+    availability = cutout.availabilitymatrix(regions, excluder, **kwargs)
+
+    duration = time.time() - start
+    logger.info(f"Completed landuse availability calculation ({duration:2.2f}s)")
+
+    # For Moldova and Ukraine: Overwrite parts not covered by Corine with
+    # externally determined available areas
+    if "availability_matrix_MD_UA" in snakemake.input.keys():
+        availability_MDUA = xr.open_dataarray(
+            snakemake.input["availability_matrix_MD_UA"]
+        )
+        availability.loc[availability_MDUA.coords] = availability_MDUA
 
     area = cutout.grid.to_crs(3035).area / 1e6
     area = xr.DataArray(
@@ -296,8 +312,19 @@ if __name__ == "__main__":
     func = getattr(cutout, resource.pop("method"))
     if client is not None:
         resource["dask_kwargs"] = {"scheduler": client}
+
+    logger.info("Calculate average capacity factor...")
+    start = time.time()
+
     capacity_factor = correction_factor * func(capacity_factor=True, **resource)
     layout = capacity_factor * area * capacity_per_sqkm
+
+    duration = time.time() - start
+    logger.info(f"Completed average capacity factor calculation ({duration:2.2f}s)")
+
+    logger.info("Calculate weighted capacity factor time series...")
+    start = time.time()
+
     profile, capacities = func(
         matrix=availability.stack(spatial=["y", "x"]),
         layout=layout,
@@ -307,17 +334,13 @@ if __name__ == "__main__":
         **resource,
     )
 
-    logger.info(f"Calculating maximal capacity per bus (method '{p_nom_max_meth}')")
-    if p_nom_max_meth == "simple":
-        p_nom_max = capacity_per_sqkm * availability @ area
-    elif p_nom_max_meth == "conservative":
-        max_cap_factor = capacity_factor.where(availability != 0).max(["x", "y"])
-        p_nom_max = capacities / max_cap_factor
-    else:
-        raise AssertionError(
-            'Config key `potential` should be one of "simple" '
-            f'(default) or "conservative", not "{p_nom_max_meth}"'
-        )
+    duration = time.time() - start
+    logger.info(
+        f"Completed weighted capacity factor time series calculation ({duration:2.2f}s)"
+    )
+
+    logger.info(f"Calculating maximal capacity per bus")
+    p_nom_max = capacity_per_sqkm * availability @ area
 
     logger.info("Calculate average distances.")
     layoutmatrix = (layout * availability).stack(spatial=["y", "x"])

scripts/build_retro_cost.py

@@ -102,7 +102,7 @@ solar_energy_transmittance = (
 )
 # solar global radiation [kWh/(m^2a)]
 solar_global_radiation = pd.Series(
-    [246, 401, 246, 148],
+    [271, 392, 271, 160],
     index=["east", "south", "west", "north"],
     name="solar_global_radiation [kWh/(m^2a)]",
 )
@@ -164,6 +164,12 @@ def prepare_building_stock_data():
         },
         inplace=True,
     )
+    building_data["feature"].replace(
+        {
+            "Construction features (U-value)": "Construction features (U-values)",
+        },
+        inplace=True,
+    )
 
     building_data.country_code = building_data.country_code.str.upper()
     building_data["subsector"].replace(
@@ -202,7 +208,8 @@ def prepare_building_stock_data():
 
     # heated floor area ----------------------------------------------------------
     area = building_data[
-        (building_data.type == "Heated area [Mm²]") & (building_data.detail != "Total")
+        (building_data.type == "Heated area [Mm²]")
+        & (building_data.subsector != "Total")
     ]
     area_tot = area[["country", "sector", "value"]].groupby(["country", "sector"]).sum()
     area = pd.concat(
@@ -829,9 +836,9 @@ def calculate_heat_losses(u_values, data_tabula, l_strength, temperature_factor)
     F_red_temp = map_to_lstrength(l_strength, F_red_temp)
 
     Q_ht = (
-        heat_transfer_perm2.groupby(level=1, axis=1)
+        heat_transfer_perm2.T.groupby(level=1)
         .sum()
-        .mul(F_red_temp.droplevel(0, axis=1))
+        .T.mul(F_red_temp.droplevel(0, axis=1))
         .mul(temperature_factor.reindex(heat_transfer_perm2.index, level=0), axis=0)
     )
 
@@ -871,7 +878,7 @@ def calculate_gain_utilisation_factor(heat_transfer_perm2, Q_ht, Q_gain):
     Calculates gain utilisation factor nu.
     """
     # time constant of the building tau [h] = c_m [Wh/(m^2K)] * 1 /(H_tr_e+H_tb*H_ve) [m^2 K /W]
-    tau = c_m / heat_transfer_perm2.groupby(level=1, axis=1).sum()
+    tau = c_m / heat_transfer_perm2.T.groupby(axis=1).sum().T
     alpha = alpha_H_0 + (tau / tau_H_0)
     # heat balance ratio
     gamma = (1 / Q_ht).mul(Q_gain.sum(axis=1), axis=0)

scripts/build_shapes.py

@@ -174,8 +174,8 @@ def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp):
             pd.MultiIndex.from_tuples(pop.pop("unit,geo\\time").str.split(","))
         )
         .loc["THS"]
-        .applymap(lambda x: pd.to_numeric(x, errors="coerce"))
-        .fillna(method="bfill", axis=1)
+        .map(lambda x: pd.to_numeric(x, errors="coerce"))
+        .bfill(axis=1)
     )["2014"]
 
     gdp = pd.read_table(nuts3gdp, na_values=[":"], delimiter=" ?\t", engine="python")
@@ -184,8 +184,8 @@ def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp):
             pd.MultiIndex.from_tuples(gdp.pop("unit,geo\\time").str.split(","))
         )
         .loc["EUR_HAB"]
-        .applymap(lambda x: pd.to_numeric(x, errors="coerce"))
-        .fillna(method="bfill", axis=1)
+        .map(lambda x: pd.to_numeric(x, errors="coerce"))
+        .bfill(axis=1)
     )["2014"]
 
     cantons = pd.read_csv(ch_cantons)

scripts/build_ship_raster.py

@@ -64,7 +64,7 @@ if __name__ == "__main__":
     with zipfile.ZipFile(snakemake.input.ship_density) as zip_f:
         zip_f.extract("shipdensity_global.tif")
         with rioxarray.open_rasterio("shipdensity_global.tif") as ship_density:
-            ship_density = ship_density.drop(["band"]).sel(
+            ship_density = ship_density.drop_vars(["band"]).sel(
                 x=slice(min(xs), max(Xs)), y=slice(max(Ys), min(ys))
             )
             ship_density.rio.to_raster(snakemake.output[0])

scripts/build_solar_thermal_profiles.py

@@ -33,10 +33,7 @@ if __name__ == "__main__":
     cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time)
 
     clustered_regions = (
-        gpd.read_file(snakemake.input.regions_onshore)
-        .set_index("name")
-        .buffer(0)
-        .squeeze()
+        gpd.read_file(snakemake.input.regions_onshore).set_index("name").buffer(0)
     )
 
     I = cutout.indicatormatrix(clustered_regions)

scripts/build_temperature_profiles.py

@@ -31,10 +31,7 @@ if __name__ == "__main__":
     cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time)
 
     clustered_regions = (
-        gpd.read_file(snakemake.input.regions_onshore)
-        .set_index("name")
-        .buffer(0)
-        .squeeze()
+        gpd.read_file(snakemake.input.regions_onshore).set_index("name").buffer(0)
     )
 
     I = cutout.indicatormatrix(clustered_regions)

scripts/cluster_network.py

@@ -16,8 +16,7 @@ Relevant Settings
     clustering:
       cluster_network:
       aggregation_strategies:
-
-    focus_weights:
+      focus_weights:
 
     solving:
         solver:
@@ -237,7 +236,7 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="cbc"):
         n_clusters >= len(N) and n_clusters <= N.sum()
     ), f"Number of clusters must be {len(N)} <= n_clusters <= {N.sum()} for this selection of countries."
 
-    if focus_weights is not None:
+    if isinstance(focus_weights, dict):
         total_focus = sum(list(focus_weights.values()))
 
         assert (
@@ -271,7 +270,7 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="cbc"):
     )
 
     opt = po.SolverFactory(solver_name)
-    if not opt.has_capability("quadratic_objective"):
+    if solver_name == "appsi_highs" or not opt.has_capability("quadratic_objective"):
         logger.warning(
             f"The configured solver `{solver_name}` does not support quadratic objectives. Falling back to `ipopt`."
         )
@@ -466,9 +465,13 @@ if __name__ == "__main__":
 
     params = snakemake.params
     solver_name = snakemake.config["solving"]["solver"]["name"]
+    solver_name = "appsi_highs" if solver_name == "highs" else solver_name
 
     n = pypsa.Network(snakemake.input.network)
 
+    # remove integer outputs for compatibility with PyPSA v0.26.0
+    n.generators.drop("n_mod", axis=1, inplace=True, errors="ignore")
+
     exclude_carriers = params.cluster_network["exclude_carriers"]
     aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers)
     conventional_carriers = set(params.conventional_carriers)

scripts/determine_availability_matrix_MD_UA.py

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+import functools
+import logging
+import time
+
+import atlite
+import fiona
+import geopandas as gpd
+import matplotlib.pyplot as plt
+import numpy as np
+from _helpers import configure_logging
+from atlite.gis import shape_availability
+from rasterio.plot import show
+
+logger = logging.getLogger(__name__)
+
+
+def get_wdpa_layer_name(wdpa_fn, layer_substring):
+    """
+    Get layername from file "wdpa_fn" whose name contains "layer_substring".
+    """
+    l = fiona.listlayers(wdpa_fn)
+    return [_ for _ in l if layer_substring in _][0]
+
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake(
+            "determine_availability_matrix_MD_UA", technology="solar"
+        )
+    configure_logging(snakemake)
+
+    nprocesses = None  # snakemake.config["atlite"].get("nprocesses")
+    noprogress = not snakemake.config["atlite"].get("show_progress", True)
+    config = snakemake.config["renewable"][snakemake.wildcards.technology]
+
+    cutout = atlite.Cutout(snakemake.input.cutout)
+    regions = (
+        gpd.read_file(snakemake.input.regions).set_index("name").rename_axis("bus")
+    )
+    buses = regions.index
+
+    excluder = atlite.ExclusionContainer(crs=3035, res=100)
+
+    corine = config.get("corine", {})
+    if "grid_codes" in corine:
+        # Land cover codes to emulate CORINE results
+        if snakemake.wildcards.technology == "solar":
+            codes = [20, 30, 40, 50, 60, 90, 100]
+        elif snakemake.wildcards.technology == "onwind":
+            codes = [20, 30, 40, 60, 100]
+        elif snakemake.wildcards.technology == "offwind-ac":
+            codes = [80, 200]
+        elif snakemake.wildcards.technology == "offwind-dc":
+            codes = [80, 200]
+        else:
+            assert False, "technology not supported"
+
+        excluder.add_raster(
+            snakemake.input.copernicus, codes=codes, invert=True, crs="EPSG:4326"
+        )
+    if "distance" in corine and corine.get("distance", 0.0) > 0.0:
+        # Land cover codes to emulate CORINE results
+        if snakemake.wildcards.technology == "onwind":
+            codes = [50]
+        else:
+            assert False, "technology not supported"
+
+        buffer = corine["distance"]
+        excluder.add_raster(
+            snakemake.input.copernicus, codes=codes, buffer=buffer, crs="EPSG:4326"
+        )
+
+    if config["natura"]:
+        wdpa_fn = (
+            snakemake.input.wdpa_marine
+            if "offwind" in snakemake.wildcards.technology
+            else snakemake.input.wdpa
+        )
+        layer = get_wdpa_layer_name(wdpa_fn, "polygons")
+        wdpa = gpd.read_file(
+            wdpa_fn,
+            bbox=regions.geometry,
+            layer=layer,
+        ).to_crs(3035)
+        if not wdpa.empty:
+            excluder.add_geometry(wdpa.geometry)
+
+        layer = get_wdpa_layer_name(wdpa_fn, "points")
+        wdpa_pts = gpd.read_file(
+            wdpa_fn,
+            bbox=regions.geometry,
+            layer=layer,
+        ).to_crs(3035)
+        wdpa_pts = wdpa_pts[wdpa_pts["REP_AREA"] > 1]
+        wdpa_pts["buffer_radius"] = np.sqrt(wdpa_pts["REP_AREA"] / np.pi) * 1000
+        wdpa_pts = wdpa_pts.set_geometry(
+            wdpa_pts["geometry"].buffer(wdpa_pts["buffer_radius"])
+        )
+        if not wdpa_pts.empty:
+            excluder.add_geometry(wdpa_pts.geometry)
+
+    if "max_depth" in config:
+        # lambda not supported for atlite + multiprocessing
+        # use named function np.greater with partially frozen argument instead
+        # and exclude areas where: -max_depth > grid cell depth
+        func = functools.partial(np.greater, -config["max_depth"])
+        excluder.add_raster(snakemake.input.gebco, codes=func, crs=4236, nodata=-1000)
+
+    if "min_shore_distance" in config:
+        buffer = config["min_shore_distance"]
+        excluder.add_geometry(snakemake.input.country_shapes, buffer=buffer)
+
+    if "max_shore_distance" in config:
+        buffer = config["max_shore_distance"]
+        excluder.add_geometry(
+            snakemake.input.country_shapes, buffer=buffer, invert=True
+        )
+
+    if "ship_threshold" in config:
+        shipping_threshold = config["ship_threshold"] * 8760 * 6
+        func = functools.partial(np.less, shipping_threshold)
+        excluder.add_raster(
+            snakemake.input.ship_density, codes=func, crs=4326, allow_no_overlap=True
+        )
+
+    kwargs = dict(nprocesses=nprocesses, disable_progressbar=noprogress)
+    if noprogress:
+        logger.info("Calculate landuse availabilities...")
+        start = time.time()
+        availability = cutout.availabilitymatrix(regions, excluder, **kwargs)
+        duration = time.time() - start
+        logger.info(f"Completed availability calculation ({duration:2.2f}s)")
+    else:
+        availability = cutout.availabilitymatrix(regions, excluder, **kwargs)
+
+    regions_geometry = regions.to_crs(3035).geometry
+    band, transform = shape_availability(regions_geometry, excluder)
+    fig, ax = plt.subplots(figsize=(4, 8))
+    gpd.GeoSeries(regions_geometry.unary_union).plot(ax=ax, color="none")
+    show(band, transform=transform, cmap="Greens", ax=ax)
+    plt.axis("off")
+    plt.savefig(snakemake.output.availability_map, bbox_inches="tight", dpi=500)
+
+    # Limit results only to buses for UA and MD
+    buses = regions.loc[regions["country"].isin(["UA", "MD"])].index.values
+    availability = availability.sel(bus=buses)
+
+    # Save and plot for verification
+    availability.to_netcdf(snakemake.output.availability_matrix)

scripts/make_summary.py

@@ -446,6 +446,10 @@ def calculate_metrics(n, label, metrics):
     if "CO2Limit" in n.global_constraints.index:
         metrics.at["co2_shadow", label] = n.global_constraints.at["CO2Limit", "mu"]
 
+    if "co2_sequestration_limit" in n.global_constraints.index:
+        metrics.at["co2_storage_shadow", label] = n.global_constraints.at[
+            "co2_sequestration_limit", "mu"
+        ]
     return metrics
 
 

scripts/make_summary_perfect.py

@@ -28,6 +28,16 @@ idx = pd.IndexSlice
 opt_name = {"Store": "e", "Line": "s", "Transformer": "s"}
 
 
+def reindex_columns(df, cols):
+    investments = cols.levels[3]
+    if len(cols.names) != len(df.columns.levels):
+        df = pd.concat([df] * len(investments), axis=1)
+        df.columns = cols
+    df = df.reindex(cols, axis=1)
+
+    return df
+
+
 def calculate_costs(n, label, costs):
     investments = n.investment_periods
     cols = pd.MultiIndex.from_product(
@@ -39,7 +49,8 @@ def calculate_costs(n, label, costs):
         ],
         names=costs.columns.names[:3] + ["year"],
     )
-    costs = costs.reindex(cols, axis=1)
+
+    costs = reindex_columns(costs, cols)
 
     for c in n.iterate_components(
         n.branch_components | n.controllable_one_port_components ^ {"Load"}
@@ -176,7 +187,7 @@ def calculate_capacities(n, label, capacities):
         ],
         names=capacities.columns.names[:3] + ["year"],
     )
-    capacities = capacities.reindex(cols, axis=1)
+    capacities = reindex_columns(capacities, cols)
 
     for c in n.iterate_components(
         n.branch_components | n.controllable_one_port_components ^ {"Load"}
@@ -229,7 +240,7 @@ def calculate_energy(n, label, energy):
         ],
         names=energy.columns.names[:3] + ["year"],
     )
-    energy = energy.reindex(cols, axis=1)
+    energy = reindex_columns(energy, cols)
 
     for c in n.iterate_components(n.one_port_components | n.branch_components):
         if c.name in n.one_port_components:
@@ -336,7 +347,7 @@ def calculate_supply_energy(n, label, supply_energy):
         ],
         names=supply_energy.columns.names[:3] + ["year"],
     )
-    supply_energy = supply_energy.reindex(cols, axis=1)
+    supply_energy = reindex_columns(supply_energy, cols)
 
     bus_carriers = n.buses.carrier.unique()
 
@@ -604,7 +615,7 @@ def calculate_price_statistics(n, label, price_statistics):
     price_statistics.at["mean", label] = n.buses_t.marginal_price[buses].mean().mean()
 
     price_statistics.at["standard_deviation", label] = (
-        n.buses_t.marginal_price[buses].droplevel(0).unstack().std()
+        n.buses_t.marginal_price[buses].std().std()
     )
 
     return price_statistics

scripts/plot_network.py

@@ -31,7 +31,7 @@ def rename_techs_tyndp(tech):
     tech = rename_techs(tech)
     if "heat pump" in tech or "resistive heater" in tech:
         return "power-to-heat"
-    elif tech in ["H2 Electrolysis", "methanation", "helmeth", "H2 liquefaction"]:
+    elif tech in ["H2 Electrolysis", "methanation", "H2 liquefaction"]:
         return "power-to-gas"
     elif tech == "H2":
         return "H2 storage"
@@ -495,7 +495,7 @@ def plot_ch4_map(network):
     # make a fake MultiIndex so that area is correct for legend
     fossil_gas.index = pd.MultiIndex.from_product([fossil_gas.index, ["fossil gas"]])
 
-    methanation_i = n.links[n.links.carrier.isin(["helmeth", "Sabatier"])].index
+    methanation_i = n.links.query("carrier == 'Sabatier'").index
     methanation = (
         abs(
             n.links_t.p1.loc[:, methanation_i].mul(

scripts/plot_summary.py

@@ -121,7 +121,6 @@ preferred_order = pd.Index(
         "gas boiler",
         "gas",
         "natural gas",
-        "helmeth",
         "methanation",
         "ammonia",
         "hydrogen storage",
@@ -452,7 +451,6 @@ def plot_carbon_budget_distribution(input_eurostat):
 
     sns.set()
     sns.set_style("ticks")
-    plt.style.use("seaborn-ticks")
     plt.rcParams["xtick.direction"] = "in"
     plt.rcParams["ytick.direction"] = "in"
     plt.rcParams["xtick.labelsize"] = 20

scripts/prepare_sector_network.py

@@ -95,12 +95,14 @@ def define_spatial(nodes, options):
         spatial.gas.industry = nodes + " gas for industry"
         spatial.gas.industry_cc = nodes + " gas for industry CC"
         spatial.gas.biogas_to_gas = nodes + " biogas to gas"
+        spatial.gas.biogas_to_gas_cc = nodes + "biogas to gas CC"
     else:
         spatial.gas.nodes = ["EU gas"]
         spatial.gas.locations = ["EU"]
         spatial.gas.biogas = ["EU biogas"]
         spatial.gas.industry = ["gas for industry"]
         spatial.gas.biogas_to_gas = ["EU biogas to gas"]
+        spatial.gas.biogas_to_gas_cc = ["EU biogas to gas CC"]
         if options.get("co2_spatial", options["co2network"]):
             spatial.gas.industry_cc = nodes + " gas for industry CC"
         else:
@@ -480,10 +482,11 @@ def add_carrier_buses(n, carrier, nodes=None):
     n.add("Carrier", carrier)
 
     unit = "MWh_LHV" if carrier == "gas" else "MWh_th"
+    # preliminary value for non-gas carriers to avoid zeros
+    capital_cost = costs.at["gas storage", "fixed"] if carrier == "gas" else 0.02
 
     n.madd("Bus", nodes, location=location, carrier=carrier, unit=unit)
 
-    # capital cost could be corrected to e.g. 0.2 EUR/kWh * annuity and O&M
     n.madd(
         "Store",
         nodes + " Store",
@@ -491,8 +494,7 @@ def add_carrier_buses(n, carrier, nodes=None):
         e_nom_extendable=True,
         e_cyclic=True,
         carrier=carrier,
-        capital_cost=0.2
-        * costs.at[carrier, "discount rate"],  # preliminary value to avoid zeros
+        capital_cost=capital_cost,
     )
 
     n.madd(
@@ -833,14 +835,13 @@ def add_ammonia(n, costs):
         bus2=nodes + " H2",
         p_nom_extendable=True,
         carrier="Haber-Bosch",
-        efficiency=1
-        / (
-            cf_industry["MWh_elec_per_tNH3_electrolysis"]
-            / cf_industry["MWh_NH3_per_tNH3"]
-        ),  # output: MW_NH3 per MW_elec
-        efficiency2=-cf_industry["MWh_H2_per_tNH3_electrolysis"]
-        / cf_industry["MWh_elec_per_tNH3_electrolysis"],  # input: MW_H2 per MW_elec
-        capital_cost=costs.at["Haber-Bosch", "fixed"],
+        efficiency=1 / costs.at["Haber-Bosch", "electricity-input"],
+        efficiency2=-costs.at["Haber-Bosch", "hydrogen-input"]
+        / costs.at["Haber-Bosch", "electricity-input"],
+        capital_cost=costs.at["Haber-Bosch", "fixed"]
+        / costs.at["Haber-Bosch", "electricity-input"],
+        marginal_cost=costs.at["Haber-Bosch", "VOM"]
+        / costs.at["Haber-Bosch", "electricity-input"],
         lifetime=costs.at["Haber-Bosch", "lifetime"],
     )
 
@@ -1051,7 +1052,7 @@ def insert_gas_distribution_costs(n, costs):
         f"Inserting gas distribution grid with investment cost factor of {f_costs}"
     )
 
-    capital_cost = costs.loc["electricity distribution grid"]["fixed"] * f_costs
+    capital_cost = costs.at["electricity distribution grid", "fixed"] * f_costs
 
     # gas boilers
     gas_b = n.links.index[
@@ -1128,6 +1129,7 @@ def add_storage_and_grids(n, costs):
             efficiency=costs.at["OCGT", "efficiency"],
             capital_cost=costs.at["OCGT", "fixed"]
             * costs.at["OCGT", "efficiency"],  # NB: fixed cost is per MWel
+            marginal_cost=costs.at["OCGT", "VOM"],
             lifetime=costs.at["OCGT", "lifetime"],
         )
 
@@ -1188,7 +1190,7 @@ def add_storage_and_grids(n, costs):
 
     if options["gas_network"]:
         logger.info(
-            "Add natural gas infrastructure, incl. LNG terminals, production and entry-points."
+            "Add natural gas infrastructure, incl. LNG terminals, production, storage and entry-points."
         )
 
         if options["H2_retrofit"]:
@@ -1233,10 +1235,25 @@ def add_storage_and_grids(n, costs):
         remove_i = n.generators[gas_i & internal_i].index
         n.generators.drop(remove_i, inplace=True)
 
-        p_nom = gas_input_nodes.sum(axis=1).rename(lambda x: x + " gas")
+        input_types = ["lng", "pipeline", "production"]
+        p_nom = gas_input_nodes[input_types].sum(axis=1).rename(lambda x: x + " gas")
         n.generators.loc[gas_i, "p_nom_extendable"] = False
         n.generators.loc[gas_i, "p_nom"] = p_nom
 
+        # add existing gas storage capacity
+        gas_i = n.stores.carrier == "gas"
+        e_nom = (
+            gas_input_nodes["storage"]
+            .rename(lambda x: x + " gas Store")
+            .reindex(n.stores.index)
+            .fillna(0.0)
+            * 1e3
+        )  # MWh_LHV
+        e_nom.clip(
+            upper=e_nom.quantile(0.98), inplace=True
+        )  # limit extremely large storage
+        n.stores.loc[gas_i, "e_nom_min"] = e_nom
+
         # add candidates for new gas pipelines to achieve full connectivity
 
         G = nx.Graph()
@@ -1371,6 +1388,7 @@ def add_storage_and_grids(n, costs):
             bus2=spatial.co2.nodes,
             p_nom_extendable=True,
             carrier="Sabatier",
+            p_min_pu=options.get("min_part_load_methanation", 0),
             efficiency=costs.at["methanation", "efficiency"],
             efficiency2=-costs.at["methanation", "efficiency"]
             * costs.at["gas", "CO2 intensity"],
@@ -1379,23 +1397,6 @@ def add_storage_and_grids(n, costs):
             lifetime=costs.at["methanation", "lifetime"],
         )
 
-    if options["helmeth"]:
-        n.madd(
-            "Link",
-            spatial.nodes,
-            suffix=" helmeth",
-            bus0=nodes,
-            bus1=spatial.gas.nodes,
-            bus2=spatial.co2.nodes,
-            carrier="helmeth",
-            p_nom_extendable=True,
-            efficiency=costs.at["helmeth", "efficiency"],
-            efficiency2=-costs.at["helmeth", "efficiency"]
-            * costs.at["gas", "CO2 intensity"],
-            capital_cost=costs.at["helmeth", "fixed"],
-            lifetime=costs.at["helmeth", "lifetime"],
-        )
-
     if options.get("coal_cc"):
         n.madd(
             "Link",
@@ -1584,14 +1585,7 @@ def add_land_transport(n, costs):
         )
 
     if ice_share > 0:
-        if "oil" not in n.buses.carrier.unique():
-            n.madd(
-                "Bus",
-                spatial.oil.nodes,
-                location=spatial.oil.locations,
-                carrier="oil",
-                unit="MWh_LHV",
-            )
+        add_carrier_buses(n, "oil")
 
         ice_efficiency = options["transport_internal_combustion_efficiency"]
 
@@ -1678,7 +1672,7 @@ def build_heat_demand(n):
     electric_nodes = n.loads.index[n.loads.carrier == "electricity"]
     n.loads_t.p_set[electric_nodes] = (
         n.loads_t.p_set[electric_nodes]
-        - electric_heat_supply.groupby(level=1, axis=1).sum()[electric_nodes]
+        - electric_heat_supply.T.groupby(level=1).sum().T[electric_nodes]
     )
 
     return heat_demand
@@ -1741,6 +1735,19 @@ def add_heat(n, costs):
             unit="MWh_th",
         )
 
+        if name == "urban central" and options.get("central_heat_vent"):
+            n.madd(
+                "Generator",
+                nodes[name] + f" {name} heat vent",
+                bus=nodes[name] + f" {name} heat",
+                location=nodes[name],
+                carrier=name + " heat vent",
+                p_nom_extendable=True,
+                p_max_pu=0,
+                p_min_pu=-1,
+                unit="MWh_th",
+            )
+
         ## Add heat load
 
         for sector in sectors:
@@ -1759,15 +1766,17 @@ def add_heat(n, costs):
             if sector in name:
                 heat_load = (
                     heat_demand[[sector + " water", sector + " space"]]
-                    .groupby(level=1, axis=1)
-                    .sum()[nodes[name]]
+                    .T.groupby(level=1)
+                    .sum()
+                    .T[nodes[name]]
                     .multiply(factor)
                 )
 
         if name == "urban central":
             heat_load = (
-                heat_demand.groupby(level=1, axis=1)
-                .sum()[nodes[name]]
+                heat_demand.T.groupby(level=1)
+                .sum()
+                .T[nodes[name]]
                 .multiply(
                     factor * (1 + options["district_heating"]["district_heating_loss"])
                 )
@@ -2039,7 +2048,11 @@ def add_heat(n, costs):
             space_heat_demand = demand * w_space[sec][node]
             # normed time profile of space heat demand 'space_pu' (values between 0-1),
             # p_max_pu/p_min_pu of retrofitting generators
-            space_pu = (space_heat_demand / space_heat_demand.max()).to_frame(name=node)
+            space_pu = (
+                (space_heat_demand / space_heat_demand.max())
+                .to_frame(name=node)
+                .fillna(0)
+            )
 
             # minimum heat demand 'dE' after retrofitting in units of original heat demand (values between 0-1)
             dE = retro_data.loc[(ct, sec), ("dE")]
@@ -2051,6 +2064,9 @@ def add_heat(n, costs):
                 * floor_area_node
                 / ((1 - dE) * space_heat_demand.max())
             )
+            if space_heat_demand.max() == 0:
+                capital_cost = capital_cost.apply(lambda b: 0 if b == np.inf else b)
+
             # number of possible retrofitting measures 'strengths' (set in list at config.yaml 'l_strength')
             # given in additional insulation thickness [m]
             # for each measure, a retrofitting generator is added at the node
@@ -2194,12 +2210,42 @@ def add_biomass(n, costs):
         bus1=spatial.gas.nodes,
         bus2="co2 atmosphere",
         carrier="biogas to gas",
-        capital_cost=costs.loc["biogas upgrading", "fixed"],
-        marginal_cost=costs.loc["biogas upgrading", "VOM"],
+        capital_cost=costs.at["biogas", "fixed"]
+        + costs.at["biogas upgrading", "fixed"],
+        marginal_cost=costs.at["biogas upgrading", "VOM"],
+        efficiency=costs.at["biogas", "efficiency"],
         efficiency2=-costs.at["gas", "CO2 intensity"],
         p_nom_extendable=True,
     )
 
+    if options.get("biogas_upgrading_cc"):
+        # Assuming for costs that the CO2 from upgrading is pure, such as in amine scrubbing. I.e., with and without CC is
+        # equivalent. Adding biomass CHP capture because biogas is often small-scale and decentral so further
+        # from e.g. CO2 grid or buyers. This is a proxy for the added cost for e.g. a raw biogas pipeline to a central upgrading facility
+
+        n.madd(
+            "Link",
+            spatial.gas.biogas_to_gas_cc,
+            bus0=spatial.gas.biogas,
+            bus1=spatial.gas.nodes,
+            bus2="co2 stored",
+            bus3="co2 atmosphere",
+            carrier="biogas to gas CC",
+            capital_cost=costs.at["biogas CC", "fixed"]
+            + costs.at["biogas upgrading", "fixed"]
+            + costs.at["biomass CHP capture", "fixed"]
+            * costs.at["biogas CC", "CO2 stored"],
+            marginal_cost=costs.at["biogas CC", "VOM"]
+            + costs.at["biogas upgrading", "VOM"],
+            efficiency=costs.at["biogas CC", "efficiency"],
+            efficiency2=costs.at["biogas CC", "CO2 stored"]
+            * costs.at["biogas CC", "capture rate"],
+            efficiency3=-costs.at["gas", "CO2 intensity"]
+            - costs.at["biogas CC", "CO2 stored"]
+            * costs.at["biogas CC", "capture rate"],
+            p_nom_extendable=True,
+        )
+
     if options["biomass_transport"]:
         # add biomass transport
         transport_costs = pd.read_csv(
@@ -2325,6 +2371,7 @@ def add_biomass(n, costs):
                 efficiency=costs.at["biomass boiler", "efficiency"],
                 capital_cost=costs.at["biomass boiler", "efficiency"]
                 * costs.at["biomass boiler", "fixed"],
+                marginal_cost=costs.at["biomass boiler", "pelletizing cost"],
                 lifetime=costs.at["biomass boiler", "lifetime"],
             )
 
@@ -2344,7 +2391,7 @@ def add_biomass(n, costs):
             + costs.at["BtL", "CO2 stored"],
             p_nom_extendable=True,
             capital_cost=costs.at["BtL", "fixed"],
-            marginal_cost=costs.at["BtL", "efficiency"] * costs.loc["BtL", "VOM"],
+            marginal_cost=costs.at["BtL", "efficiency"] * costs.at["BtL", "VOM"],
         )
 
         # TODO: Update with energy penalty
@@ -2365,7 +2412,7 @@ def add_biomass(n, costs):
             p_nom_extendable=True,
             capital_cost=costs.at["BtL", "fixed"]
             + costs.at["biomass CHP capture", "fixed"] * costs.at["BtL", "CO2 stored"],
-            marginal_cost=costs.at["BtL", "efficiency"] * costs.loc["BtL", "VOM"],
+            marginal_cost=costs.at["BtL", "efficiency"] * costs.at["BtL", "VOM"],
         )
 
     # BioSNG from solid biomass
@@ -2384,7 +2431,7 @@ def add_biomass(n, costs):
             + costs.at["BioSNG", "CO2 stored"],
             p_nom_extendable=True,
             capital_cost=costs.at["BioSNG", "fixed"],
-            marginal_cost=costs.at["BioSNG", "efficiency"] * costs.loc["BioSNG", "VOM"],
+            marginal_cost=costs.at["BioSNG", "efficiency"] * costs.at["BioSNG", "VOM"],
         )
 
         # TODO: Update with energy penalty for CC
@@ -2408,7 +2455,7 @@ def add_biomass(n, costs):
             capital_cost=costs.at["BioSNG", "fixed"]
             + costs.at["biomass CHP capture", "fixed"]
             * costs.at["BioSNG", "CO2 stored"],
-            marginal_cost=costs.at["BioSNG", "efficiency"] * costs.loc["BioSNG", "VOM"],
+            marginal_cost=costs.at["BioSNG", "efficiency"] * costs.at["BioSNG", "VOM"],
         )
 
 
@@ -2646,6 +2693,8 @@ def add_industry(n, costs):
             p_min_pu=options.get("min_part_load_methanolisation", 0),
             capital_cost=costs.at["methanolisation", "fixed"]
             * options["MWh_MeOH_per_MWh_H2"],  # EUR/MW_H2/a
+            marginal_cost=options["MWh_MeOH_per_MWh_H2"]
+            * costs.at["methanolisation", "VOM"],
             lifetime=costs.at["methanolisation", "lifetime"],
             efficiency=options["MWh_MeOH_per_MWh_H2"],
             efficiency2=-options["MWh_MeOH_per_MWh_H2"] / options["MWh_MeOH_per_MWh_e"],
@@ -2773,7 +2822,7 @@ def add_industry(n, costs):
                 nodes_heat[name] + f" {name} oil boiler",
                 p_nom_extendable=True,
                 bus0=spatial.oil.nodes,
-                bus1=nodes_heat[name] + f" {name}  heat",
+                bus1=nodes_heat[name] + f" {name} heat",
                 bus2="co2 atmosphere",
                 carrier=f"{name} oil boiler",
                 efficiency=costs.at["decentral oil boiler", "efficiency"],
@@ -2793,6 +2842,8 @@ def add_industry(n, costs):
         efficiency=costs.at["Fischer-Tropsch", "efficiency"],
         capital_cost=costs.at["Fischer-Tropsch", "fixed"]
         * costs.at["Fischer-Tropsch", "efficiency"],  # EUR/MW_H2/a
+        marginal_cost=costs.at["Fischer-Tropsch", "efficiency"]
+        * costs.at["Fischer-Tropsch", "VOM"],
         efficiency2=-costs.at["oil", "CO2 intensity"]
         * costs.at["Fischer-Tropsch", "efficiency"],
         p_nom_extendable=True,
@@ -3044,8 +3095,13 @@ def add_waste_heat(n):
     if not urban_central.empty:
         urban_central = urban_central.str[: -len(" urban central heat")]
 
+        link_carriers = n.links.carrier.unique()
+
         # TODO what is the 0.95 and should it be a config option?
-        if options["use_fischer_tropsch_waste_heat"]:
+        if (
+            options["use_fischer_tropsch_waste_heat"]
+            and "Fischer-Tropsch" in link_carriers
+        ):
             n.links.loc[urban_central + " Fischer-Tropsch", "bus3"] = (
                 urban_central + " urban central heat"
             )
@@ -3053,8 +3109,48 @@ def add_waste_heat(n):
                 0.95 - n.links.loc[urban_central + " Fischer-Tropsch", "efficiency"]
             )
 
+        if options["use_methanation_waste_heat"] and "Sabatier" in link_carriers:
+            n.links.loc[urban_central + " Sabatier", "bus3"] = (
+                urban_central + " urban central heat"
+            )
+            n.links.loc[urban_central + " Sabatier", "efficiency3"] = (
+                0.95 - n.links.loc[urban_central + " Sabatier", "efficiency"]
+            )
+
+        # DEA quotes 15% of total input (11% of which are high-value heat)
+        if options["use_haber_bosch_waste_heat"] and "Haber-Bosch" in link_carriers:
+            n.links.loc[urban_central + " Haber-Bosch", "bus3"] = (
+                urban_central + " urban central heat"
+            )
+            total_energy_input = (
+                cf_industry["MWh_H2_per_tNH3_electrolysis"]
+                + cf_industry["MWh_elec_per_tNH3_electrolysis"]
+            ) / cf_industry["MWh_NH3_per_tNH3"]
+            electricity_input = (
+                cf_industry["MWh_elec_per_tNH3_electrolysis"]
+                / cf_industry["MWh_NH3_per_tNH3"]
+            )
+            n.links.loc[urban_central + " Haber-Bosch", "efficiency3"] = (
+                0.15 * total_energy_input / electricity_input
+            )
+
+        if (
+            options["use_methanolisation_waste_heat"]
+            and "methanolisation" in link_carriers
+        ):
+            n.links.loc[urban_central + " methanolisation", "bus4"] = (
+                urban_central + " urban central heat"
+            )
+            n.links.loc[urban_central + " methanolisation", "efficiency4"] = (
+                costs.at["methanolisation", "heat-output"]
+                / costs.at["methanolisation", "hydrogen-input"]
+            )
+
         # TODO integrate usable waste heat efficiency into technology-data from DEA
-        if options.get("use_electrolysis_waste_heat", False):
+        if (
+            options.get("use_electrolysis_waste_heat", False)
+            and "H2 Electrolysis" in link_carriers
+        ):
             n.links.loc[urban_central + " H2 Electrolysis", "bus2"] = (
                 urban_central + " urban central heat"
             )
@@ -3062,7 +3158,7 @@ def add_waste_heat(n):
                 0.84 - n.links.loc[urban_central + " H2 Electrolysis", "efficiency"]
             )
 
-        if options["use_fuel_cell_waste_heat"]:
+        if options["use_fuel_cell_waste_heat"] and "H2 Fuel Cell" in link_carriers:
             n.links.loc[urban_central + " H2 Fuel Cell", "bus2"] = (
                 urban_central + " urban central heat"
             )
@@ -3428,6 +3524,57 @@ def set_temporal_aggregation(n, opts, solver_name):
     return n
 
 
+def lossy_bidirectional_links(n, carrier, efficiencies={}):
+    "Split bidirectional links into two unidirectional links to include transmission losses."
+
+    carrier_i = n.links.query("carrier == @carrier").index
+
+    if (
+        not any((v != 1.0) or (v >= 0) for v in efficiencies.values())
+        or carrier_i.empty
+    ):
+        return
+
+    efficiency_static = efficiencies.get("efficiency_static", 1)
+    efficiency_per_1000km = efficiencies.get("efficiency_per_1000km", 1)
+    compression_per_1000km = efficiencies.get("compression_per_1000km", 0)
+
+    logger.info(
+        f"Specified losses for {carrier} transmission "
+        f"(static: {efficiency_static}, per 1000km: {efficiency_per_1000km}, compression per 1000km: {compression_per_1000km}). "
+        "Splitting bidirectional links."
+    )
+
+    n.links.loc[carrier_i, "p_min_pu"] = 0
+    n.links.loc[
+        carrier_i, "efficiency"
+    ] = efficiency_static * efficiency_per_1000km ** (
+        n.links.loc[carrier_i, "length"] / 1e3
+    )
+    rev_links = (
+        n.links.loc[carrier_i].copy().rename({"bus0": "bus1", "bus1": "bus0"}, axis=1)
+    )
+    rev_links["length_original"] = rev_links["length"]
+    rev_links["capital_cost"] = 0
+    rev_links["length"] = 0
+    rev_links["reversed"] = True
+    rev_links.index = rev_links.index.map(lambda x: x + "-reversed")
+
+    n.links = pd.concat([n.links, rev_links], sort=False)
+    n.links["reversed"] = n.links["reversed"].fillna(False)
+    n.links["length_original"] = n.links["length_original"].fillna(n.links.length)
+
+    # do compression losses after concatenation to take electricity consumption at bus0 in either direction
+    carrier_i = n.links.query("carrier == @carrier").index
+    if compression_per_1000km > 0:
+        n.links.loc[carrier_i, "bus2"] = n.links.loc[carrier_i, "bus0"].map(
+            n.buses.location
+        )  # electricity
+        n.links.loc[carrier_i, "efficiency2"] = (
+            -compression_per_1000km * n.links.loc[carrier_i, "length_original"] / 1e3
+        )
+
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -3505,6 +3652,15 @@ if __name__ == "__main__":
     if "nodistrict" in opts:
         options["district_heating"]["progress"] = 0.0
 
+    if "nowasteheat" in opts:
+        logger.info("Disabling waste heat.")
+        options["use_fischer_tropsch_waste_heat"] = False
+        options["use_methanolisation_waste_heat"] = False
+        options["use_haber_bosch_waste_heat"] = False
+        options["use_methanation_waste_heat"] = False
+        options["use_fuel_cell_waste_heat"] = False
+        options["use_electrolysis_waste_heat"] = False
+
     if "T" in opts:
         add_land_transport(n, costs)
 
@@ -3594,6 +3750,18 @@ if __name__ == "__main__":
     if options["electricity_grid_connection"]:
         add_electricity_grid_connection(n, costs)
 
+    for k, v in options["transmission_efficiency"].items():
+        lossy_bidirectional_links(n, k, v)
+
+    # Workaround: Remove lines with conflicting (and unrealistic) properties
+    # cf. https://github.com/PyPSA/pypsa-eur/issues/444
+    if snakemake.config["solving"]["options"]["transmission_losses"]:
+        idx = n.lines.query("num_parallel == 0").index
+        logger.info(
+            f"Removing {len(idx)} line(s) with properties conflicting with transmission losses functionality."
+        )
+        n.mremove("Line", idx)
+
     first_year_myopic = (snakemake.params.foresight in ["myopic", "perfect"]) and (
         snakemake.params.planning_horizons[0] == investment_year
     )

scripts/retrieve_databundle.py

@@ -36,7 +36,7 @@ import logging
 import tarfile
 from pathlib import Path
 
-from _helpers import configure_logging, progress_retrieve
+from _helpers import configure_logging, progress_retrieve, validate_checksum
 
 logger = logging.getLogger(__name__)
 
@@ -65,6 +65,8 @@ if __name__ == "__main__":
     disable_progress = snakemake.config["run"].get("disable_progressbar", False)
     progress_retrieve(url, tarball_fn, disable=disable_progress)
 
+    validate_checksum(tarball_fn, url)
+
     logger.info("Extracting databundle.")
     tarfile.open(tarball_fn).extractall(to_fn)
 

scripts/retrieve_gas_infrastructure_data.py

@@ -11,7 +11,7 @@ import logging
 import zipfile
 from pathlib import Path
 
-from _helpers import progress_retrieve
+from _helpers import progress_retrieve, validate_checksum
 
 logger = logging.getLogger(__name__)
 
@@ -35,6 +35,8 @@ if __name__ == "__main__":
     disable_progress = snakemake.config["run"].get("disable_progressbar", False)
     progress_retrieve(url, zip_fn, disable=disable_progress)
 
+    validate_checksum(zip_fn, url)
+
     logger.info("Extracting databundle.")
     zipfile.ZipFile(zip_fn).extractall(to_fn)
 

scripts/retrieve_sector_databundle.py

@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
 import tarfile
 from pathlib import Path
 
-from _helpers import configure_logging, progress_retrieve
+from _helpers import configure_logging, progress_retrieve, validate_checksum
 
 if __name__ == "__main__":
     if "snakemake" not in globals():
@@ -34,6 +34,8 @@ if __name__ == "__main__":
     disable_progress = snakemake.config["run"].get("disable_progressbar", False)
     progress_retrieve(url, tarball_fn, disable=disable_progress)
 
+    validate_checksum(tarball_fn, url)
+
     logger.info("Extracting databundle.")
     tarfile.open(tarball_fn).extractall(to_fn)
 

scripts/simplify_network.py

@@ -536,6 +536,9 @@ if __name__ == "__main__":
     n = pypsa.Network(snakemake.input.network)
     Nyears = n.snapshot_weightings.objective.sum() / 8760
 
+    # remove integer outputs for compatibility with PyPSA v0.26.0
+    n.generators.drop("n_mod", axis=1, inplace=True, errors="ignore")
+
     n, trafo_map = simplify_network_to_380(n)
 
     technology_costs = load_costs(

scripts/solve_network.py

@@ -26,8 +26,11 @@ Additionally, some extra constraints specified in :mod:`solve_network` are added
     the workflow for all scenarios in the configuration file (``scenario:``)
     based on the rule :mod:`solve_network`.
 """
+import importlib
 import logging
+import os
 import re
+import sys
 
 import numpy as np
 import pandas as pd
@@ -687,6 +690,35 @@ def add_battery_constraints(n):
     n.model.add_constraints(lhs == 0, name="Link-charger_ratio")
 
 
+def add_lossy_bidirectional_link_constraints(n):
+    if not n.links.p_nom_extendable.any() or not "reversed" in n.links.columns:
+        return
+
+    n.links["reversed"] = n.links.reversed.fillna(0).astype(bool)
+    carriers = n.links.loc[n.links.reversed, "carrier"].unique()
+
+    forward_i = n.links.query(
+        "carrier in @carriers and ~reversed and p_nom_extendable"
+    ).index
+
+    def get_backward_i(forward_i):
+        return pd.Index(
+            [
+                re.sub(r"-(\d{4})$", r"-reversed-\1", s)
+                if re.search(r"-\d{4}$", s)
+                else s + "-reversed"
+                for s in forward_i
+            ]
+        )
+
+    backward_i = get_backward_i(forward_i)
+
+    lhs = n.model["Link-p_nom"].loc[backward_i]
+    rhs = n.model["Link-p_nom"].loc[forward_i]
+
+    n.model.add_constraints(lhs == rhs, name="Link-bidirectional_sync")
+
+
 def add_chp_constraints(n):
     electric = (
         n.links.index.str.contains("urban central")
@@ -745,9 +777,13 @@ def add_pipe_retrofit_constraint(n):
     """
     Add constraint for retrofitting existing CH4 pipelines to H2 pipelines.
     """
-    gas_pipes_i = n.links.query("carrier == 'gas pipeline' and p_nom_extendable").index
+    if "reversed" not in n.links.columns:
+        n.links["reversed"] = False
+    gas_pipes_i = n.links.query(
+        "carrier == 'gas pipeline' and p_nom_extendable and ~reversed"
+    ).index
     h2_retrofitted_i = n.links.query(
-        "carrier == 'H2 pipeline retrofitted' and p_nom_extendable"
+        "carrier == 'H2 pipeline retrofitted' and p_nom_extendable and ~reversed"
     ).index
 
     if h2_retrofitted_i.empty or gas_pipes_i.empty:
@@ -786,25 +822,20 @@ def extra_functionality(n, snapshots):
         if "EQ" in o:
             add_EQ_constraints(n, o)
     add_battery_constraints(n)
+    add_lossy_bidirectional_link_constraints(n)
     add_pipe_retrofit_constraint(n)
     if n._multi_invest:
         add_carbon_constraint(n, snapshots)
         add_carbon_budget_constraint(n, snapshots)
         add_retrofit_gas_boiler_constraint(n, snapshots)
 
-    if "additional_functionality" in snakemake.input.keys():
-        import importlib
-        import os
-        import sys
-
-        sys.path.append(os.path.dirname(snakemake.input.additional_functionality))
-        additional_functionality = importlib.import_module(
-            os.path.splitext(
-                os.path.basename(snakemake.input.additional_functionality)
-            )[0]
-        )
-
-        additional_functionality.additional_functionality(n, snapshots, snakemake)
+    if snakemake.params.custom_extra_functionality:
+        source_path = snakemake.params.custom_extra_functionality
+        assert os.path.exists(source_path), f"{source_path} does not exist"
+        sys.path.append(os.path.dirname(source_path))
+        module_name = os.path.splitext(os.path.basename(source_path))[0]
+        module = importlib.import_module(module_name)
+        module.custom_extra_functionality(n, snapshots)
 
 
 def solve_network(n, config, solving, opts="", **kwargs):
@@ -853,10 +884,9 @@ def solve_network(n, config, solving, opts="", **kwargs):
             f"Solving status '{status}' with termination condition '{condition}'"
         )
     if "infeasible" in condition:
-        m = n.model
-        labels = m.compute_infeasibilities()
-        print(labels)
-        m.print_infeasibilities()
+        labels = n.model.compute_infeasibilities()
+        logger.info("Labels:\n" + labels)
+        n.model.print_infeasibilities()
         raise RuntimeError("Solving status 'infeasible'")
 
     return n