Merge branch 'master' into land-transport-fix

Snakefile

@@ -7,13 +7,11 @@ from shutil import copyfile, move, rmtree
 from pathlib import Path
 import yaml
 
-from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider
 from snakemake.utils import min_version
 
-from scripts._helpers import path_provider
+min_version("8.5")
 
-min_version("7.7")
+from scripts._helpers import path_provider
-HTTP = HTTPRemoteProvider()
 
 default_files = {
     "config/config.default.yaml": "config/config.yaml",

config/config.default.yaml

@@ -813,8 +813,8 @@ solving:
     cbc-default: {} # Used in CI
     glpk-default: {} # Used in CI
 
-  mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
+  mem_mb: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
-  walltime: "12:00:00"
+  runtime: 6h #runtime in humanfriendly style https://humanfriendly.readthedocs.io/en/latest/
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#plotting
 plotting:

data/custom_powerplants.csv

@@ -2,36 +2,36 @@
 1266,Khmelnitskiy,Nuclear,,PP,UA,1901.8916595755832,,0.0,0.0,0.0,0.0,1988.0,2005.0,,,50.3023,26.6466,[nan],"{'GEO': ['GEO3842'], 'GPD': ['WRI1005111'], 'CARMA': ['CARMA22000']}"
 1268,Kaniv,Hydro,Reservoir,PP,UA,452.1656050955414,,0.0,0.0,0.0,0.0,1972.0,2003.0,,,49.76653,31.47165,[nan],"{'GEO': ['GEO43017'], 'GPD': ['WRI1005122'], 'CARMA': ['CARMA21140']}"
 1269,Kahovska kakhovka,Hydro,Reservoir,PP,UA,352.45222929936307,,0.0,0.0,0.0,0.0,1955.0,1956.0,,,46.77858,33.36965,[nan],"{'GEO': ['GEO43018'], 'GPD': ['WRI1005118'], 'CARMA': ['CARMA20855']}"
-1347,Kharkiv,Natural Gas,Steam Turbine,CHP,UA,494.94274967602314,,0.0,0.0,0.0,0.0,1979.0,1980.0,,,49.9719,36107,[nan],"{'GEO': ['GEO43027'], 'GPD': ['WRI1005126'], 'CARMA': ['CARMA21972']}"
+1347,Kharkiv,Natural Gas,Steam Turbine,CHP,UA,494.94274967602314,,0.0,0.0,0.0,0.0,1979.0,1980.0,,,49.9719,36.107,[nan],"{'GEO': ['GEO43027'], 'GPD': ['WRI1005126'], 'CARMA': ['CARMA21972']}"
 1348,Kremenchuk,Hydro,Reservoir,PP,UA,617.0382165605096,,0.0,0.0,0.0,0.0,1959.0,1960.0,,,49.07759,33.2505,[nan],"{'GEO': ['GEO43019'], 'GPD': ['WRI1005121'], 'CARMA': ['CARMA23072']}"
 1377,Krivorozhskaya,Hard Coal,Steam Turbine,PP,UA,2600.0164509342876,,0.0,0.0,0.0,0.0,1965.0,1992.0,,,47.5432,33.6583,[nan],"{'GEO': ['GEO42989'], 'GPD': ['WRI1005100'], 'CARMA': ['CARMA23176']}"
 1407,Zmiyevskaya zmiivskaya,Hard Coal,Steam Turbine,PP,UA,2028.3816283884514,,0.0,0.0,0.0,0.0,1960.0,2005.0,,,49.5852,36.5231,[nan],"{'GEO': ['GEO42999'], 'GPD': ['WRI1005103'], 'CARMA': ['CARMA51042']}"
 1408,Pridneprovskaya,Hard Coal,Steam Turbine,CHP,UA,1627.3152609570984,,0.0,0.0,0.0,0.0,1959.0,1966.0,,,48.4051,35.1131,[nan],"{'GEO': ['GEO42990'], 'GPD': ['WRI1005102'], 'CARMA': ['CARMA35874']}"
 1409,Kurakhovskaya,Hard Coal,Steam Turbine,PP,UA,1371.0015824607397,,0.0,0.0,0.0,0.0,1972.0,2003.0,,,47.9944,37.24022,[nan],"{'GEO': ['GEO42994'], 'GPD': ['WRI1005104'], 'CARMA': ['CARMA23339']}"
-1410,Dobrotvorsky,Hard Coal,Steam Turbine,PP,UA,553.1949895604868,,0.0,0.0,0.0,0.0,1960.0,1964.0,,,50.2133,24375,[nan],"{'GEO': ['GEO42992'], 'GPD': ['WRI1005096'], 'CARMA': ['CARMA10971']}"
+1410,Dobrotvorsky,Hard Coal,Steam Turbine,PP,UA,553.1949895604868,,0.0,0.0,0.0,0.0,1960.0,1964.0,,,50.2133,24.375,[nan],"{'GEO': ['GEO42992'], 'GPD': ['WRI1005096'], 'CARMA': ['CARMA10971']}"
 1422,Zuyevskaya,Hard Coal,Steam Turbine,PP,UA,1147.87960333801,,0.0,0.0,0.0,0.0,1982.0,2007.0,,,48.0331,38.28615,[nan],"{'GEO': ['GEO42995'], 'GPD': ['WRI1005106'], 'CARMA': ['CARMA51083']}"
 1423,Zaporozhye,Nuclear,,PP,UA,5705.67497872675,,0.0,0.0,0.0,0.0,1985.0,1996.0,,,47.5119,34.5863,[nan],"{'GEO': ['GEO6207'], 'GPD': ['WRI1005114'], 'CARMA': ['CARMA50875']}"
 1424,Trypilska,Hard Coal,Steam Turbine,PP,UA,1659.5849686814602,,0.0,0.0,0.0,0.0,1969.0,1972.0,,,50.1344,30.7468,[nan],"{'GEO': ['GEO43000'], 'GPD': ['WRI1005099'], 'CARMA': ['CARMA46410']}"
 1425,Tashlyk,Hydro,Pumped Storage,Store,UA,285.55968954109585,,0.0,0.0,0.0,0.0,2006.0,2007.0,,,47.7968,31.1811,[nan],"{'GEO': ['GEO43025'], 'GPD': ['WRI1005117'], 'CARMA': ['CARMA44696']}"
 1426,Starobeshivska,Hard Coal,Steam Turbine,PP,UA,1636.5351774497733,,0.0,0.0,0.0,0.0,1961.0,1967.0,,,47.7997,38.00612,[nan],"{'GEO': ['GEO43003'], 'GPD': ['WRI1005105'], 'CARMA': ['CARMA43083']}"
-1427,South,Nuclear,,PP,UA,2852.837489363375,,0.0,0.0,0.0,0.0,1983.0,1989.0,,,47812,31.22,[nan],"{'GEO': ['GEO5475'], 'GPD': ['WRI1005113'], 'CARMA': ['CARMA42555']}"
+1427,South,Nuclear,,PP,UA,2852.837489363375,,0.0,0.0,0.0,0.0,1983.0,1989.0,,,47.812,31.22,[nan],"{'GEO': ['GEO5475'], 'GPD': ['WRI1005113'], 'CARMA': ['CARMA42555']}"
 1428,Rovno rivne,Nuclear,,PP,UA,2695.931427448389,,0.0,0.0,0.0,0.0,1981.0,2006.0,,,51.3245,25.89744,[nan],"{'GEO': ['GEO5174'], 'GPD': ['WRI1005112'], 'CARMA': ['CARMA38114']}"
-1429,Ladyzhinska,Hard Coal,Steam Turbine,PP,UA,1659.5849686814602,,0.0,0.0,0.0,0.0,1970.0,1971.0,,,48706,29.2202,[nan],"{'GEO': ['GEO42993'], 'GPD': ['WRI1005098'], 'CARMA': ['CARMA24024']}"
+1429,Ladyzhinska,Hard Coal,Steam Turbine,PP,UA,1659.5849686814602,,0.0,0.0,0.0,0.0,1970.0,1971.0,,,48.706,29.2202,[nan],"{'GEO': ['GEO42993'], 'GPD': ['WRI1005098'], 'CARMA': ['CARMA24024']}"
-1430,Kiev,Hydro,Pumped Storage,PP,UA,635.8694635681177,,0.0,0.0,0.0,0.0,1964.0,1972.0,,,50.5998,30501,"[nan, nan]","{'GEO': ['GEO43024', 'GEO43023'], 'GPD': ['WRI1005123', 'WRI1005124'], 'CARMA': ['CARMA23516', 'CARMA23517']}"
+1430,Kiev,Hydro,Pumped Storage,PP,UA,635.8694635681177,,0.0,0.0,0.0,0.0,1964.0,1972.0,,,50.5998,30.501,"[nan, nan]","{'GEO': ['GEO43024', 'GEO43023'], 'GPD': ['WRI1005123', 'WRI1005124'], 'CARMA': ['CARMA23516', 'CARMA23517']}"
 2450,Cet chisinau,Natural Gas,,PP,MD,306.0,,0.0,0.0,0.0,0.0,,,,,47.027550000000005,28.8801,"[nan, nan]","{'GPD': ['WRI1002985', 'WRI1002984'], 'CARMA': ['CARMA8450', 'CARMA8451']}"
 2460,Hydropower che costesti,Hydro,,PP,MD,16.0,,0.0,0.0,0.0,0.0,1978.0,,,,47.8381,27.2246,[nan],"{'GPD': ['WRI1002987'], 'CARMA': ['CARMA9496']}"
 2465,Moldavskaya gres,Hard Coal,,PP,MD,2520.0,,0.0,0.0,0.0,0.0,,,,,46.6292,29.9407,[nan],"{'GPD': ['WRI1002989'], 'CARMA': ['CARMA28979']}"
-2466,Hydropower dubasari,Hydro,,PP,MD,48.0,,0.0,0.0,0.0,0.0,,,,,47.2778,29123,[nan],"{'GPD': ['WRI1002988'], 'CARMA': ['CARMA11384']}"
+2466,Hydropower dubasari,Hydro,,PP,MD,48.0,,0.0,0.0,0.0,0.0,,,,,47.2778,29.123,[nan],"{'GPD': ['WRI1002988'], 'CARMA': ['CARMA11384']}"
 2676,Cet nord balti,Natural Gas,,PP,MD,24.0,,0.0,0.0,0.0,0.0,,,,,47.7492,27.8938,[nan],"{'GPD': ['WRI1002986'], 'CARMA': ['CARMA3071']}"
 2699,Dniprodzerzhynsk,Hydro,Reservoir,PP,UA,360.3503184713376,,0.0,0.0,0.0,0.0,1963.0,1964.0,,,48.5485,34.541015,[nan],"{'GEO': ['GEO43020'], 'GPD': ['WRI1005119']}"
 2707,Burshtynska tes,Hard Coal,Steam Turbine,PP,UA,2212.779958241947,,0.0,0.0,0.0,0.0,1965.0,1984.0,,,49.21038,24.66654,[nan],"{'GEO': ['GEO42991'], 'GPD': ['WRI1005097']}"
 2708,Danipro dnieper,Hydro,Reservoir,PP,UA,1484.8407643312103,,0.0,0.0,0.0,0.0,1932.0,1947.0,,,47.86944,35.08611,[nan],"{'GEO': ['GEO43016'], 'GPD': ['WRI1005120']}"
 2709,Dniester,Hydro,Pumped Storage,Store,UA,612.7241020616891,,0.0,0.0,0.0,0.0,2009.0,2011.0,,,48.51361,27.47333,[nan],"{'GEO': ['GEO43022'], 'GPD': ['WRI1005116', 'WRI1005115']}"
-2710,Kiev,Natural Gas,Steam Turbine,CHP,UA,458.2803237740955,,0.0,0.0,0.0,0.0,1982.0,1984.0,,,50532,30.6625,[nan],"{'GEO': ['GEO42998'], 'GPD': ['WRI1005125']}"
+2710,Kiev,Natural Gas,Steam Turbine,CHP,UA,458.2803237740955,,0.0,0.0,0.0,0.0,1982.0,1984.0,,,50.532,30.6625,[nan],"{'GEO': ['GEO42998'], 'GPD': ['WRI1005125']}"
 2712,Luganskaya,Hard Coal,Steam Turbine,PP,UA,1060.2903966575996,,0.0,0.0,0.0,0.0,1962.0,1969.0,,,48.74781,39.2624,[nan],"{'GEO': ['GEO42996'], 'GPD': ['WRI1005110']}"
-2713,Slavyanskaya,Hard Coal,Steam Turbine,PP,UA,737.5933194139823,,0.0,0.0,0.0,0.0,1971.0,1971.0,,,48872,37.76567,[nan],"{'GEO': ['GEO43002'], 'GPD': ['WRI1005109']}"
+2713,Slavyanskaya,Hard Coal,Steam Turbine,PP,UA,737.5933194139823,,0.0,0.0,0.0,0.0,1971.0,1971.0,,,48.872,37.76567,[nan],"{'GEO': ['GEO43002'], 'GPD': ['WRI1005109']}"
 2714,Vuhlehirska uglegorskaya,Hard Coal,Steam Turbine,PP,UA,3319.1699373629203,,0.0,0.0,0.0,0.0,1972.0,1977.0,,,48.4633,38.20328,[nan],"{'GEO': ['GEO43001'], 'GPD': ['WRI1005107']}"
 2715,Zaporiska,Hard Coal,Steam Turbine,PP,UA,3319.1699373629203,,0.0,0.0,0.0,0.0,1972.0,1977.0,,,47.5089,34.6253,[nan],"{'GEO': ['GEO42988'], 'GPD': ['WRI1005101']}"
 3678,Mironovskaya,Hard Coal,,PP,UA,815.0,,0.0,0.0,0.0,0.0,,,,,48.3407,38.4049,[nan],"{'GPD': ['WRI1005108'], 'CARMA': ['CARMA28679']}"
 3679,Kramatorskaya,Hard Coal,,PP,UA,120.0,,0.0,0.0,0.0,0.0,1974.0,,,,48.7477,37.5723,[nan],"{'GPD': ['WRI1075856'], 'CARMA': ['CARMA54560']}"
-3680,Chernihiv,Hard Coal,,PP,UA,200.0,,0.0,0.0,0.0,0.0,1968.0,,,,51455,31.2602,[nan],"{'GPD': ['WRI1075853'], 'CARMA': ['CARMA8190']}"
+3680,Chernihiv,Hard Coal,,PP,UA,200.0,,0.0,0.0,0.0,0.0,1968.0,,,,51.455,31.2602,[nan],"{'GPD': ['WRI1075853'], 'CARMA': ['CARMA8190']}"

doc/release_notes.rst

@@ -10,11 +10,16 @@ Release Notes
 Upcoming Release
 ================
 
+
 * fix bug in land transport for temperature correction of ICEs and fuel cell cars
 
 * restructure land transport, demand is now attached to one single node, the
 different car types (ICE, EV, fuel cell) are modelled as links
 
+* Upgrade to Snakemake v8.5+. This version is the new minimum version required.
+  To upgrade an existing environment, run ``conda install -c bioconda
+  snakemake-minimal">=8.5"`` and ``pip install snakemake-storage-plugin-http``
+  (https://github.com/PyPSA/pypsa-eur/pull/825).
 
 * Corrected a bug leading to power plants operating after their DateOut
   (https://github.com/PyPSA/pypsa-eur/pull/958). Added additional grouping years
@@ -105,6 +110,8 @@ different car types (ICE, EV, fuel cell) are modelled as links
 
 * Bugfix: allow modelling sector-coupled landlocked regions. (Fixed handling of offshore wind.)
 
+* Adapt the disabling of transmission expansion in myopic foresight optimisations when limit is already reached to also handle cost limits.
+
 PyPSA-Eur 0.10.0 (19th February 2024)
 =====================================
 

envs/environment.yaml

@@ -20,8 +20,7 @@ dependencies:
 - openpyxl!=3.1.1
 - pycountry
 - seaborn
-  # snakemake 8 introduced a number of breaking changes which the workflow has yet to be made compatible with
+- snakemake-minimal>=8.5
-- snakemake-minimal>=7.7.0,<8.0.0
 - memory_profiler
 - yaml
 - pytables
@@ -61,4 +60,7 @@ dependencies:
 
 - pip:
   - tsam>=2.3.1
+  - snakemake-storage-plugin-http
+  - snakemake-executor-plugin-slurm
+  - snakemake-executor-plugin-cluster-generic
   - highspy

envs/retrieve.yaml

@@ -8,6 +8,11 @@ channels:
 - bioconda
 dependencies:
 - python>=3.8
-- snakemake-minimal>=7.7.0,<8.0.0
+- pip
+- snakemake-minimal>=8.5
 - pandas>=2.1
 - tqdm
+- pip:
+  - snakemake-storage-plugin-http
+  - snakemake-executor-plugin-slurm
+  - snakemake-executor-plugin-cluster-generic

rules/build_sector.smk

@@ -93,7 +93,7 @@ rule build_gas_network:
 
 rule build_gas_input_locations:
     input:
-        gem=HTTP.remote(
+        gem=storage(
             "https://globalenergymonitor.org/wp-content/uploads/2023/07/Europe-Gas-Tracker-2023-03-v3.xlsx",
             keep_local=True,
         ),
@@ -293,7 +293,7 @@ rule build_biomass_potentials:
     params:
         biomass=config_provider("biomass"),
     input:
-        enspreso_biomass=HTTP.remote(
+        enspreso_biomass=storage(
             "https://zenodo.org/records/10356004/files/ENSPRESO_BIOMASS.xlsx",
             keep_local=True,
         ),
@@ -325,8 +325,8 @@ rule build_biomass_potentials:
 
 rule build_biomass_transport_costs:
     input:
-        transport_cost_data=HTTP.remote(
+        transport_cost_data=storage(
-            "publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf",
+            "https://publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf",
             keep_local=True,
         ),
     output:
@@ -350,7 +350,7 @@ rule build_sequestration_potentials:
             "sector", "regional_co2_sequestration_potential"
         ),
     input:
-        sequestration_potential=HTTP.remote(
+        sequestration_potential=storage(
             "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson",
             keep_local=True,
         ),

rules/retrieve.smk

@@ -64,9 +64,8 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True
 
     rule retrieve_cutout:
         input:
-            HTTP.remote(
+            storage(
-                "zenodo.org/record/6382570/files/{cutout}.nc",
+                "https://zenodo.org/record/6382570/files/{cutout}.nc",
-                static=True,
             ),
         output:
             protected("cutouts/" + CDIR + "{cutout}.nc"),
@@ -104,10 +103,9 @@ if config["enable"]["retrieve"] and config["enable"].get(
 
     rule retrieve_natura_raster:
         input:
-            HTTP.remote(
+            storage(
-                "zenodo.org/record/4706686/files/natura.tiff",
+                "https://zenodo.org/record/4706686/files/natura.tiff",
                 keep_local=True,
-                static=True,
             ),
         output:
             resources("natura.tiff"),
@@ -200,10 +198,9 @@ if config["enable"]["retrieve"]:
 
     rule retrieve_ship_raster:
         input:
-            HTTP.remote(
+            storage(
                 "https://zenodo.org/record/6953563/files/shipdensity_global.zip",
                 keep_local=True,
-                static=True,
             ),
         output:
             protected("data/shipdensity_global.zip"),
@@ -223,9 +220,8 @@ if config["enable"]["retrieve"]:
     # Website: https://land.copernicus.eu/global/products/lc
     rule download_copernicus_land_cover:
         input:
-            HTTP.remote(
+            storage(
-                "zenodo.org/record/3939050/files/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif",
+                "https://zenodo.org/record/3939050/files/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif",
-                static=True,
             ),
         output:
             "data/Copernicus_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif",
@@ -240,9 +236,8 @@ if config["enable"]["retrieve"]:
     # Website: https://ec.europa.eu/jrc/en/luisa
     rule retrieve_luisa_land_cover:
         input:
-            HTTP.remote(
+            storage(
-                "jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/LUISA/EUROPE/Basemaps/LandUse/2018/LATEST/LUISA_basemap_020321_50m.tif",
+                "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/LUISA/EUROPE/Basemaps/LandUse/2018/LATEST/LUISA_basemap_020321_50m.tif",
-                static=True,
             ),
         output:
             "data/LUISA_basemap_020321_50m.tif",
@@ -285,11 +280,7 @@ if config["enable"]["retrieve"]:
     # Website: https://www.protectedplanet.net/en/thematic-areas/wdpa
     rule download_wdpa:
         input:
-            HTTP.remote(
+            storage(url, keep_local=True),
-                url,
-                static=True,
-                keep_local=True,
-            ),
         params:
             zip="data/WDPA_shp.zip",
             folder=directory("data/WDPA"),
@@ -311,9 +302,8 @@ if config["enable"]["retrieve"]:
         # extract the main zip and then merge the contained 3 zipped shapefiles
         # Website: https://www.protectedplanet.net/en/thematic-areas/marine-protected-areas
         input:
-            HTTP.remote(
+            storage(
-                f"d1gam3xoknrgr2.cloudfront.net/current/WDPA_WDOECM_{bYYYY}_Public_marine_shp.zip",
+                f"https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_WDOECM_{bYYYY}_Public_marine_shp.zip",
-                static=True,
                 keep_local=True,
             ),
         params:
@@ -336,10 +326,9 @@ if config["enable"]["retrieve"]:
 
     rule retrieve_monthly_co2_prices:
         input:
-            HTTP.remote(
+            storage(
                 "https://www.eex.com/fileadmin/EEX/Downloads/EUA_Emission_Spot_Primary_Market_Auction_Report/Archive_Reports/emission-spot-primary-market-auction-report-2019-data.xls",
                 keep_local=True,
-                static=True,
             ),
         output:
             "data/validation/emission-spot-primary-market-auction-report-2019-data.xls",

rules/solve_electricity.smk

@@ -29,7 +29,7 @@ rule solve_network:
     threads: solver_threads
     resources:
         mem_mb=memory,
-        walltime=config_provider("solving", "walltime", default="12:00:00"),
+        runtime=config_provider("solving", "runtime", default="6h"),
     shadow:
         "minimal"
     conda:
@@ -60,7 +60,7 @@ rule solve_operations_network:
     threads: 4
     resources:
         mem_mb=(lambda w: 10000 + 372 * int(w.clusters)),
-        walltime=config_provider("solving", "walltime", default="12:00:00"),
+        runtime=config_provider("solving", "runtime", default="6h"),
     shadow:
         "minimal"
     conda:

rules/solve_myopic.smk

@@ -126,12 +126,14 @@ rule solve_sector_network_myopic:
     log:
         solver=RESULTS
         + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
+        memory=RESULTS
+        + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log",
         python=RESULTS
         + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
     threads: solver_threads
     resources:
-        mem_mb=config_provider("solving", "mem"),
+        mem_mb=config_provider("solving", "mem_mb"),
-        walltime=config_provider("solving", "walltime", default="12:00:00"),
+        runtime=config_provider("solving", "runtime", default="6h"),
     benchmark:
         (
             RESULTS

rules/solve_overnight.smk

@@ -30,8 +30,8 @@ rule solve_sector_network:
         + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
     threads: solver_threads
     resources:
-        mem_mb=config_provider("solving", "mem"),
+        mem_mb=config_provider("solving", "mem_mb"),
-        walltime=config_provider("solving", "walltime", default="12:00:00"),
+        runtime=config_provider("solving", "runtime", default="6h"),
     benchmark:
         (
             RESULTS

scripts/_helpers.py

@@ -354,7 +354,16 @@ def mock_snakemake(
 
     import snakemake as sm
     from pypsa.descriptors import Dict
+    from snakemake.api import Workflow
+    from snakemake.common import SNAKEFILE_CHOICES
     from snakemake.script import Snakemake
+    from snakemake.settings import (
+        ConfigSettings,
+        DAGSettings,
+        ResourceSettings,
+        StorageSettings,
+        WorkflowSettings,
+    )
 
     script_dir = Path(__file__).parent.resolve()
     if root_dir is None:
@@ -374,7 +383,7 @@ def mock_snakemake(
             f" {root_dir} or scripts directory {script_dir}"
         )
     try:
-        for p in sm.SNAKEFILE_CHOICES:
+        for p in SNAKEFILE_CHOICES:
             if os.path.exists(p):
                 snakefile = p
                 break
@@ -383,8 +392,18 @@ def mock_snakemake(
         elif isinstance(configfiles, str):
             configfiles = [configfiles]
 
-        workflow = sm.Workflow(
+        resource_settings = ResourceSettings()
-            snakefile, overwrite_configfiles=configfiles, rerun_triggers=[]
+        config_settings = ConfigSettings(configfiles=configfiles)
+        workflow_settings = WorkflowSettings()
+        storage_settings = StorageSettings()
+        dag_settings = DAGSettings(rerun_triggers=[])
+        workflow = Workflow(
+            config_settings,
+            resource_settings,
+            workflow_settings,
+            storage_settings,
+            dag_settings,
+            storage_provider_settings=dict(),
         )
         workflow.include(snakefile)
 

scripts/add_brownfield.py

@@ -124,29 +124,57 @@ def add_brownfield(n, n_p, year):
             n.links.loc[new_pipes, "p_nom_min"] = 0.0
 
 
-def disable_grid_expansion_if_LV_limit_hit(n):
+def disable_grid_expansion_if_limit_hit(n):
-    if "lv_limit" not in n.global_constraints.index:
+    """
-        return
+    Check if transmission expansion limit is already reached; then turn off.
 
+    In particular, this function checks if the total transmission
+    capital cost or volume implied by s_nom_min and p_nom_min are
+    numerically close to the respective global limit set in
+    n.global_constraints. If so, the nominal capacities are set to the
+    minimum and extendable is turned off; the corresponding global
+    constraint is then dropped.
+    """
+    cols = {"cost": "capital_cost", "volume": "length"}
+    for limit_type in ["cost", "volume"]:
+        glcs = n.global_constraints.query(
+            f"type == 'transmission_expansion_{limit_type}_limit'"
+        )
+
+        for name, glc in glcs.iterrows():
             total_expansion = (
-        n.lines.eval("s_nom_min * length").sum()
+                (
-        + n.links.query("carrier == 'DC'").eval("p_nom_min * length").sum()
+                    n.lines.query("p_nom_extendable")
+                    .eval(f"s_nom_min * {cols[limit_type]}")
+                    .sum()
+                )
+                + (
+                    n.links.query("carrier == 'DC' and p_nom_extendable")
+                    .eval(f"p_nom_min * {cols[limit_type]}")
+                    .sum()
+                )
             ).sum()
 
-    lv_limit = n.global_constraints.at["lv_limit", "constant"]
+            # Allow small numerical differences
-
+            if np.abs(glc.constant - total_expansion) / glc.constant < 1e-6:
-    # allow small numerical differences
+                logger.info(
-    if lv_limit - total_expansion < 1:
+                    f"Transmission expansion {limit_type} is already reached, disabling expansion and limit"
-        logger.info("LV is already reached, disabling expansion and LV limit")
+                )
                 extendable_acs = n.lines.query("s_nom_extendable").index
                 n.lines.loc[extendable_acs, "s_nom_extendable"] = False
-        n.lines.loc[extendable_acs, "s_nom"] = n.lines.loc[extendable_acs, "s_nom_min"]
+                n.lines.loc[extendable_acs, "s_nom"] = n.lines.loc[
+                    extendable_acs, "s_nom_min"
+                ]
 
-        extendable_dcs = n.links.query("carrier == 'DC' and p_nom_extendable").index
+                extendable_dcs = n.links.query(
+                    "carrier == 'DC' and p_nom_extendable"
+                ).index
                 n.links.loc[extendable_dcs, "p_nom_extendable"] = False
-        n.links.loc[extendable_dcs, "p_nom"] = n.links.loc[extendable_dcs, "p_nom_min"]
+                n.links.loc[extendable_dcs, "p_nom"] = n.links.loc[
+                    extendable_dcs, "p_nom_min"
+                ]
 
-        n.global_constraints.drop("lv_limit", inplace=True)
+                n.global_constraints.drop(name, inplace=True)
 
 
 def adjust_renewable_profiles(n, input_profiles, params, year):
@@ -233,7 +261,7 @@ if __name__ == "__main__":
 
     add_brownfield(n, n_p, year)
 
-    disable_grid_expansion_if_LV_limit_hit(n)
+    disable_grid_expansion_if_limit_hit(n)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output[0])

scripts/add_electricity.py

@@ -586,7 +586,7 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **par
         # fill missing max hours to params value and
         # assume no natural inflow due to lack of data
         max_hours = params.get("PHS_max_hours", 6)
-        phs = phs.replace({"max_hours": {0: max_hours}})
+        phs = phs.replace({"max_hours": {0: max_hours, np.nan: max_hours}})
         n.madd(
             "StorageUnit",
             phs.index,

scripts/build_electricity_demand.py

@@ -273,7 +273,7 @@ if __name__ == "__main__":
 
     if "UA" in countries:
         # attach load of UA (best data only for entsoe transparency)
-        load_ua = load_timeseries(snakemake.input[0], "2018", ["UA"], False)
+        load_ua = load_timeseries(snakemake.input[0], "2018", ["UA"])
         snapshot_year = str(snapshots.year.unique().item())
         time_diff = pd.Timestamp("2018") - pd.Timestamp(snapshot_year)
         load_ua.index -= (

scripts/build_gas_input_locations.py

@@ -25,7 +25,7 @@ def read_scigrid_gas(fn):
 
 
 def build_gem_lng_data(fn):
-    df = pd.read_excel(fn[0], sheet_name="LNG terminals - data")
+    df = pd.read_excel(fn, sheet_name="LNG terminals - data")
     df = df.set_index("ComboID")
 
     remove_country = ["Cyprus", "Turkey"]  # noqa: F841
@@ -46,7 +46,7 @@ def build_gem_lng_data(fn):
 
 
 def build_gem_prod_data(fn):
-    df = pd.read_excel(fn[0], sheet_name="Gas extraction - main")
+    df = pd.read_excel(fn, sheet_name="Gas extraction - main")
     df = df.set_index("GEM Unit ID")
 
     remove_country = ["Cyprus", "Türkiye"]  # noqa: F841
@@ -60,7 +60,7 @@ def build_gem_prod_data(fn):
               & ~Longitude.isna()"
     ).copy()
 
-    p = pd.read_excel(fn[0], sheet_name="Gas extraction - production")
+    p = pd.read_excel(fn, sheet_name="Gas extraction - production")
     p = p.set_index("GEM Unit ID")
     p = p[p["Fuel description"] == "gas"]