merge master

config/config.default.yaml

@@ -134,35 +134,25 @@ electricity:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite
 atlite:
-  default_cutout: europe-2013-era5
+  default_cutout: europe-2013-sarah3-era5
   nprocesses: 4
   show_progress: false
   cutouts:
     # use 'base' to determine geographical bounds and time span from config
     # base:
       # module: era5
-    europe-2013-era5:
-      module: era5 # in priority order
+    europe-2013-sarah3-era5:
+      module: [sarah, era5] # in priority order
       x: [-12., 42.]
-      y: [33., 72]
+      y: [33., 72.]
       dx: 0.3
       dy: 0.3
       time: ['2013', '2013']
-    europe-2013-sarah:
-      module: [sarah, era5] # in priority order
-      x: [-12., 42.]
-      y: [33., 65]
-      dx: 0.2
-      dy: 0.2
-      time: ['2013', '2013']
-      sarah_interpolate: false
-      sarah_dir:
-      features: [influx, temperature]
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#renewable
 renewable:
   onwind:
-    cutout: europe-2013-era5
+    cutout: europe-2013-sarah3-era5
     resource:
       method: wind
       turbine: Vestas_V112_3MW
@@ -181,7 +171,7 @@ renewable:
     excluder_resolution: 100
     clip_p_max_pu: 1.e-2
   offwind-ac:
-    cutout: europe-2013-era5
+    cutout: europe-2013-sarah3-era5
     resource:
       method: wind
       turbine: NREL_ReferenceTurbine_2020ATB_5.5MW
@@ -197,7 +187,7 @@ renewable:
     excluder_resolution: 200
     clip_p_max_pu: 1.e-2
   offwind-dc:
-    cutout: europe-2013-era5
+    cutout: europe-2013-sarah3-era5
     resource:
       method: wind
       turbine: NREL_ReferenceTurbine_2020ATB_5.5MW
@@ -213,7 +203,7 @@ renewable:
     excluder_resolution: 200
     clip_p_max_pu: 1.e-2
   offwind-float:
-    cutout: europe-2013-era5
+    cutout: europe-2013-sarah3-era5
     resource:
       method: wind
       turbine: NREL_ReferenceTurbine_5MW_offshore
@@ -231,7 +221,7 @@ renewable:
     max_depth: 1000
     clip_p_max_pu: 1.e-2
   solar:
-    cutout: europe-2013-sarah
+    cutout: europe-2013-sarah3-era5
     resource:
       method: pv
       panel: CSi
@@ -246,7 +236,7 @@ renewable:
     excluder_resolution: 100
     clip_p_max_pu: 1.e-2
   solar-hsat:
-    cutout: europe-2013-sarah
+    cutout: europe-2013-sarah3-era5
     resource:
       method: pv
       panel: CSi
@@ -261,7 +251,7 @@ renewable:
     excluder_resolution: 100
     clip_p_max_pu: 1.e-2
   hydro:
-    cutout: europe-2013-era5
+    cutout: europe-2013-sarah3-era5
     carriers: [ror, PHS, hydro]
     PHS_max_hours: 6
     hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float
@@ -295,7 +285,7 @@ lines:
   under_construction: 'keep' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
   dynamic_line_rating:
     activate: false
-    cutout: europe-2013-era5
+    cutout: europe-2013-sarah3-era5
     correction_factor: 0.95
     max_voltage_difference: false
     max_line_rating: false
@@ -786,6 +776,7 @@ solving:
   options:
     clip_p_max_pu: 1.e-2
     load_shedding: false
+    curtailment_mode: false
     noisy_costs: true
     skip_iterations: true
     rolling_horizon: false
@@ -830,7 +821,7 @@ solving:
   solver_options:
     highs-default:
       # refer to https://ergo-code.github.io/HiGHS/dev/options/definitions/
-      threads: 4
+      threads: 1
       solver: "ipm"
       run_crossover: "off"
       small_matrix_value: 1e-6
@@ -841,7 +832,7 @@ solving:
       parallel: "on"
       random_seed: 123
     gurobi-default:
-      threads: 4
+      threads: 8
       method: 2 # barrier
       crossover: 0
       BarConvTol: 1.e-6
@@ -1063,7 +1054,7 @@ plotting:
     V2G: '#e5ffa8'
     land transport EV: '#baf238'
     land transport demand: '#38baf2'
-    Li ion: '#baf238'
+    EV battery: '#baf238'
     # hot water storage
     water tanks: '#e69487'
     residential rural water tanks: '#f7b7a3'

doc/configtables/atlite.csv

@@ -3,7 +3,7 @@ default_cutout,--,str,"Defines a default cutout."
 nprocesses,--,int,"Number of parallel processes in cutout preparation"
 show_progress,bool,true/false,"Whether progressbar for atlite conversion processes should be shown. False saves time."
 cutouts,,,
--- {name},--,"Convention is to name cutouts like ``<region>-<year>-<source>`` (e.g. ``europe-2013-era5``).","Name of the cutout netcdf file. The user may specify multiple cutouts under configuration ``atlite: cutouts:``. Reference is used in configuration ``renewable: {technology}: cutout:``. The cutout ``base`` may be used to automatically calculate temporal and spatial bounds of the network."
+-- {name},--,"Convention is to name cutouts like ``<region>-<year>-<source>`` (e.g. ``europe-2013-sarah3-era5``).","Name of the cutout netcdf file. The user may specify multiple cutouts under configuration ``atlite: cutouts:``. Reference is used in configuration ``renewable: {technology}: cutout:``. The cutout ``base`` may be used to automatically calculate temporal and spatial bounds of the network."
 -- -- module,--,"Subset of {'era5','sarah'}","Source of the reanalysis weather dataset (e.g. `ERA5 <https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5>`_ or `SARAH-2 <https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=SARAH_V002>`_)"
 -- -- x,°,"Float interval within [-180, 180]","Range of longitudes to download weather data for. If not defined, it defaults to the spatial bounds of all bus shapes."
 -- -- y,°,"Float interval within [-90, 90]","Range of latitudes to download weather data for. If not defined, it defaults to the spatial bounds of all bus shapes."

doc/configtables/hydro.csv

@@ -1,5 +1,5 @@
 ,Unit,Values,Description
-cutout,--,Must be 'europe-2013-era5',Specifies the directory where the relevant weather data ist stored.
+cutout,--,Must be 'europe-2013-sarah3-era5',Specifies the directory where the relevant weather data ist stored.
 carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydro power plants to build per-unit availability time series for. 'ror' stands for run-of-river plants, 'PHS' represents pumped-hydro storage, and 'hydro' stands for hydroelectric dams."
 PHS_max_hours,h,float,Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation <https://pypsa.readthedocs.io/en/latest/components.html#storage-unit>`_.
 hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}",Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation <https://pypsa.readthedocs.io/en/latest/components.html#storage-unit>`_.

doc/configtables/lines.csv

@@ -8,7 +8,7 @@ under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove co
 reconnect_crimea,--,"true or false","Whether to reconnect Crimea to the Ukrainian grid"
 dynamic_line_rating,,,
 -- activate,bool,"true or false","Whether to take dynamic line rating into account"
--- cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
+-- cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-sarah3-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
 -- correction_factor,--,"float","Factor to compensate for overestimation of wind speeds in hourly averaged wind data"
 -- max_voltage_difference,deg,"float","Maximum voltage angle difference in degrees or 'false' to disable"
 -- max_line_rating,--,"float","Maximum line rating relative to nominal capacity without DLR, e.g. 1.3 or 'false' to disable"

doc/configtables/offwind-ac.csv

@@ -1,5 +1,5 @@
 ,Unit,Values,Description
-cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
+cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-sarah3-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
 resource,,,
 -- method,--,"Must be 'wind'","A superordinate technology type."
 -- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`_.  Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve."

doc/configtables/offwind-dc.csv

@@ -1,5 +1,5 @@
 ,Unit,Values,Description
-cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
+cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-sarah3-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
 resource,,,
 -- method,--,"Must be 'wind'","A superordinate technology type."
 -- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`_. Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve."

doc/configtables/onwind.csv

@@ -1,5 +1,5 @@
 ,Unit,Values,Description
-cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
+cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-sarah3-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
 resource,,,
 -- method,--,"Must be 'wind'","A superordinate technology type."
 -- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`_. Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve."

doc/configtables/solar.csv

@@ -1,5 +1,5 @@
 ,Unit,Values,Description
-cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module can be ERA5 or SARAH-2.","Specifies the directory where the relevant weather data ist stored that is specified at ``atlite/cutouts`` configuration. Both ``sarah`` and ``era5`` work."
+cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-sarah3-era5') or reference an existing folder in the directory ``cutouts``. Source module can be ERA5 or SARAH-2.","Specifies the directory where the relevant weather data ist stored that is specified at ``atlite/cutouts`` configuration. Both ``sarah`` and ``era5`` work."
 resource,,,
 -- method,--,"Must be 'pv'","A superordinate technology type."
 -- panel,--,"One of {'Csi', 'CdTe', 'KANENA'} as defined in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/solarpanel>`_ . Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the solar panel technology and its characteristic attributes."

doc/configtables/solving.csv

@@ -2,6 +2,7 @@
 options,,,
 -- clip_p_max_pu,p.u.,float,To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero.
 -- load_shedding,bool/float,"{'true','false', float}","Add generators with very high marginal cost to simulate load shedding and avoid problem infeasibilities. If load shedding is a float, it denotes the marginal cost in EUR/kWh."
+-- curtailment_mode,bool/float,"{'true','false'}","Fixes the dispatch profiles of generators with time-varying p_max_pu by setting ``p_min_pu = p_max_pu`` and adds an auxiliary curtailment generator (with negative sign to absorb excess power) at every AC bus. This can speed up the solving process as the curtailment decision is aggregated into a single generator per region. Defaults to ``false``."
 -- noisy_costs,bool,"{'true','false'}","Add random noise to marginal cost of generators by :math:`\mathcal{U}(0.009,0,011)` and capital cost of lines and links by :math:`\mathcal{U}(0.09,0,11)`."
 -- skip_iterations,bool,"{'true','false'}","Skip iterating, do not update impedances of branches. Defaults to true."
 -- rolling_horizon,bool,"{'true','false'}","Switch for rule :mod:`solve_operations_network` whether to optimize the network in a rolling horizon manner, where the snapshot range is split into slices of size `horizon` which are solved consecutively. This setting has currently no effect on sector-coupled networks."

doc/preparation.rst

@@ -16,7 +16,7 @@ using the ``retrieve*`` rules (:ref:`data`).
 Having downloaded the necessary data,
 
 - :mod:`build_shapes` generates GeoJSON files with shapes of the countries, exclusive economic zones and `NUTS3 <https://en.wikipedia.org/wiki/Nomenclature_of_Territorial_Units_for_Statistics>`__ areas.
-- :mod:`build_cutout` prepares smaller weather data portions from `ERA5 <https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5>`__ for cutout ``europe-2013-era5`` and SARAH for cutout ``europe-2013-sarah``.
+- :mod:`build_cutout` prepares smaller weather data portions from `ERA5 <https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5>`__ for cutout ``europe-2013-sarah3-era5`` and SARAH for cutout ``europe-2013-sarah``.
 
 With these and the externally extracted ENTSO-E online map topology
 (``data/entsoegridkit``), it can build a base PyPSA network with the following rules:

doc/release_notes.rst

@@ -13,6 +13,13 @@ Upcoming Release
 * Upadte JRC-IDEES-2015 to  `JRC-IDEES-2021
 <https://publications.jrc.ec.europa.eu/repository/handle/JRC137809>`__.
 
+
+* Updata Ammonia production from USGS to 2022 `data
+<https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/media/files/myb1-2022-nitro-ert.xlsx>`__.
+
+
+* Renamed the carrier of batteries in BEVs from `battery storage` to `EV battery` and the corresponding bus carrier from `Li ion` to `EV battery`. This is to avoid confusion with stationary battery storage.
+
 * Changed default assumptions about waste heat usage from PtX and fuel cells in district heating.
   The default value for the link efficiency scaling factor was changed from 100% to 25%.
   It can be set to other values in the configuration ``sector: use_TECHNOLOGY_waste_heat``.

doc/wildcards.rst

@@ -142,13 +142,6 @@ The ``{sector_opts}`` wildcard is only used for sector-coupling studies.
    :widths: 10,20,10,10
    :file: configtables/sector-opts.csv
 
-.. _scope:
-
-The ``{scope}`` wildcard
-========================
-
-Takes values ``residential``, ``urban``, ``total``.
-
 .. _planning_horizons:
 
 The ``{planning_horizons}`` wildcard

rules/build_sector.smk

@@ -151,18 +151,18 @@ rule build_daily_heat_demand:
         snapshots=config_provider("snapshots"),
         drop_leap_day=config_provider("enable", "drop_leap_day"),
     input:
-        pop_layout=resources("pop_layout_{scope}.nc"),
+        pop_layout=resources("pop_layout_total.nc"),
         regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
         cutout=heat_demand_cutout,
     output:
-        heat_demand=resources("daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"),
+        heat_demand=resources("daily_heat_demand_total_elec_s{simpl}_{clusters}.nc"),
     resources:
         mem_mb=20000,
     threads: 8
     log:
-        logs("build_daily_heat_demand_{scope}_{simpl}_{clusters}.loc"),
+        logs("build_daily_heat_demand_total_{simpl}_{clusters}.loc"),
     benchmark:
-        benchmarks("build_daily_heat_demand/{scope}_s{simpl}_{clusters}")
+        benchmarks("build_daily_heat_demand/total_s{simpl}_{clusters}")
     conda:
         "../envs/environment.yaml"
     script:
@@ -175,16 +175,16 @@ rule build_hourly_heat_demand:
         drop_leap_day=config_provider("enable", "drop_leap_day"),
     input:
         heat_profile="data/heat_load_profile_BDEW.csv",
-        heat_demand=resources("daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"),
+        heat_demand=resources("daily_heat_demand_total_elec_s{simpl}_{clusters}.nc"),
     output:
-        heat_demand=resources("hourly_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"),
+        heat_demand=resources("hourly_heat_demand_total_elec_s{simpl}_{clusters}.nc"),
     resources:
         mem_mb=2000,
     threads: 8
     log:
-        logs("build_hourly_heat_demand_{scope}_{simpl}_{clusters}.loc"),
+        logs("build_hourly_heat_demand_total_{simpl}_{clusters}.loc"),
     benchmark:
-        benchmarks("build_hourly_heat_demand/{scope}_s{simpl}_{clusters}")
+        benchmarks("build_hourly_heat_demand/total_s{simpl}_{clusters}")
     conda:
         "../envs/environment.yaml"
     script:
@@ -196,19 +196,19 @@ rule build_temperature_profiles:
         snapshots=config_provider("snapshots"),
         drop_leap_day=config_provider("enable", "drop_leap_day"),
     input:
-        pop_layout=resources("pop_layout_{scope}.nc"),
+        pop_layout=resources("pop_layout_total.nc"),
         regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
         cutout=heat_demand_cutout,
     output:
-        temp_soil=resources("temp_soil_{scope}_elec_s{simpl}_{clusters}.nc"),
-        temp_air=resources("temp_air_{scope}_elec_s{simpl}_{clusters}.nc"),
+        temp_soil=resources("temp_soil_total_elec_s{simpl}_{clusters}.nc"),
+        temp_air=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"),
     resources:
         mem_mb=20000,
     threads: 8
     log:
-        logs("build_temperature_profiles_{scope}_{simpl}_{clusters}.log"),
+        logs("build_temperature_profiles_total_{simpl}_{clusters}.log"),
     benchmark:
-        benchmarks("build_temperature_profiles/{scope}_s{simpl}_{clusters}")
+        benchmarks("build_temperature_profiles/total_s{simpl}_{clusters}")
     conda:
         "../envs/environment.yaml"
     script:
@@ -220,18 +220,10 @@ rule build_cop_profiles:
         heat_pump_sink_T=config_provider("sector", "heat_pump_sink_T"),
     input:
         temp_soil_total=resources("temp_soil_total_elec_s{simpl}_{clusters}.nc"),
-        temp_soil_rural=resources("temp_soil_rural_elec_s{simpl}_{clusters}.nc"),
-        temp_soil_urban=resources("temp_soil_urban_elec_s{simpl}_{clusters}.nc"),
         temp_air_total=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"),
-        temp_air_rural=resources("temp_air_rural_elec_s{simpl}_{clusters}.nc"),
-        temp_air_urban=resources("temp_air_urban_elec_s{simpl}_{clusters}.nc"),
     output:
         cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
-        cop_soil_rural=resources("cop_soil_rural_elec_s{simpl}_{clusters}.nc"),
-        cop_soil_urban=resources("cop_soil_urban_elec_s{simpl}_{clusters}.nc"),
         cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
-        cop_air_rural=resources("cop_air_rural_elec_s{simpl}_{clusters}.nc"),
-        cop_air_urban=resources("cop_air_urban_elec_s{simpl}_{clusters}.nc"),
     resources:
         mem_mb=20000,
     log:
@@ -263,18 +255,18 @@ rule build_solar_thermal_profiles:
         drop_leap_day=config_provider("enable", "drop_leap_day"),
         solar_thermal=config_provider("solar_thermal"),
     input:
-        pop_layout=resources("pop_layout_{scope}.nc"),
+        pop_layout=resources("pop_layout_total.nc"),
         regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
         cutout=solar_thermal_cutout,
     output:
-        solar_thermal=resources("solar_thermal_{scope}_elec_s{simpl}_{clusters}.nc"),
+        solar_thermal=resources("solar_thermal_total_elec_s{simpl}_{clusters}.nc"),
     resources:
         mem_mb=20000,
     threads: 16
     log:
-        logs("build_solar_thermal_profiles_{scope}_s{simpl}_{clusters}.log"),
+        logs("build_solar_thermal_profiles_total_s{simpl}_{clusters}.log"),
     benchmark:
-        benchmarks("build_solar_thermal_profiles/{scope}_s{simpl}_{clusters}")
+        benchmarks("build_solar_thermal_profiles/total_s{simpl}_{clusters}")
     conda:
         "../envs/environment.yaml"
     script:
@@ -1024,32 +1016,14 @@ rule prepare_sector_network:
             "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv"
         ),
         temp_soil_total=resources("temp_soil_total_elec_s{simpl}_{clusters}.nc"),
-        temp_soil_rural=resources("temp_soil_rural_elec_s{simpl}_{clusters}.nc"),
-        temp_soil_urban=resources("temp_soil_urban_elec_s{simpl}_{clusters}.nc"),
         temp_air_total=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"),
-        temp_air_rural=resources("temp_air_rural_elec_s{simpl}_{clusters}.nc"),
-        temp_air_urban=resources("temp_air_urban_elec_s{simpl}_{clusters}.nc"),
         cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
-        cop_soil_rural=resources("cop_soil_rural_elec_s{simpl}_{clusters}.nc"),
-        cop_soil_urban=resources("cop_soil_urban_elec_s{simpl}_{clusters}.nc"),
         cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
-        cop_air_rural=resources("cop_air_rural_elec_s{simpl}_{clusters}.nc"),
-        cop_air_urban=resources("cop_air_urban_elec_s{simpl}_{clusters}.nc"),
         solar_thermal_total=lambda w: (
             resources("solar_thermal_total_elec_s{simpl}_{clusters}.nc")
             if config_provider("sector", "solar_thermal")(w)
             else []
         ),
-        solar_thermal_urban=lambda w: (
-            resources("solar_thermal_urban_elec_s{simpl}_{clusters}.nc")
-            if config_provider("sector", "solar_thermal")(w)
-            else []
-        ),
-        solar_thermal_rural=lambda w: (
-            resources("solar_thermal_rural_elec_s{simpl}_{clusters}.nc")
-            if config_provider("sector", "solar_thermal")(w)
-            else []
-        ),
         egs_potentials=lambda w: (
             resources("egs_potentials_s{simpl}_{clusters}.csv")
             if config_provider("sector", "enhanced_geothermal", "enable")(w)

rules/retrieve.smk

@@ -89,7 +89,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True
     rule retrieve_cutout:
         input:
             storage(
-                "https://zenodo.org/records/6382570/files/{cutout}.nc",
+                "https://zenodo.org/records/12791128/files/{cutout}.nc",
             ),
         output:
             protected("cutouts/" + CDIR + "{cutout}.nc"),

scripts/add_brownfield.py

@@ -89,10 +89,6 @@ def add_brownfield(n, n_p, year):
     # deal with gas network
     pipe_carrier = ["gas pipeline"]
     if snakemake.params.H2_retrofit:
-        # drop capacities of previous year to avoid duplicating
-        to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year)
-        n.mremove("Link", n.links.loc[to_drop].index)
-
         # subtract the already retrofitted from today's gas grid capacity
         h2_retrofitted_fixed_i = n.links[
             (n.links.carrier == "H2 pipeline retrofitted")
@@ -115,10 +111,6 @@ def add_brownfield(n, n_p, year):
             index=pipe_capacity.index
         ).fillna(0)
         n.links.loc[gas_pipes_i, "p_nom"] = remaining_capacity
-    else:
-        new_pipes = n.links.carrier.isin(pipe_carrier) & (n.links.build_year == year)
-        n.links.loc[new_pipes, "p_nom"] = 0.0
-        n.links.loc[new_pipes, "p_nom_min"] = 0.0
 
 
 def disable_grid_expansion_if_limit_hit(n):

scripts/add_electricity.py

@@ -852,7 +852,7 @@ if __name__ == "__main__":
         fuel_price = pd.read_csv(
             snakemake.input.fuel_price, index_col=0, header=0, parse_dates=True
         )
-        fuel_price = fuel_price.reindex(n.snapshots).fillna(method="ffill")
+        fuel_price = fuel_price.reindex(n.snapshots).ffill()
     else:
         fuel_price = None
 

scripts/base_network.py

@@ -72,6 +72,7 @@ Creates the network topology from an ENTSO-E map extract, and create Voronoi sha
 """
 
 import logging
+import warnings
 from itertools import product
 
 import geopandas as gpd
@@ -85,9 +86,9 @@ import shapely.wkt
 import yaml
 from _helpers import REGION_COLS, configure_logging, get_snapshots, set_scenario_config
 from packaging.version import Version, parse
-from scipy import spatial
 from scipy.sparse import csgraph
-from shapely.geometry import LineString, Point, Polygon
+from scipy.spatial import KDTree
+from shapely.geometry import LineString, Point
 
 PD_GE_2_2 = parse(pd.__version__) >= Version("2.2")
 
@@ -118,7 +119,7 @@ def _find_closest_links(links, new_links, distance_upper_bound=1.5):
     querycoords = np.vstack(
         [new_links[["x1", "y1", "x2", "y2"]], new_links[["x2", "y2", "x1", "y1"]]]
     )
-    tree = spatial.KDTree(treecoords)
+    tree = KDTree(treecoords)
     dist, ind = tree.query(querycoords, distance_upper_bound=distance_upper_bound)
     found_b = ind < len(links)
     found_i = np.arange(len(new_links) * 2)[found_b] % len(new_links)
@@ -273,7 +274,7 @@ def _add_links_from_tyndp(buses, links, links_tyndp, europe_shape):
             return buses, links
 
     tree_buses = buses.query("carrier=='AC'")
-    tree = spatial.KDTree(tree_buses[["x", "y"]])
+    tree = KDTree(tree_buses[["x", "y"]])
     _, ind0 = tree.query(links_tyndp[["x1", "y1"]])
     ind0_b = ind0 < len(tree_buses)
     links_tyndp.loc[ind0_b, "bus0"] = tree_buses.index[ind0[ind0_b]]
@@ -788,59 +789,26 @@ def base_network(
     return n
 
 
-def voronoi_partition_pts(points, outline):
+def voronoi(points, outline, crs=4326):
     """
-    Compute the polygons of a voronoi partition of `points` within the polygon
-    `outline`. Taken from
-    https://github.com/FRESNA/vresutils/blob/master/vresutils/graph.py.
-
-    Attributes
-    ----------
-    points : Nx2 - ndarray[dtype=float]
-    outline : Polygon
-    Returns
-    -------
-    polygons : N - ndarray[dtype=Polygon|MultiPolygon]
+    Create Voronoi polygons from a set of points within an outline.
     """
-    points = np.asarray(points)
-
-    if len(points) == 1:
-        polygons = [outline]
-    else:
-        xmin, ymin = np.amin(points, axis=0)
-        xmax, ymax = np.amax(points, axis=0)
-        xspan = xmax - xmin
-        yspan = ymax - ymin
-
-        # to avoid any network positions outside all Voronoi cells, append
-        # the corners of a rectangle framing these points
-        vor = spatial.Voronoi(
-            np.vstack(
-                (
-                    points,
-                    [
-                        [xmin - 3.0 * xspan, ymin - 3.0 * yspan],
-                        [xmin - 3.0 * xspan, ymax + 3.0 * yspan],
-                        [xmax + 3.0 * xspan, ymin - 3.0 * yspan],
-                        [xmax + 3.0 * xspan, ymax + 3.0 * yspan],
-                    ],
-                )
-            )
+    pts = gpd.GeoSeries(
+        gpd.points_from_xy(points.x, points.y),
+        index=points.index,
+        crs=crs,
     )
+    voronoi = pts.voronoi_polygons(extend_to=outline).clip(outline)
 
-        polygons = []
-        for i in range(len(points)):
-            poly = Polygon(vor.vertices[vor.regions[vor.point_region[i]]])
+    # can be removed with shapely 2.1 where order is preserved
+    # https://github.com/shapely/shapely/issues/2020
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", category=UserWarning)
+        pts = gpd.GeoDataFrame(geometry=pts)
+        voronoi = gpd.GeoDataFrame(geometry=voronoi)
+        joined = gpd.sjoin_nearest(pts, voronoi, how="right")
 
-            if not poly.is_valid:
-                poly = poly.buffer(0)
-
-            with np.errstate(invalid="ignore"):
-                poly = poly.intersection(outline)
-
-            polygons.append(poly)
-
-    return polygons
+    return joined.dissolve(by="Bus").squeeze()
 
 
 def build_bus_shapes(n, country_shapes, offshore_shapes, countries):
@@ -870,9 +838,7 @@ def build_bus_shapes(n, country_shapes, offshore_shapes, countries):
                     "name": onshore_locs.index,
                     "x": onshore_locs["x"],
                     "y": onshore_locs["y"],
-                    "geometry": voronoi_partition_pts(
-                        onshore_locs.values, onshore_shape
-                    ),
+                    "geometry": voronoi(onshore_locs, onshore_shape),
                     "country": country,
                 },
                 crs=n.crs,
@@ -888,7 +854,7 @@ def build_bus_shapes(n, country_shapes, offshore_shapes, countries):
                 "name": offshore_locs.index,
                 "x": offshore_locs["x"],
                 "y": offshore_locs["y"],
-                "geometry": voronoi_partition_pts(offshore_locs.values, offshore_shape),
+                "geometry": voronoi(offshore_locs, offshore_shape),
                 "country": country,
             },
             crs=n.crs,

scripts/build_cop_profiles.py

@@ -21,20 +21,12 @@ Relevant Settings
 Inputs:
 -------
 - ``resources/<run_name>/temp_soil_total_elec_s<simpl>_<clusters>.nc``: Soil temperature (total) time series.
-- ``resources/<run_name>/temp_soil_rural_elec_s<simpl>_<clusters>.nc``: Soil temperature (rural) time series.
-- ``resources/<run_name>/temp_soil_urban_elec_s<simpl>_<clusters>.nc``: Soil temperature (urban) time series.
 - ``resources/<run_name>/temp_air_total_elec_s<simpl>_<clusters>.nc``: Ambient air temperature (total) time series.
-- ``resources/<run_name>/temp_air_rural_elec_s<simpl>_<clusters>.nc``: Ambient air temperature (rural) time series.
-- ``resources/<run_name>/temp_air_urban_elec_s<simpl>_<clusters>.nc``: Ambient air temperature (urban) time series.
 
 Outputs:
 --------
 - ``resources/cop_soil_total_elec_s<simpl>_<clusters>.nc``: COP (ground-sourced) time series (total).
-- ``resources/cop_soil_rural_elec_s<simpl>_<clusters>.nc``: COP (ground-sourced) time series (rural).
-- ``resources/cop_soil_urban_elec_s<simpl>_<clusters>.nc``: COP (ground-sourced) time series (urban).
 - ``resources/cop_air_total_elec_s<simpl>_<clusters>.nc``: COP (air-sourced) time series (total).
-- ``resources/cop_air_rural_elec_s<simpl>_<clusters>.nc``: COP (air-sourced) time series (rural).
-- ``resources/cop_air_urban_elec_s<simpl>_<clusters>.nc``: COP (air-sourced) time series (urban).
 
 
 References
@@ -67,12 +59,11 @@ if __name__ == "__main__":
 
     set_scenario_config(snakemake)
 
-    for area in ["total", "urban", "rural"]:
     for source in ["air", "soil"]:
-            source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"])
+        source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_total"])
 
         delta_T = snakemake.params.heat_pump_sink_T - source_T
 
         cop = coefficient_of_performance(delta_T, source)
 
-            cop.to_netcdf(snakemake.output[f"cop_{source}_{area}"])
+        cop.to_netcdf(snakemake.output[f"cop_{source}_total"])

scripts/build_cutout.py

@@ -103,7 +103,7 @@ if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
 
-        snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5")
+        snakemake = mock_snakemake("build_cutout", cutout="europe-2013-sarah3-era5")
     configure_logging(snakemake)
     set_scenario_config(snakemake)
 

scripts/build_hourly_heat_demand.py

@@ -22,12 +22,12 @@ Inputs
 ------
 
 - ``data/heat_load_profile_BDEW.csv``: Intraday heat profile for water and space heating demand for the residential and services sectors for weekends and weekdays.
-- ``resources/daily_heat_demand_<scope>_elec_s<simpl>_<clusters>.nc``: Daily heat demand per cluster.
+- ``resources/daily_heat_demand_total_elec_s<simpl>_<clusters>.nc``: Daily heat demand per cluster.
 
 Outputs
 -------
 
-- ``resources/hourly_heat_demand_<scope>_elec_s<simpl>_<clusters>.nc``:
+- ``resources/hourly_heat_demand_total_elec_s<simpl>_<clusters>.nc``:
 """
 
 from itertools import product

scripts/build_industry_sector_ratios_intermediate.py

@@ -129,11 +129,12 @@ def build_industry_sector_ratios_intermediate():
         ]
         today_sector_ratios_ct.loc[:, ~missing_mask] = today_sector_ratios_ct.loc[
             :, ~missing_mask
-        ].fillna(0)
+        ].fillna(future_sector_ratios)
         intermediate_sector_ratios[ct] = (
             today_sector_ratios_ct * (1 - fraction_future)
             + future_sector_ratios * fraction_future
         )
+
     intermediate_sector_ratios = pd.concat(intermediate_sector_ratios, axis=1)
 
     intermediate_sector_ratios.to_csv(snakemake.output.industry_sector_ratios)

scripts/build_retro_cost.py

@@ -890,7 +890,7 @@ def calculate_gain_utilisation_factor(heat_transfer_perm2, Q_ht, Q_gain):
     Calculates gain utilisation factor nu.
     """
     # time constant of the building tau [h] = c_m [Wh/(m^2K)] * 1 /(H_tr_e+H_tb*H_ve) [m^2 K /W]
-    tau = c_m / heat_transfer_perm2.T.groupby(axis=1).sum().T
+    tau = c_m / heat_transfer_perm2.groupby().sum()
     alpha = alpha_H_0 + (tau / tau_H_0)
     # heat balance ratio
     gamma = (1 / Q_ht).mul(Q_gain.sum(axis=1), axis=0)

scripts/build_temperature_profiles.py

@@ -25,15 +25,15 @@ Relevant Settings
 Inputs
 ------
 
-- ``resources/<run_name>/pop_layout_<scope>.nc``:
+- ``resources/<run_name>/pop_layout_total.nc``:
 - ``resources/<run_name>/regions_onshore_elec_s<simpl>_<clusters>.geojson``:
 - ``cutout``: Weather data cutout, as specified in config
 
 Outputs
 -------
 
-- ``resources/temp_soil_<scope>_elec_s<simpl>_<clusters>.nc``:
-- ``resources/temp_air_<scope>_elec_s<simpl>_<clusters>.nc`
+- ``resources/temp_soil_total_elec_s<simpl>_<clusters>.nc``:
+- ``resources/temp_air_total_elec_s<simpl>_<clusters>.nc`
 """
 
 import atlite

scripts/cluster_gas_network.py

@@ -58,6 +58,9 @@ def build_clustered_gas_network(df, bus_regions, length_factor=1.25):
     # drop pipes within the same region
     df = df.loc[df.bus1 != df.bus0]
 
+    if df.empty:
+        return df
+
     # recalculate lengths as center to center * length factor
     df["length"] = df.apply(
         lambda p: length_factor

scripts/make_summary_perfect.py

@@ -631,7 +631,7 @@ def calculate_co2_emissions(n, label, df):
     weightings = n.snapshot_weightings.generators.mul(
         n.investment_period_weightings["years"]
         .reindex(n.snapshots)
-        .fillna(method="bfill")
+        .bfill()
         .fillna(1.0),
         axis=0,
     )

scripts/plot_validation_electricity_production.py

@@ -70,7 +70,7 @@ if __name__ == "__main__":
     optimized = optimized[["Generator", "StorageUnit"]].droplevel(0, axis=1)
     optimized = optimized.rename(columns=n.buses.country, level=0)
     optimized = optimized.rename(columns=carrier_groups, level=1)
-    optimized = optimized.groupby(axis=1, level=[0, 1]).sum()
+    optimized = optimized.T.groupby(level=[0, 1]).sum().T
 
     data = pd.concat([historic, optimized], keys=["Historic", "Optimized"], axis=1)
     data.columns.names = ["Kind", "Country", "Carrier"]

scripts/prepare_network.py

@@ -137,9 +137,7 @@ def add_emission_prices(n, emission_prices={"co2": 0.0}, exclude_co2=False):
 def add_dynamic_emission_prices(n):
     co2_price = pd.read_csv(snakemake.input.co2_price, index_col=0, parse_dates=True)
     co2_price = co2_price[~co2_price.index.duplicated()]
-    co2_price = (
-        co2_price.reindex(n.snapshots).fillna(method="ffill").fillna(method="bfill")
-    )
+    co2_price = co2_price.reindex(n.snapshots).ffill().bfill()
 
     emissions = (
         n.generators.carrier.map(n.carriers.co2_emissions) / n.generators.efficiency

scripts/prepare_perfect_foresight.py

@@ -250,7 +250,7 @@ def adjust_stores(n):
     n.stores.loc[cyclic_i, "e_cyclic_per_period"] = True
     n.stores.loc[cyclic_i, "e_cyclic"] = False
     # non cyclic store assumptions
-    non_cyclic_store = ["co2", "co2 stored", "solid biomass", "biogas", "Li ion"]
+    non_cyclic_store = ["co2", "co2 stored", "solid biomass", "biogas", "EV battery"]
     co2_i = n.stores[n.stores.carrier.isin(non_cyclic_store)].index
     n.stores.loc[co2_i, "e_cyclic_per_period"] = False
     n.stores.loc[co2_i, "e_cyclic"] = False

scripts/prepare_sector_network.py

@@ -1238,12 +1238,14 @@ def add_storage_and_grids(n, costs):
             gas_pipes["p_nom_min"] = 0.0
             # 0.1 EUR/MWkm/a to prefer decommissioning to address degeneracy
             gas_pipes["capital_cost"] = 0.1 * gas_pipes.length
+            gas_pipes["p_nom_extendable"] = True
         else:
             gas_pipes["p_nom_max"] = np.inf
             gas_pipes["p_nom_min"] = gas_pipes.p_nom
             gas_pipes["capital_cost"] = (
                 gas_pipes.length * costs.at["CH4 (g) pipeline", "fixed"]
             )
+            gas_pipes["p_nom_extendable"] = False
 
         n.madd(
             "Link",
@@ -1252,14 +1254,14 @@ def add_storage_and_grids(n, costs):
             bus1=gas_pipes.bus1 + " gas",
             p_min_pu=gas_pipes.p_min_pu,
             p_nom=gas_pipes.p_nom,
-            p_nom_extendable=True,
+            p_nom_extendable=gas_pipes.p_nom_extendable,
             p_nom_max=gas_pipes.p_nom_max,
             p_nom_min=gas_pipes.p_nom_min,
             length=gas_pipes.length,
             capital_cost=gas_pipes.capital_cost,
             tags=gas_pipes.name,
             carrier="gas pipeline",
-            lifetime=costs.at["CH4 (g) pipeline", "lifetime"],
+            lifetime=np.inf,
         )
 
         # remove fossil generators where there is neither
@@ -1541,14 +1543,14 @@ def add_EVs(
     temperature,
 ):
 
-    n.add("Carrier", "Li ion")
+    n.add("Carrier", "EV battery")
 
     n.madd(
         "Bus",
         spatial.nodes,
         suffix=" EV battery",
         location=spatial.nodes,
-        carrier="Li ion",
+        carrier="EV battery",
         unit="MWh_el",
     )
 
@@ -1621,9 +1623,9 @@ def add_EVs(
         n.madd(
             "Store",
             spatial.nodes,
-            suffix=" battery storage",
+            suffix=" EV battery",
             bus=spatial.nodes + " EV battery",
-            carrier="battery storage",
+            carrier="EV battery",
             e_cyclic=True,
             e_nom=e_nom,
             e_max_pu=1,
@@ -2773,10 +2775,11 @@ def add_industry(n, costs):
         )
 
     domestic_navigation = pop_weighted_energy_totals.loc[
-        nodes, "total domestic navigation"
+        nodes, ["total domestic navigation"]
     ].squeeze()
     international_navigation = (
-        pd.read_csv(snakemake.input.shipping_demand, index_col=0).squeeze() * nyears
+        pd.read_csv(snakemake.input.shipping_demand, index_col=0).squeeze(axis=1)
+        * nyears
     )
     all_navigation = domestic_navigation + international_navigation
     p_set = all_navigation * 1e6 / nhours
@@ -3944,12 +3947,11 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake(
             "prepare_sector_network",
-            # configfiles="test/config.overnight.yaml",
             simpl="",
             opts="",
-            clusters="37",
-            ll="v1.0",
-            sector_opts="730H-T-H-B-I-A-dist1",
+            clusters="1",
+            ll="vopt",
+            sector_opts="",
             planning_horizons="2050",
         )
 

scripts/solve_network.py

@@ -471,6 +471,22 @@ def prepare_network(
             p_nom=1e9,  # kW
         )
 
+    if solve_opts.get("curtailment_mode"):
+        n.add("Carrier", "curtailment", color="#fedfed", nice_name="Curtailment")
+        n.generators_t.p_min_pu = n.generators_t.p_max_pu
+        buses_i = n.buses.query("carrier == 'AC'").index
+        n.madd(
+            "Generator",
+            buses_i,
+            suffix=" curtailment",
+            bus=buses_i,
+            p_min_pu=-1,
+            p_max_pu=0,
+            marginal_cost=-0.1,
+            carrier="curtailment",
+            p_nom=1e6,
+        )
+
     if solve_opts.get("noisy_costs"):
         for t in n.iterate_components():
             # if 'capital_cost' in t.df: