Merge remote-tracking branch 'upstream/master' into year-specific-techs

2024-01-11 17:25:50 +01:00 · 2024-01-11 17:25:50 +01:00 · e9873e7471
commit e9873e7471
parent a4802b7018 a6edce58fb
43 changed files with 1317 additions and 788 deletions
--- a/CITATION.cff
+++ b/CITATION.cff
@ -6,7 +6,7 @@ cff-version: 1.1.0
 message: "If you use this package, please cite it in the following way."
 title: "PyPSA-Eur: An open sector-coupled optimisation model of the European energy system"
 repository: https://github.com/pypsa/pypsa-eur
-version: 0.8.1
+version: 0.9.0
 license: MIT
 authors:
  - family-names: Brown
--- a/1
+++ b/1
@ -125,6 +125,7 @@ rule sync:
    shell:
        """
        rsync -uvarh --ignore-missing-args --files-from=.sync-send . {params.cluster}
+        rsync -uvarh --no-g {params.cluster}/resources . || echo "No resources directory, skipping rsync"
        rsync -uvarh --no-g {params.cluster}/results . || echo "No results directory, skipping rsync"
        rsync -uvarh --no-g {params.cluster}/logs . || echo "No logs directory, skipping rsync"
        """
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@ -3,7 +3,7 @@
 # SPDX-License-Identifier: CC0-1.0

 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#top-level-configuration
-version: 0.8.1
+version: 0.9.0
 tutorial: false

 logging:
@ -59,6 +59,9 @@ snapshots:
  start: "2013-01-01"
  end: "2014-01-01"
  inclusive: 'left'
+  resolution: false
+  segmentation: false
+  #representative: false

 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#enable
 enable:
@ -74,6 +77,7 @@ enable:
  retrieve_natura_raster: true
  custom_busmap: false

+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#co2-budget
 co2_budget:
  2020: 0.701
@ -87,7 +91,9 @@ co2_budget:
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity
 electricity:
  voltages: [220., 300., 380., 500., 750.]
+  gaslimit_enable: false
  gaslimit: false
+  co2limit_enable: false
  co2limit: 7.75e+7
  co2base: 1.487e+9
  agg_p_nom_limits: data/agg_p_nom_minmax.csv
@ -110,6 +116,7 @@ electricity:

  powerplants_filter: (DateOut >= 2022 or DateOut != DateOut)
  custom_powerplants: false
+  everywhere_powerplants: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]

  conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]
  renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro]
@ -124,6 +131,10 @@ electricity:
      Onshore: [onwind]
      PV: [solar]

+  autarky:
+    enable: false
+    by_country: false
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite
 atlite:
  default_cutout: europe-2013-era5
@ -221,7 +232,7 @@ renewable:
      orientation:
        slope: 35.
        azimuth: 180.
-    capacity_per_sqkm: 1.7
+    capacity_per_sqkm: 5.1
    # correction_factor: 0.854337
    corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32]
    luisa: false # [1111, 1121, 1122, 1123, 1130, 1210, 1221, 1222, 1230, 1241, 1242, 1310, 1320, 1330, 1410, 1421, 1422, 2110, 2120, 2130, 2210, 2220, 2230, 2310, 2410, 2420, 3210, 3320, 3330]
@ -254,7 +265,7 @@ lines:
    750.: "Al/St 560/50 4-bundle 750.0"
  s_max_pu: 0.7
  s_nom_max: .inf
-  max_extension: .inf
+  max_extension: 20000 #MW
  length_factor: 1.25
  reconnect_crimea: true
  under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
@ -269,7 +280,7 @@ lines:
 links:
  p_max_pu: 1.0
  p_nom_max: .inf
-  max_extension: .inf
+  max_extension: 30000 #MW
  include_tyndp: true
  under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity

@ -279,7 +290,7 @@ transformers:
  s_nom: 2000.
  type: ''

-# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#load
+# docs-load in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#load
 load:
  power_statistics: true
  interpolate_limit: 3
@ -304,6 +315,7 @@ pypsa_eur:
  - offwind-dc
  - solar
  - ror
+  - nuclear
  StorageUnit:
  - PHS
  - hydro
@ -366,8 +378,11 @@ sector:
    potential: 0.6
    progress:
      2020: 0.0
+      2025: 0.15
      2030: 0.3
+      2035: 0.45
      2040: 0.6
+      2045: 0.8
      2050: 1.0
    district_heating_loss: 0.15
  cluster_heat_buses: false
@ -390,18 +405,27 @@ sector:
  v2g: true
  land_transport_fuel_cell_share:
    2020: 0
-    2030: 0.05
-    2040: 0.1
-    2050: 0.15
+    2025: 0
+    2030: 0
+    2035: 0
+    2040: 0
+    2045: 0
+    2050: 0
  land_transport_electric_share:
    2020: 0
-    2030: 0.25
-    2040: 0.6
-    2050: 0.85
+    2025: 0.15
+    2030: 0.3
+    2035: 0.45
+    2040: 0.7
+    2045: 0.85
+    2050: 1
  land_transport_ice_share:
    2020: 1
+    2025: 0.85
    2030: 0.7
+    2035: 0.55
    2040: 0.3
+    2045: 0.15
    2050: 0
  transport_fuel_cell_efficiency: 0.5
  transport_internal_combustion_efficiency: 0.3
@ -415,18 +439,27 @@ sector:
  shipping_hydrogen_liquefaction: false
  shipping_hydrogen_share:
    2020: 0
+    2025: 0
    2030: 0
+    2035: 0
    2040: 0
+    2045: 0
    2050: 0
  shipping_methanol_share:
    2020: 0
+    2025: 0.15
    2030: 0.3
+    2035: 0.5
    2040: 0.7
+    2045: 0.85
    2050: 1
  shipping_oil_share:
    2020: 1
+    2025: 0.85
    2030: 0.7
+    2035: 0.5
    2040: 0.3
+    2045: 0.15
    2050: 0
  shipping_methanol_efficiency: 0.46
  shipping_oil_efficiency: 0.40
@ -455,6 +488,7 @@ sector:
    decentral: 3
    central: 180
  boilers: true
+  resistive_heaters: true
  oil_boilers: false
  biomass_boiler: true
  chp: true
@ -472,6 +506,8 @@ sector:
  hydrogen_turbine: false
  SMR: true
  SMR_cc: true
+  regional_methanol_demand: false
+  regional_oil_demand: false
  regional_co2_sequestration_potential:
    enable: false
    attribute: 'conservative estimate Mt'
@ -484,6 +520,7 @@ sector:
  co2_sequestration_lifetime: 50
  co2_spatial: false
  co2network: false
+  co2_network_cost_factor: 1
  cc_fraction: 0.9
  hydrogen_underground_storage: true
  hydrogen_underground_storage_locations:
@ -623,7 +660,9 @@ costs:
    battery: 0.
    battery inverter: 0.
  emission_prices:
+    enable: false
    co2: 0.
+    co2_monthly_prices: false

 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering
 clustering:
@ -661,10 +700,16 @@ solving:
    track_iterations: false
    min_iterations: 4
    max_iterations: 6
-    transmission_losses: 0
+    transmission_losses: 2
    linearized_unit_commitment: true
    horizon: 365

+  constraints:
+    CCL: false
+    EQ: false
+    BAU: false
+    SAFE: false
+
  solver:
    name: gurobi
    options: gurobi-default
@ -778,7 +823,6 @@ plotting:
    hydroelectricity: '#298c81'
    PHS: '#51dbcc'
    hydro+PHS: "#08ad97"
-    wave: '#a7d4cf'
    # solar
    solar: "#f9d002"
    solar PV: "#f9d002"
@ -991,6 +1035,7 @@ plotting:
    CO2 sequestration: '#f29dae'
    DAC: '#ff5270'
    co2 stored: '#f2385a'
+    co2 sequestered: '#f2682f'
    co2: '#f29dae'
    co2 vent: '#ffd4dc'
    CO2 pipeline: '#f5627f'
--- a/data/custom_extra_functionality.py
+++ b/data/custom_extra_functionality.py
@ -4,7 +4,7 @@
 # SPDX-License-Identifier: MIT


-def custom_extra_functionality(n, snapshots):
+def custom_extra_functionality(n, snapshots, snakemake):
    """
    Add custom extra functionality constraints.
    """
--- a/doc/conf.py
+++ b/doc/conf.py
@ -80,9 +80,9 @@ author = "Tom Brown (KIT, TUB, FIAS), Jonas Hoersch (KIT, FIAS), Fabian Hofmann
 # built documents.
 #
 # The short X.Y version.
-version = "0.8"
+version = "0.9"
 # The full version, including alpha/beta/rc tags.
-release = "0.8.1"
+release = "0.9.0"

 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
--- a/doc/configtables/costs.csv
+++ b/doc/configtables/costs.csv
@ -1,9 +1,12 @@
 ,Unit,Values,Description
-year,--,"YYYY; e.g. '2030'","Year for which to retrieve cost assumptions of ``resources/costs.csv``."
-version,--,"vX.X.X; e.g. 'v0.5.0'","Version of ``technology-data`` repository to use."
-rooftop_share,--,float,"Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV)."
-fill_values,--,float,"Default values if not specified for a technology in ``resources/costs.csv``."
-capital_cost,EUR/MW,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
-marginal_cost,EUR/MWh,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
-emission_prices,,,"Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs."
-- co2,EUR/t,float,"Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``."
+year,--,YYYY; e.g. '2030',Year for which to retrieve cost assumptions of ``resources/costs.csv``.
+version,--,vX.X.X; e.g. 'v0.5.0',Version of ``technology-data`` repository to use.
+rooftop_share,--,float,Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV).
+social_discountrate,p.u.,float,Social discount rate to compare costs in different investment periods. 0.02 corresponds to a social discount rate of 2%.
+fill_values,--,float,Default values if not specified for a technology in ``resources/costs.csv``.
+capital_cost,EUR/MW,Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.,"For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
+marginal_cost,EUR/MWh,Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.,"For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
+emission_prices,,,Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs.
+-- enable,bool,true or false,Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well)
+-- co2,EUR/t,float,Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``.
+-- co2_monthly_price,bool,true or false,Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``
--- a/doc/configtables/electricity.csv
+++ b/doc/configtables/electricity.csv
@ -1,6 +1,8 @@
 ,Unit,Values,Description
 voltages,kV,"Any subset of {220., 300., 380.}",Voltage levels to consider
+gaslimit_enable,bool,true or false,Add an overall absolute gas limit configured in ``electricity: gaslimit``.
 gaslimit,MWhth,float or false,Global gas usage limit
+co2limit_enable,bool,true or false,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``.
 co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide emissions
 co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard.
 agg_p_nom_limits,file,path,Reference to ``.csv`` file specifying per carrier generator nominal capacity constraints for individual countries if ``'CCL'`` is in ``{opts}`` wildcard. Defaults to ``data/agg_p_nom_minmax.csv``.
@ -22,6 +24,8 @@ powerplants_filter,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/
 ,,,
 custom_powerplants,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ strings here, e.g. ``Country in ['Germany']``",Filter query for the custom powerplant database.
 ,,,
+everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to add to every node in the model with zero initial capacity. To be used in combination with ``extendable_carriers`` to allow for building conventional powerplants irrespective of existing locations."
+,,,
 conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound."
 ,,,
 renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model.
@ -34,3 +38,6 @@ estimate_renewable_capacities,,,
 -- -- Offshore,--,"Any subset of {offwind-ac, offwind-dc}","List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) onshore technology."
 -- -- Offshore,--,{onwind},"List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) offshore technology."
 -- -- PV,--,{solar},"List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) PV technology."
+autarky,,,
+-- enable,bool,true or false,Require each node to be autarkic by removing all lines and links.
+-- by_country,bool,true or false,Require each country to be autarkic by removing all cross-border lines and links. ``electricity: autarky`` must be enabled.
--- a/doc/configtables/licenses-sector.csv
+++ b/doc/configtables/licenses-sector.csv
@ -9,7 +9,6 @@ Swiss energy statistics from Swiss Federal Office of Energy,switzerland-sfoe/,un
 BASt emobility statistics,emobility/,unknown,http://www.bast.de/DE/Verkehrstechnik/Fachthemen/v2-verkehrszaehlung/Stundenwerte.html?nn=626916
 BDEW heating profile,heat_load_profile_BDEW.csv,unknown,https://github.com/oemof/demandlib
 heating profiles for Aarhus,heat_load_profile_DK_AdamJensen.csv,unknown,Adam Jensen MA thesis at Aarhus University
-George Lavidas wind/wave costs,WindWaveWEC_GLTB.xlsx,unknown,George Lavidas
 co2 budgets,co2_budget.csv,CC BY 4.0,https://arxiv.org/abs/2004.11009
 existing heating potentials,existing_infrastructure/existing_heating_raw.csv,unknown,https://ec.europa.eu/energy/studies/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment_en?redir=1
 IRENA existing VRE capacities,existing_infrastructure/{solar|onwind|offwind}_capcity_IRENA.csv,unknown,https://www.irena.org/Statistics/Download-Data
--- a/doc/configtables/opts.csv
+++ b/doc/configtables/opts.csv
@ -1,13 +1,13 @@
 Trigger, Description, Definition, Status
 ``nH``; i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L110>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L146>`__), In active use
-``nSEG``; e.g. ``4380SEG``, "Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use
-``Co2L``, Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L19>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L154>`__, In active use
-``Ep``, Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L24>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L158>`__, In active use
-``Ept``, Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``, In active use
-``CCL``, Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use
-``EQ``, "Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.", ``solve_network``, In active use
-``ATK``, "Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links.", ``prepare_network``, In active use
-``BAU``, Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L66>`__, Untested
-``SAFE``, Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L73>`__, Untested
+``nSEG``; e.g. ``4380SEG``,"Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use
+``Co2L``,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L19>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L154>`__, In active use
+``Ep``,Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L24>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L158>`__, In active use
+``Ept``,Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``, In active use,
+``CCL``,Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use
+``EQ``,Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption., ``solve_network``, In active use
+``ATK``,Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links., ``prepare_network``, In active use
+``BAU``,Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L66>`__, Untested
+``SAFE``,Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L73>`__, Untested
 ``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use
 ``CH4L``,"Add an overall absolute gas limit. If configured in ``electricity: gaslimit`` it is given in MWh thermal, if a float is appended, the overall gaslimit is assumed to be given in TWh thermal (e.g. ``CH4L200`` limits gas dispatch to 200 TWh termal)", ``prepare_network``: ``add_gaslimit()``, In active use
--- a/doc/configtables/sector.csv
+++ b/doc/configtables/sector.csv
@ -62,7 +62,8 @@ tes,--,"{true, false}",Add option for storing thermal energy in large water pits
 tes_tau,,,The time constant used to calculate the decay of thermal energy in thermal energy storage (TES): 1- :math:`e^{-1/24τ}`.
 -- decentral,days,float,The time constant in decentralized thermal energy storage (TES)
 -- central,days,float,The time constant in centralized thermal energy storage (TES)
-boilers,--,"{true, false}",Add option for transforming electricity into heat using resistive heater
+boilers,--,"{true, false}",Add option for transforming gas into heat using gas boilers
+resistive_heaters,--,"{true, false}",Add option for transforming electricity into heat using resistive heaters (independently from gas boilers)
 oil_boilers,--,"{true, false}",Add option for transforming oil into heat using boilers
 biomass_boiler,--,"{true, false}",Add option for transforming biomass into heat using boilers
 chp,--,"{true, false}",Add option for using Combined Heat and Power (CHP)
@ -79,6 +80,8 @@ hydrogen_fuel_cell,--,"{true, false}",Add option to include hydrogen fuel cell f
 hydrogen_turbine,--,"{true, false}",Add option to include hydrogen turbine for re-electrification. Assuming OCGT technology costs
 SMR,--,"{true, false}",Add option for transforming natural gas into hydrogen and CO2 using Steam Methane Reforming (SMR)
 SMR CC,--,"{true, false}",Add option for transforming natural gas into hydrogen and CO2 using Steam Methane Reforming (SMR) and Carbon Capture (CC)
+regional_methanol_demand,--,"{true, false}",Spatially resolve methanol demand. Set to true if regional CO2 constraints needed.
+regional_oil_demand,--,"{true, false}",Spatially resolve oil demand. Set to true if regional CO2 constraints needed.
 regional_co2 _sequestration_potential,,,
 -- enable,--,"{true, false}",Add option for regionally-resolved geological carbon dioxide sequestration potentials based on `CO2StoP <https://setis.ec.europa.eu/european-co2-storage-database_en>`_.
 -- attribute,--,string,Name of the attribute for the sequestration potential
@ -88,9 +91,11 @@ regional_co2 _sequestration_potential,,,
 -- years_of_storage,years,float,The years until potential exhausted at optimised annual rate
 co2_sequestration_potential,MtCO2/a,float,The potential of sequestering CO2 in Europe per year
 co2_sequestration_cost,currency/tCO2,float,The cost of sequestering a ton of CO2
+co2_sequestration_lifetime,years,int,The lifetime of a CO2 sequestration site
 co2_spatial,--,"{true, false}","Add option to spatially resolve carrier representing stored carbon dioxide. This allows for more detailed modelling of CCUTS, e.g. regarding the capturing of industrial process emissions, usage as feedstock for electrofuels, transport of carbon dioxide, and geological sequestration sites."
 ,,,
 co2network,--,"{true, false}",Add option for planning a new carbon dioxide transmission network
+co2_network_cost_factor,p.u.,float,The cost factor for the capital cost of the carbon dioxide transmission network
 ,,,
 cc_fraction,--,float,The default fraction of CO2 captured with post-combustion capture
 hydrogen_underground _storage,--,"{true, false}",Add options for storing hydrogen underground. Storage potential depends regionally.
@ -126,3 +131,10 @@ biogas_upgrading_cc,--,"{true, false}",Add option to capture CO2 from biomass up
 conventional_generation,,,Add a more detailed description of conventional carriers. Any power generation requires the consumption of fuel from nodes representing that fuel.
 biomass_to_liquid,--,"{true, false}",Add option for transforming solid biomass into liquid fuel with the same properties as oil
 biosng,--,"{true, false}",Add option for transforming solid biomass into synthesis gas with the same properties as natural gas
+limit_max_growth,,,
+-- enable,--,"{true, false}",Add option to limit the maximum growth of a carrier
+-- factor,p.u.,float,The maximum growth factor of a carrier (e.g. 1.3 allows  30% larger than max historic growth)
+-- max_growth,,,
+-- -- {carrier},GW,float,The historic maximum growth of a carrier
+-- max_relative_growth,
+-- -- {carrier},p.u.,float,The historic maximum relative growth of a carrier
--- a/doc/configtables/snapshots.csv
+++ b/doc/configtables/snapshots.csv
@ -1,4 +1,6 @@
 ,Unit,Values,Description
-start,--,"str or datetime-like; e.g. YYYY-MM-DD","Left bound of date range"
-end,--,"str or datetime-like; e.g. YYYY-MM-DD","Right bound of date range"
+start,--,str or datetime-like; e.g. YYYY-MM-DD,Left bound of date range
+end,--,str or datetime-like; e.g. YYYY-MM-DD,Right bound of date range
 inclusive,--,"One of {'neither', 'both', ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``both`` or neither side ``None``."
+resolution ,--,"{false,``nH``; i.e. ``2H``-``6H``}",Resample the time-resolution by averaging over every ``n`` snapshots
+segmentation,--,"{false,``n``; e.g. ``4380``}","Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load."
--- a/doc/configtables/solving.csv
+++ b/doc/configtables/solving.csv
@ -13,6 +13,11 @@ options,,,
 -- transmission_losses,int,[0-9],"Add piecewise linear approximation of transmission losses based on n tangents. Defaults to 0, which means losses are ignored."
 -- linearized_unit_commitment,bool,"{'true','false'}",Whether to optimise using the linearized unit commitment formulation.
 -- horizon,--,int,Number of snapshots to consider in each iteration. Defaults to 100.
+constraints ,,,
+-- CCL,bool,"{'true','false'}",Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``.
+-- EQ,bool/string,"{'false',`n(c| )``; i.e. ``0.5``-``0.7c``}",Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.
+-- BAU,bool,"{'true','false'}",Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``
+-- SAFE,bool,"{'true','false'}",Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network.
 solver,,,
 -- name,--,"One of {'gurobi', 'cplex', 'cbc', 'glpk', 'ipopt'}; potentially more possible",Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow.
 -- options,--,Key listed under ``solver_options``.,Link to specific parameter settings.
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@ -383,7 +383,7 @@ overwrite the existing values.

 .. literalinclude:: ../config/config.default.yaml
   :language: yaml
-   :start-after:   type:
+   :start-after: # docs-load
   :end-before: # docs

 .. csv-table::
--- a/doc/foresight.rst
+++ b/doc/foresight.rst
@ -41,7 +41,7 @@ Perfect foresight scenarios

 .. warning::

-  Perfect foresight is currently implemented as a first test version.
+  Perfect foresight is currently implemented as an experimental test version.

 For running perfect foresight scenarios, you can adjust the
 ``config/config.perfect.yaml``:
--- a/doc/img/intro-workflow.png
+++ b/doc/img/intro-workflow.png
--- a/doc/index.rst
+++ b/doc/index.rst
@ -35,6 +35,8 @@ PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy Syste
   :target: https://stackoverflow.com/questions/tagged/pypsa
   :alt: Stackoverflow

+|
+
 PyPSA-Eur is an open model dataset of the European energy system at the
 transmission network level that covers the full ENTSO-E area. It covers demand
 and supply for all energy sectors. From version v0.8.0, PyPSA-Eur includes all
@ -209,24 +211,6 @@ If you want to cite a specific PyPSA-Eur version, each release of PyPSA-Eur is s
   :target: https://doi.org/10.5281/zenodo.3520874


-Pre-Built Networks as a Dataset
-===============================
-
-There are pre-built networks available as a dataset on Zenodo as well for every release of PyPSA-Eur.
-
-.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3601881.svg
-   :target: https://doi.org/10.5281/zenodo.3601881
-
-The included ``.nc`` files are PyPSA network files which can be imported with PyPSA via:
-
-.. code:: python
-
-    import pypsa
-
-    filename = "elec_s_1024_ec.nc"  # example
-    n = pypsa.Network(filename)
-
-
 Operating Systems
 =================

--- a/doc/introduction.rst
+++ b/doc/introduction.rst
@ -89,8 +89,8 @@ Folder Structure
 - ``results``: Stores the solved PyPSA network data, summary files and plots.
 - ``logs``: Stores log files.
 - ``benchmarks``: Stores ``snakemake`` benchmarks.
- ``test``: Includes the test configuration files used for continuous integration.
 - ``doc``: Includes the documentation of PyPSA-Eur.
+- ``graphics``: Includes some graphics for the documentation of PyPSA-Eur.

 System Requirements
 ===================
--- a/doc/preparation.rst
+++ b/doc/preparation.rst
@ -94,6 +94,13 @@ Rule ``build_electricity_demand``

 .. automodule:: build_electricity_demand

+.. _monthlyprices:
+
+Rule ``build_monthly_prices``
+=============================
+
+.. automodule:: build_monthly_prices
+
 .. _ship:

 Rule ``build_ship_raster``
@ -102,6 +109,12 @@ Rule ``build_ship_raster``

 .. automodule:: build_ship_raster

+.. _availabilitymatrixmdua:
+
+Rule ``determine_availability_matrix_MD_UA``
+============================================
+
+.. automodule:: determine_availability_matrix_MD_UA

 .. _renewableprofiles:

--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@ -10,64 +10,22 @@ Release Notes
 Upcoming Release
 ================

+* New configuration option ``everywhere_powerplants`` to build conventional powerplants everywhere, irrespective of existing powerplants locations, in the network (https://github.com/PyPSA/pypsa-eur/pull/850).
+
+* Remove option for wave energy as technology data is not maintained. 
+
+
+PyPSA-Eur 0.9.0 (5th January 2024)
+==================================
+
+**New Features**
+
 * Add option to specify losses for bidirectional links, e.g. pipelines or HVDC
  links, in configuration file under ``sector: transmission_efficiency:``. Users
  can specify static or length-dependent values as well as a length-dependent
  electricity demand for compression, which is implemented as a multi-link to
  the local electricity buses. The bidirectional links will then be split into
-  two unidirectional links with linked capacities.
-
-* Pin ``snakemake`` version to below 8.0.0, as the new version is not yet
-  supported by ``pypsa-eur``.
-
-* Updated Global Energy Monitor LNG terminal data to March 2023 version.
-
-* For industry distribution, use EPRTR as fallback if ETS data is not available.
-
-* The minimum capacity for renewable generators when using the myopic option has been fixed.
-
-* Files downloaded from zenodo are now write-protected to prevent accidental re-download.
-
-* Files extracted from sector-coupled data bundle have been moved from ``data/`` to ``data/sector-bundle``.
-
-* New feature multi-decade optimisation with  perfect foresight.
-
-* It is now possible to specify years for biomass potentials which do not exist
-  in the JRC-ENSPRESO database, e.g. 2037. These are linearly interpolated.
-
-* In pathway mode, the biomass potential is linked to the investment year.
-
-* Rule ``purge`` now initiates a dialog to confirm if purge is desired.
-
-* Rule ``retrieve_irena`` get updated values for renewables capacities.
-
-* Rule ``retrieve_wdpa`` updated to not only check for current and previous, but also potentially next months dataset availability.
-
-* Split configuration to enable SMR and SMR CC.
-
-* Bugfix: The unit of the capital cost of Haber-Bosch plants was corrected.
-
-* The configuration setting for country focus weights when clustering the
-  network has been moved from ``focus_weights:`` to ``clustering:
-  focus_weights:``. Backwards compatibility to old config files is maintained.
-
-* Extend options for waste usage from Haber-Bosch, methanolisation and methanation.
-
-* Use electrolysis waste heat by default.
-
-* Add new ``sector_opts`` wildcard option "nowasteheat" to disable all waste heat usage.
-
-* Set minimum part loads for PtX processes to 30% for methanolisation and methanation, and to 70% for Fischer-Tropsch synthesis.
-
-* Add VOM as marginal cost to PtX processes.
-
-* Add pelletizing costs for biomass boilers.
-
-* The ``mock_snakemake`` function can now be used with a Snakefile from a different directory using the new ``root_dir`` argument.
-
-* Switch to using hydrogen and electricity inputs for Haber-Bosch from https://github.com/PyPSA/technology-data.
-
-* Add option to capture CO2 contained in biogas when upgrading (``sector: biogas_to_gas_cc``).
+  two unidirectional links with linked capacities (https://github.com/PyPSA/pypsa-eur/pull/739).

 * Merged option to extend geographical scope to Ukraine and Moldova. These
  countries are excluded by default and is currently constrained to power-sector
@ -77,24 +35,21 @@ Upcoming Release
  Moldova). Moldova can currently only be included in conjunction with Ukraine
  due to the absence of demand data. The Crimean power system is manually
  reconnected to the main Ukrainian grid with the configuration option
-  `reconnect_crimea`.
+  `reconnect_crimea` (https://github.com/PyPSA/pypsa-eur/pull/321).
+
+* New experimental support for multi-decade optimisation with perfect foresight
+  (``foresight: perfect``). Maximum growth rates for carriers, global carbon
+  budget constraints and emission constraints for particular investment periods.

 * Add option to reference an additional source file where users can specify
  custom ``extra_functionality`` constraints in the configuration file. The
  default setting points to an empty hull at
-  ``data/custom_extra_functionality.py``.
-
-* Validate downloads from Zenodo using MD5 checksums. This identifies corrupted
-  or incomplete downloads.
+  ``data/custom_extra_functionality.py`` (https://github.com/PyPSA/pypsa-eur/pull/824).

 * Add locations, capacities and costs of existing gas storage using Global
  Energy Monitor's `Europe Gas Tracker
-  <https://globalenergymonitor.org/projects/europe-gas-tracker>`_.
-
-* Remove HELMETH option.
-
-* Print Irreducible Infeasible Subset (IIS) if model is infeasible. Only for
-  solvers with IIS support.
+  <https://globalenergymonitor.org/projects/europe-gas-tracker>`_
+  (https://github.com/PyPSA/pypsa-eur/pull/835).

 * Add option to use `LUISA Base Map
  <https://publications.jrc.ec.europa.eu/repository/handle/JRC124621>`_ 50m land
@ -104,13 +59,240 @@ Upcoming Release
  dataset's full advantages, set the excluder resolution to 50m
  (``excluder_resolution: 50``). For land category codes, see `Annex 1 of the
  technical documentation
-  <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_.
+  <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_
+  (https://github.com/PyPSA/pypsa-eur/pull/842).
+
+* Add option to capture CO2 contained in biogas when upgrading (``sector:
+  biogas_to_gas_cc``) (https://github.com/PyPSA/pypsa-eur/pull/615).
+
+* If load shedding is activated, it is now applied to all carriers, not only
+  electricity (https://github.com/PyPSA/pypsa-eur/pull/784).
+
+* Add option for heat vents in district heating (``sector:
+  central_heat_vent:``). The combination of must-run conditions for some
+  power-to-X processes, waste heat usage enabled and decreasing heating demand,
+  can lead to infeasibilities in pathway optimisation for some investment
+  periods since larger Fischer-Tropsch capacities are needed in early years but
+  the waste heat exceeds the heat demand in later investment periods.
+  (https://github.com/PyPSA/pypsa-eur/pull/791).
+
+* Allow possibility to go from copperplated to regionally resolved methanol and
+  oil demand with switches ``sector: regional_methanol_demand: true`` and
+  ``sector: regional_oil_demand: true``. This allows nodal/regional CO2
+  constraints to be applied (https://github.com/PyPSA/pypsa-eur/pull/827).
+
+* Allow retrofitting of existing gas boilers to hydrogen boilers in pathway
+  optimisation.
+
+* Add option to add time-varying CO2 emission prices (electricity-only, ``costs:
+  emission_prices: co2_monthly_prices: true``). This is linked to the new
+  ``{opts}`` wildcard option ``Ept``.
+
+* Network clustering can now consider efficiency classes when aggregating
+  carriers. The option ``clustering: consider_efficiency_classes:`` aggregates
+  each carriers into the top 10-quantile (high), the bottom 90-quantile (low),
+  and everything in between (medium).
+
+* Added option ``conventional: dynamic_fuel_price:`` to consider the monthly
+  fluctuating fuel prices for conventional generators. Refer to the CSV file
+  ``data/validation/monthly_fuel_price.csv``.
+
+* For hydro-electricity, add switches ``flatten_dispatch`` to consider an upper
+  limit for the hydro dispatch. The limit is given by the average capacity
+  factor plus the buffer given in  ``flatten_dispatch_buffer``.
+
+* Extend options for waste heat usage from Haber-Bosch, methanolisation and
+  methanation (https://github.com/PyPSA/pypsa-eur/pull/834).
+
+* Add new ``sector_opts`` wildcard option "nowasteheat" to disable all waste
+  heat usage (https://github.com/PyPSA/pypsa-eur/pull/834).
+
+* Add new rule ``retrieve_irena`` to automatically retrieve up-to-date values
+  for existing renewables capacities (https://github.com/PyPSA/pypsa-eur/pull/756).
+
+* Print Irreducible Infeasible Subset (IIS) if model is infeasible. Only for
+  solvers with IIS support (https://github.com/PyPSA/pypsa-eur/pull/841).
+
+* More wildcard options now have a corresponding config entry. If the wildcard
+  is given, then its value is used. If the wildcard is not given but the options
+  in config are enabled, then the value from config is used. If neither is
+  given, the options are skipped (https://github.com/PyPSA/pypsa-eur/pull/827).
+
+* Validate downloads from Zenodo using MD5 checksums. This identifies corrupted
+  or incomplete downloads (https://github.com/PyPSA/pypsa-eur/pull/821).
+
+* Add rule ``sync`` to synchronise with a remote machine using the ``rsync``
+  library. Configuration settings are found under ``remote:``.
+
+**Breaking Changes**
+
+* Remove all negative loads on the ``co2 atmosphere`` bus representing emissions
+  for e.g. fixed fossil demands for transport oil. Instead these are handled
+  more transparently with a fixed transport oil demand and a link taking care of
+  the emissions to the ``co2 atmosphere`` bus. This is also a preparation for
+  endogenous transport optimisation, where demand will be subject to
+  optimisation (e.g. fuel switching in the transport sector)
+  (https://github.com/PyPSA/pypsa-eur/pull/827).
+
+* Process emissions from steam crackers (i.e. naphtha processing for HVC) are
+  now piped from the consumption link to the process emissions bus where the
+  model can decide about carbon capture. Previously the process emissions for
+  naphtha were a fixed load (https://github.com/PyPSA/pypsa-eur/pull/827).
+
+* Distinguish between stored and sequestered CO2. Stored CO2 is stored
+  overground in tanks and can be used for CCU (e.g. methanolisation).
+  Sequestered CO2 is stored underground and can no longer be used for CCU. This
+  distinction is made because storage in tanks is more expensive than
+  underground storage. The link that connects stored and sequestered CO2 is
+  unidirectional (https://github.com/PyPSA/pypsa-eur/pull/844).
+
+* Files extracted from sector-coupled data bundle have been moved from ``data/``
+  to ``data/sector-bundle``.
+
+* Split configuration to enable SMR and SMR CC (``sector: smr:`` and ``sector:
+  smr_cc:``) (https://github.com/PyPSA/pypsa-eur/pull/757).
+
+* Add separate option to add resistive heaters to the technology choices
+  (``sector: resistive_heaters:``). Previously they were always added when
+  boilers were added (https://github.com/PyPSA/pypsa-eur/pull/808).
+
+* Remove HELMETH option (``sector: helmeth:``).
+
+* Remove "conservative" renewable potentials estimation option
+  (https://github.com/PyPSA/pypsa-eur/pull/838).
+
+* With this release we stop posting updates to the network pre-builts.
+
+**Changes**
+
+* Updated Global Energy Monitor LNG terminal data to March 2023 version
+  (https://github.com/PyPSA/pypsa-eur/pull/707).
+
+* For industry distribution, use EPRTR as fallback if ETS data is not available
+  (https://github.com/PyPSA/pypsa-eur/pull/721).
+
+* It is now possible to specify years for biomass potentials which do not exist
+  in the JRC-ENSPRESO database, e.g. 2037. These are linearly interpolated
+  (https://github.com/PyPSA/pypsa-eur/pull/744).
+
+* In pathway mode, the biomass potential is linked to the investment year
+  (https://github.com/PyPSA/pypsa-eur/pull/744).
+
+* Increase allowed deployment density of solar to 5.1 MW/sqkm by default.
+
+* Default to full electrification of land transport by 2050.
+
+* Provide exogenous transition settings in 5-year steps.
+
+* Default to approximating transmission losses in HVAC lines
+  (``transmission_losses: 2``).
+
+* Use electrolysis waste heat by default.
+
+* Set minimum part loads for PtX processes to 30% for methanolisation and
+  methanation, and to 70% for Fischer-Tropsch synthesis.
+
+* Add VOM as marginal cost to PtX processes
+  (https://github.com/PyPSA/pypsa-eur/pull/830).
+
+* Add pelletizing costs for biomass boilers (https://github.com/PyPSA/pypsa-eur/pull/833).
+
+* Update default offshore wind turbine model to "NREL Reference 2020 ATB 5.5 MW"
+  (https://github.com/PyPSA/pypsa-eur/pull/832).
+
+* Switch to using hydrogen and electricity inputs for Haber-Bosch from
+  https://github.com/PyPSA/technology-data (https://github.com/PyPSA/pypsa-eur/pull/831).
+
+* The configuration setting for country focus weights when clustering the
+  network has been moved from ``focus_weights:`` to ``clustering:
+  focus_weights:``. Backwards compatibility to old config files is maintained
+  (https://github.com/PyPSA/pypsa-eur/pull/794).
+
+* The ``mock_snakemake`` function can now be used with a Snakefile from a
+  different directory using the new ``root_dir`` argument
+  (https://github.com/PyPSA/pypsa-eur/pull/771).
+
+* Rule ``purge`` now initiates a dialog to confirm if purge is desired
+  (https://github.com/PyPSA/pypsa-eur/pull/745).
+
+* Files downloaded from zenodo are now write-protected to prevent accidental
+  re-download (https://github.com/PyPSA/pypsa-eur/pull/730).
+
+* Performance improvements for rule ``build_ship_raster``
+  (https://github.com/PyPSA/pypsa-eur/pull/845).
+
+* Improve time logging in :mod:`build_renewable_profiles`
+  (https://github.com/PyPSA/pypsa-eur/pull/837).
+
+* In myopic pathway optimisation, disable power grid expansion if line volume
+  already hit (https://github.com/PyPSA/pypsa-eur/pull/840).
+
+* JRC-ENSPRESO data is now downloaded from a Zenodo mirror because the link was
+  unreliable (https://github.com/PyPSA/pypsa-eur/pull/801).
+
+* Add focus weights option for clustering to documentation
+  (https://github.com/PyPSA/pypsa-eur/pull/781).
+
+* Add proxy for biomass transport costs if no explicit biomass transport network
+  is considered (https://github.com/PyPSA/pypsa-eur/pull/711).

 **Bugs and Compatibility**

-* A bug preventing custom powerplants specified in ``data/custom_powerplants.csv`` was fixed. (https://github.com/PyPSA/pypsa-eur/pull/732)
-* Fix nodal fraction in ``add_existing_year`` when using distributed generators
-* Fix typo in buses definition for oil boilers in ``add_industry`` in ``prepare_sector_network``
+* The minimum PyPSA version is now 0.26.1.
+
+* Update to ``tsam>=0.2.3`` for performance improvents in temporal clustering.
+
+* Pin ``snakemake`` version to below 8.0.0, as the new version is not yet
+  supported. The next release will switch to the requirement ``snakemake>=8``.
+
+* Bugfix: Add coke and coal demand for integrated steelworks
+  (https://github.com/PyPSA/pypsa-eur/pull/718).
+
+* Bugfix: Make :mod:`build_renewable_profiles` consider subsets of cutout time
+  scope (https://github.com/PyPSA/pypsa-eur/pull/709).
+
+* Bugfix: In :mod:`simplify network`, remove 'underground' column to avoid
+  consense error (https://github.com/PyPSA/pypsa-eur/pull/714).
+
+* Bugfix: Fix in :mod:`add_existing_baseyear` to account for the case when there
+  is no rural heating demand for some nodes in network
+  (https://github.com/PyPSA/pypsa-eur/pull/706).
+
+* Bugfix: The unit of the capital cost of Haber-Bosch plants was corrected
+  (https://github.com/PyPSA/pypsa-eur/pull/829).
+
+* The minimum capacity for renewable generators when using the myopic option has
+  been fixed (https://github.com/PyPSA/pypsa-eur/pull/728).
+
+* Compatibility for running with single node and single country
+  (https://github.com/PyPSA/pypsa-eur/pull/839).
+
+* A bug preventing the addition of custom powerplants specified in
+  ``data/custom_powerplants.csv`` was fixed.
+  (https://github.com/PyPSA/pypsa-eur/pull/732)
+
+* Fix nodal fraction in :mod:`add_existing_year` when using distributed
+  generators (https://github.com/PyPSA/pypsa-eur/pull/798).
+
+* Bugfix: District heating without progress caused division by zero
+  (https://github.com/PyPSA/pypsa-eur/pull/796).
+
+* Bugfix: Drop duplicates in :mod:`build_industrial_distribution_keys`, which
+  can occur through the geopandas ``.sjoin()`` function if a point is located on
+  a border (https://github.com/PyPSA/pypsa-eur/pull/726).
+
+* For network clustering fall back to ``ipopt`` when ``highs`` is designated
+  solver (https://github.com/PyPSA/pypsa-eur/pull/795).
+
+* Fix typo in buses definition for oil boilers in ``add_industry`` in
+  :mod:`prepare_sector_network` (https://github.com/PyPSA/pypsa-eur/pull/812).
+
+* Resolve code issues for endogenous building retrofitting. Select correct
+  sector names, address deprecations, distinguish between district heating,
+  decentral heating in urban areas or rural areas for floor area calculations
+  (https://github.com/PyPSA/pypsa-eur/pull/808).
+
+* Addressed various deprecations.


 PyPSA-Eur 0.8.1 (27th July 2023)
@ -1499,8 +1681,4 @@ Release Process

 * Make a `GitHub release <https://github.com/PyPSA/pypsa-eur-sec/releases>`_, which automatically triggers archiving to the `zenodo code repository <https://doi.org/10.5281/zenodo.3520874>`_ with `MIT license <https://opensource.org/licenses/MIT>`_.

-* Create pre-built networks for ``config.default.yaml`` by running ``snakemake -call prepare_sector_networks``.
-
-* Upload pre-built networks to `zenodo data repository <https://doi.org/10.5281/zenodo.3601881>`_ with `CC BY 4.0 <https://creativecommons.org/licenses/by/4.0/>`_ license.
-
 * Send announcement on the `PyPSA mailing list <https://groups.google.com/forum/#!forum/pypsa>`_.
--- a/doc/spatial_resolution.rst
+++ b/doc/spatial_resolution.rst
@ -45,7 +45,7 @@ Here are some examples of how spatial resolution is set for different sectors in

 •	CO2: It can be modeled as a single node for Europe or it can be nodally resolved with CO2 transport pipelines if activated in the `config <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L248>`_. It should mentioned that in single node mode a transport and storage cost is added for sequestered CO2, the cost of which can be adjusted in the `config <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L247>`_.

-•	Liquid hydrocarbons: Modeled as a single node for Europe, since transport costs for liquids are low and no bottlenecks are expected.
+•	Carbonaceous fuels: Modeled as a single node for Europe by default, since transport costs for liquids are low and no bottlenecks are expected. Can be regionally resolved in configuration.

 **Electricity distribution network**

--- a/envs/environment.fixed.yaml
+++ b/envs/environment.fixed.yaml
@ -1,10 +1,11 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0

 name: pypsa-eur
 channels:
 - bioconda
+- gurobi
 - http://conda.anaconda.org/gurobi
 - conda-forge
 - defaults
@ -12,94 +13,96 @@ dependencies:
 - _libgcc_mutex=0.1
 - _openmp_mutex=4.5
 - affine=2.4.0
- alsa-lib=1.2.9
+- alsa-lib=1.2.10
 - ampl-mp=3.1.0
 - amply=0.1.6
- anyio=3.7.1
+- anyio=4.2.0
 - appdirs=1.4.4
- argon2-cffi=21.3.0
+- argon2-cffi=23.1.0
 - argon2-cffi-bindings=21.2.0
- asttokens=2.2.1
- async-lru=2.0.3
+- arrow=1.3.0
+- asttokens=2.4.1
+- async-lru=2.0.4
 - atk-1.0=2.38.0
- atlite=0.2.11
+- atlite=0.2.12
 - attr=2.5.1
- attrs=23.1.0
- aws-c-auth=0.7.0
- aws-c-cal=0.6.0
- aws-c-common=0.8.23
+- attrs=23.2.0
+- aws-c-auth=0.7.8
+- aws-c-cal=0.6.9
+- aws-c-common=0.9.10
 - aws-c-compression=0.2.17
- aws-c-event-stream=0.3.1
- aws-c-http=0.7.11
- aws-c-io=0.13.28
- aws-c-mqtt=0.8.14
- aws-c-s3=0.3.13
- aws-c-sdkutils=0.1.11
- aws-checksums=0.1.16
- aws-crt-cpp=0.20.3
- aws-sdk-cpp=1.10.57
- babel=2.12.1
- backcall=0.2.0
- backports=1.0
- backports.functools_lru_cache=1.6.5
+- aws-c-event-stream=0.3.2
+- aws-c-http=0.7.15
+- aws-c-io=0.13.36
+- aws-c-mqtt=0.10.0
+- aws-c-s3=0.4.7
+- aws-c-sdkutils=0.1.13
+- aws-checksums=0.1.17
+- aws-crt-cpp=0.25.1
+- aws-sdk-cpp=1.11.210
+- babel=2.14.0
 - beautifulsoup4=4.12.2
- bleach=6.0.0
- blosc=1.21.4
- bokeh=3.2.1
- boost-cpp=1.78.0
+- bleach=6.1.0
+- blosc=1.21.5
+- bokeh=3.3.2
 - bottleneck=1.3.7
- branca=0.6.0
- brotli=1.0.9
- brotli-bin=1.0.9
- brotli-python=1.0.9
+- branca=0.7.0
+- brotli=1.1.0
+- brotli-bin=1.1.0
+- brotli-python=1.1.0
 - bzip2=1.0.8
- c-ares=1.19.1
- c-blosc2=2.10.0
- ca-certificates=2023.7.22
- cairo=1.16.0
- cartopy=0.21.1
+- c-ares=1.24.0
+- c-blosc2=2.12.0
+- ca-certificates=2023.11.17
+- cached-property=1.5.2
+- cached_property=1.5.2
+- cairo=1.18.0
+- cartopy=0.22.0
 - cdsapi=0.6.1
- certifi=2023.7.22
- cffi=1.15.1
- cfitsio=4.2.0
- cftime=1.6.2
- charset-normalizer=3.2.0
- click=8.1.6
+- certifi=2023.11.17
+- cffi=1.16.0
+- cfitsio=4.3.1
+- cftime=1.6.3
+- charset-normalizer=3.3.2
+- click=8.1.7
 - click-plugins=1.1.1
 - cligj=0.7.2
- cloudpickle=2.2.1
+- cloudpickle=3.0.0
+- coin-or-cbc=2.10.10
+- coin-or-cgl=0.60.7
+- coin-or-clp=1.17.8
+- coin-or-osi=0.108.8
+- coin-or-utils=2.11.9
+- coincbc=2.10.10
 - colorama=0.4.6
- comm=0.1.3
+- comm=0.1.4
 - configargparse=1.7
 - connection_pool=0.0.3
- contourpy=1.1.0
- country_converter=1.0.0
- curl=8.2.0
- cycler=0.11.0
+- contourpy=1.2.0
+- country_converter=1.2
+- cycler=0.12.1
 - cytoolz=0.12.2
- dask=2023.7.1
- dask-core=2023.7.1
+- dask=2023.12.1
+- dask-core=2023.12.1
 - datrie=0.8.2
 - dbus=1.13.6
- debugpy=1.6.7
+- debugpy=1.8.0
 - decorator=5.1.1
 - defusedxml=0.7.1
 - deprecation=2.1.0
 - descartes=1.1.0
- distributed=2023.7.1
+- distributed=2023.12.1
 - distro=1.8.0
 - docutils=0.20.1
 - dpath=2.1.6
 - entrypoints=0.4
- entsoe-py=0.5.10
+- entsoe-py=0.6.1
 - et_xmlfile=1.1.0
- exceptiongroup=1.1.2
- executing=1.2.0
+- exceptiongroup=1.2.0
+- executing=2.0.1
 - expat=2.5.0
- filelock=3.12.2
- fiona=1.9.4
- flit-core=3.9.0
- folium=0.14.0
+- fiona=1.9.5
+- folium=0.15.1
 - font-ttf-dejavu-sans-mono=2.37
 - font-ttf-inconsolata=3.000
 - font-ttf-source-code-pro=2.038
@ -107,165 +110,184 @@ dependencies:
 - fontconfig=2.14.2
 - fonts-conda-ecosystem=1
 - fonts-conda-forge=1
- fonttools=4.41.1
+- fonttools=4.47.0
+- fqdn=1.5.1
 - freetype=2.12.1
- freexl=1.0.6
+- freexl=2.0.0
 - fribidi=1.0.10
- fsspec=2023.6.0
- gdal=3.7.0
+- fsspec=2023.12.2
+- gdal=3.7.3
 - gdk-pixbuf=2.42.10
 - geographiclib=1.52
- geojson-rewind=1.0.2
- geopandas=0.13.2
- geopandas-base=0.13.2
- geopy=2.3.0
- geos=3.11.2
+- geojson-rewind=1.1.0
+- geopandas=0.14.1
+- geopandas-base=0.14.1
+- geopy=2.4.1
+- geos=3.12.1
 - geotiff=1.7.1
 - gettext=0.21.1
 - gflags=2.2.2
 - giflib=5.2.1
- gitdb=4.0.10
- gitpython=3.1.32
- glib=2.76.4
- glib-tools=2.76.4
+- gitdb=4.0.11
+- gitpython=3.1.40
+- glib=2.78.3
+- glib-tools=2.78.3
 - glog=0.6.0
- gmp=6.2.1
+- glpk=5.0
+- gmp=6.3.0
 - graphite2=1.3.13
- graphviz=8.1.0
- gst-plugins-base=1.22.5
- gstreamer=1.22.5
+- graphviz=9.0.0
+- gst-plugins-base=1.22.8
+- gstreamer=1.22.8
 - gtk2=2.24.33
 - gts=0.7.6
- harfbuzz=7.3.0
+- gurobi=11.0.0
+- harfbuzz=8.3.0
 - hdf4=4.2.15
- hdf5=1.14.1
+- hdf5=1.14.3
 - humanfriendly=10.0
- icu=72.1
- idna=3.4
- importlib-metadata=6.8.0
- importlib_metadata=6.8.0
- importlib_resources=6.0.0
+- icu=73.2
+- idna=3.6
+- importlib-metadata=7.0.1
+- importlib_metadata=7.0.1
+- importlib_resources=6.1.1
 - iniconfig=2.0.0
- ipopt=3.14.12
- ipykernel=6.24.0
- ipython=8.14.0
- ipython_genutils=0.2.0
- ipywidgets=8.0.7
- jedi=0.18.2
+- ipopt=3.14.13
+- ipykernel=6.28.0
+- ipython=8.19.0
+- ipywidgets=8.1.1
+- isoduration=20.11.0
+- jedi=0.19.1
 - jinja2=3.1.2
- joblib=1.3.0
- json-c=0.16
+- joblib=1.3.2
+- json-c=0.17
 - json5=0.9.14
- jsonschema=4.18.4
- jsonschema-specifications=2023.7.1
+- jsonpointer=2.4
+- jsonschema=4.20.0
+- jsonschema-specifications=2023.12.1
+- jsonschema-with-format-nongpl=4.20.0
 - jupyter=1.0.0
- jupyter-lsp=2.2.0
- jupyter_client=8.3.0
+- jupyter-lsp=2.2.1
+- jupyter_client=8.6.0
 - jupyter_console=6.6.3
- jupyter_core=5.3.1
- jupyter_events=0.6.3
- jupyter_server=2.7.0
- jupyter_server_terminals=0.4.4
- jupyterlab=4.0.3
- jupyterlab_pygments=0.2.2
- jupyterlab_server=2.24.0
- jupyterlab_widgets=3.0.8
- kealib=1.5.1
+- jupyter_core=5.6.1
+- jupyter_events=0.9.0
+- jupyter_server=2.12.1
+- jupyter_server_terminals=0.5.1
+- jupyterlab=4.0.10
+- jupyterlab_pygments=0.3.0
+- jupyterlab_server=2.25.2
+- jupyterlab_widgets=3.0.9
+- kealib=1.5.3
 - keyutils=1.6.1
- kiwisolver=1.4.4
- krb5=1.21.1
+- kiwisolver=1.4.5
+- krb5=1.21.2
 - lame=3.100
- lcms2=2.15
+- lcms2=2.16
 - ld_impl_linux-64=2.40
 - lerc=4.0.0
- libabseil=20230125.3
- libaec=1.0.6
- libarchive=3.6.2
- libarrow=12.0.1
+- libabseil=20230802.1
+- libaec=1.1.2
+- libarchive=3.7.2
+- libarrow=14.0.2
+- libarrow-acero=14.0.2
+- libarrow-dataset=14.0.2
+- libarrow-flight=14.0.2
+- libarrow-flight-sql=14.0.2
+- libarrow-gandiva=14.0.2
+- libarrow-substrait=14.0.2
 - libblas=3.9.0
- libbrotlicommon=1.0.9
- libbrotlidec=1.0.9
- libbrotlienc=1.0.9
- libcap=2.67
+- libboost-headers=1.84.0
+- libbrotlicommon=1.1.0
+- libbrotlidec=1.1.0
+- libbrotlienc=1.1.0
+- libcap=2.69
 - libcblas=3.9.0
 - libclang=15.0.7
 - libclang13=15.0.7
 - libcrc32c=1.1.2
 - libcups=2.3.3
- libcurl=8.2.0
- libdeflate=1.18
+- libcurl=8.5.0
+- libdeflate=1.19
 - libedit=3.1.20191231
 - libev=4.33
 - libevent=2.1.12
 - libexpat=2.5.0
 - libffi=3.4.2
 - libflac=1.4.3
- libgcc-ng=13.1.0
- libgcrypt=1.10.1
+- libgcc-ng=13.2.0
+- libgcrypt=1.10.3
 - libgd=2.3.3
- libgdal=3.7.0
- libgfortran-ng=13.1.0
- libgfortran5=13.1.0
- libglib=2.76.4
- libgomp=13.1.0
+- libgdal=3.7.3
+- libgfortran-ng=13.2.0
+- libgfortran5=13.2.0
+- libglib=2.78.3
+- libgomp=13.2.0
 - libgoogle-cloud=2.12.0
 - libgpg-error=1.47
- libgrpc=1.56.2
+- libgrpc=1.59.3
+- libhwloc=2.9.1
 - libiconv=1.17
- libjpeg-turbo=2.1.5.1
+- libjpeg-turbo=3.0.0
 - libkml=1.3.0
 - liblapack=3.9.0
 - liblapacke=3.9.0
 - libllvm15=15.0.7
 - libnetcdf=4.9.2
- libnghttp2=1.52.0
- libnsl=2.0.0
+- libnghttp2=1.58.0
+- libnl=3.9.0
+- libnsl=2.0.1
 - libnuma=2.0.16
 - libogg=1.3.4
- libopenblas=0.3.23
+- libopenblas=0.3.25
 - libopus=1.3.1
+- libparquet=14.0.2
 - libpng=1.6.39
- libpq=15.3
- libprotobuf=4.23.3
- librsvg=2.56.1
+- libpq=16.1
+- libprotobuf=4.24.4
+- libre2-11=2023.06.02
+- librsvg=2.56.3
 - librttopo=1.1.0
- libsndfile=1.2.0
+- libsndfile=1.2.2
 - libsodium=1.0.18
 - libspatialindex=1.9.3
- libspatialite=5.0.1
- libsqlite=3.42.0
+- libspatialite=5.1.0
+- libspral=2023.08.02
+- libsqlite=3.44.2
 - libssh2=1.11.0
- libstdcxx-ng=13.1.0
- libsystemd0=253
- libthrift=0.18.1
- libtiff=4.5.1
- libtool=2.4.7
+- libstdcxx-ng=13.2.0
+- libsystemd0=255
+- libthrift=0.19.0
+- libtiff=4.6.0
 - libutf8proc=2.8.0
 - libuuid=2.38.1
 - libvorbis=1.3.7
- libwebp=1.3.1
- libwebp-base=1.3.1
+- libwebp=1.3.2
+- libwebp-base=1.3.2
 - libxcb=1.15
- libxkbcommon=1.5.0
- libxml2=2.11.4
+- libxcrypt=4.4.36
+- libxkbcommon=1.6.0
+- libxml2=2.11.6
 - libxslt=1.1.37
- libzip=1.9.2
+- libzip=1.10.1
 - libzlib=1.2.13
+- linopy=0.3.2
 - locket=1.0.0
 - lxml=4.9.3
 - lz4=4.3.2
 - lz4-c=1.9.4
 - lzo=2.10
- mapclassify=2.5.0
+- mapclassify=2.6.1
 - markupsafe=2.1.3
- matplotlib=3.5.3
- matplotlib-base=3.5.3
+- matplotlib=3.8.2
+- matplotlib-base=3.8.2
 - matplotlib-inline=0.1.6
 - memory_profiler=0.61.0
- metis=5.1.1
- mistune=3.0.0
- mpg123=1.31.3
- msgpack-python=1.0.5
+- metis=5.1.0
+- minizip=4.0.4
+- mistune=3.0.2
+- mpg123=1.32.3
+- msgpack-python=1.0.7
 - mumps-include=5.2.1
 - mumps-seq=5.2.1
 - munch=4.0.0
@ -273,200 +295,202 @@ dependencies:
 - mysql-common=8.0.33
 - mysql-libs=8.0.33
 - nbclient=0.8.0
- nbconvert=7.7.2
- nbconvert-core=7.7.2
- nbconvert-pandoc=7.7.2
- nbformat=5.9.1
+- nbconvert=7.14.0
+- nbconvert-core=7.14.0
+- nbconvert-pandoc=7.14.0
+- nbformat=5.9.2
 - ncurses=6.4
- nest-asyncio=1.5.6
- netcdf4=1.6.4
- networkx=3.1
+- nest-asyncio=1.5.8
+- netcdf4=1.6.5
+- networkx=3.2.1
 - nomkl=1.0
- notebook=7.0.0
+- notebook=7.0.6
 - notebook-shim=0.2.3
 - nspr=4.35
- nss=3.89
- numexpr=2.8.4
- numpy=1.25.1
- openjdk=17.0.3
+- nss=3.96
+- numexpr=2.8.8
+- numpy=1.26.2
+- openjdk=21.0.1
 - openjpeg=2.5.0
 - openpyxl=3.1.2
- openssl=3.1.1
- orc=1.9.0
- overrides=7.3.1
- packaging=23.1
- pandas=2.0.3
+- openssl=3.2.0
+- orc=1.9.2
+- overrides=7.4.0
+- packaging=23.2
+- pandas=2.1.4
 - pandoc=3.1.3
 - pandocfilters=1.5.0
 - pango=1.50.14
 - parso=0.8.3
- partd=1.4.0
- patsy=0.5.3
- pcre2=10.40
+- partd=1.4.1
+- patsy=0.5.5
+- pcre2=10.42
 - pexpect=4.8.0
 - pickleshare=0.7.5
- pillow=10.0.0
- pip=23.2.1
- pixman=0.40.0
+- pillow=10.2.0
+- pip=23.3.2
+- pixman=0.42.2
 - pkgutil-resolve-name=1.3.10
- plac=1.3.5
- platformdirs=3.9.1
- pluggy=1.2.0
+- plac=1.4.2
+- platformdirs=4.1.0
+- pluggy=1.3.0
 - ply=3.11
- pooch=1.7.0
- poppler=23.05.0
+- poppler=23.12.0
 - poppler-data=0.4.12
- postgresql=15.3
- powerplantmatching=0.5.7
- progressbar2=4.2.0
- proj=9.2.1
- prometheus_client=0.17.1
- prompt-toolkit=3.0.39
- prompt_toolkit=3.0.39
- psutil=5.9.5
+- postgresql=16.1
+- powerplantmatching=0.5.8
+- progressbar2=4.3.2
+- proj=9.3.0
+- prometheus_client=0.19.0
+- prompt-toolkit=3.0.42
+- prompt_toolkit=3.0.42
+- psutil=5.9.7
 - pthread-stubs=0.4
 - ptyprocess=0.7.0
 - pulp=2.7.0
 - pulseaudio-client=16.1
 - pure_eval=0.2.2
 - py-cpuinfo=9.0.0
- pyarrow=12.0.1
+- pyarrow=14.0.2
+- pyarrow-hotfix=0.6
 - pycountry=22.3.5
 - pycparser=2.21
- pygments=2.15.1
+- pygments=2.17.2
 - pyomo=6.6.1
- pyparsing=3.1.0
- pyproj=3.6.0
- pyqt=5.15.7
- pyqt5-sip=12.11.0
+- pyparsing=3.1.1
+- pyproj=3.6.1
+- pypsa=0.26.2
+- pyqt=5.15.9
+- pyqt5-sip=12.12.2
 - pyshp=2.3.1
 - pysocks=1.7.1
- pytables=3.8.0
- pytest=7.4.0
- python=3.10.12
+- pytables=3.9.2
+- pytest=7.4.4
+- python=3.11.7
 - python-dateutil=2.8.2
- python-fastjsonschema=2.18.0
+- python-fastjsonschema=2.19.1
 - python-json-logger=2.0.7
- python-tzdata=2023.3
- python-utils=3.7.0
- python_abi=3.10
- pytz=2023.3
+- python-tzdata=2023.4
+- python-utils=3.8.1
+- python_abi=3.11
+- pytz=2023.3.post1
 - pyxlsb=1.0.10
- pyyaml=6.0
- pyzmq=25.1.0
+- pyyaml=6.0.1
+- pyzmq=25.1.2
 - qt-main=5.15.8
- qtconsole=5.4.3
- qtconsole-base=5.4.3
- qtpy=2.3.1
- rasterio=1.3.8
- rdma-core=28.9
- re2=2023.03.02
+- qtconsole-base=5.5.1
+- qtpy=2.4.1
+- rasterio=1.3.9
+- rdma-core=49.0
+- re2=2023.06.02
 - readline=8.2
- referencing=0.30.0
+- referencing=0.32.0
 - requests=2.31.0
 - reretry=0.11.8
 - rfc3339-validator=0.1.4
 - rfc3986-validator=0.1.1
- rioxarray=0.14.1
- rpds-py=0.9.2
- rtree=1.0.1
- s2n=1.3.46
- scikit-learn=1.3.0
- scipy=1.11.1
+- rioxarray=0.15.0
+- rpds-py=0.16.2
+- rtree=1.1.0
+- s2n=1.4.1
+- scikit-learn=1.3.2
+- scipy=1.11.4
 - scotch=6.0.9
- seaborn=0.12.2
- seaborn-base=0.12.2
+- seaborn=0.13.0
+- seaborn-base=0.13.0
 - send2trash=1.8.2
- setuptools=68.0.0
- setuptools-scm=7.1.0
- setuptools_scm=7.1.0
- shapely=2.0.1
- sip=6.7.10
+- setuptools=69.0.3
+- setuptools-scm=8.0.4
+- setuptools_scm=8.0.4
+- shapely=2.0.2
+- sip=6.7.12
 - six=1.16.0
- smart_open=6.3.0
- smmap=3.0.5
- snakemake-minimal=7.30.2
+- smart_open=6.4.0
+- smmap=5.0.0
+- snakemake-minimal=7.32.4
 - snappy=1.1.10
 - sniffio=1.3.0
 - snuggs=1.4.7
 - sortedcontainers=2.4.0
- soupsieve=2.3.2.post1
- sqlite=3.42.0
+- soupsieve=2.5
+- sqlite=3.44.2
 - stack_data=0.6.2
- statsmodels=0.14.0
+- statsmodels=0.14.1
 - stopit=1.1.2
- tabula-py=2.6.0
+- tabula-py=2.7.0
 - tabulate=0.9.0
- tblib=1.7.0
- terminado=0.17.1
+- tblib=3.0.0
+- terminado=0.18.0
 - threadpoolctl=3.2.0
- throttler=1.2.1
- tiledb=2.13.2
+- throttler=1.2.2
+- tiledb=2.18.2
 - tinycss2=1.2.1
- tk=8.6.12
+- tk=8.6.13
 - toml=0.10.2
 - tomli=2.0.1
 - toolz=0.12.0
 - toposort=1.10
- tornado=6.3.2
- tqdm=4.65.0
- traitlets=5.9.0
- typing-extensions=4.7.1
- typing_extensions=4.7.1
+- tornado=6.3.3
+- tqdm=4.66.1
+- traitlets=5.14.1
+- types-python-dateutil=2.8.19.14
+- typing-extensions=4.9.0
+- typing_extensions=4.9.0
 - typing_utils=0.1.0
- tzcode=2023c
- tzdata=2023c
- ucx=1.14.1
- unicodedata2=15.0.0
- unidecode=1.3.6
- unixodbc=2.3.10
- urllib3=2.0.4
- wcwidth=0.2.6
+- tzcode=2023d
+- tzdata=2023d
+- ucx=1.15.0
+- unidecode=1.3.7
+- unixodbc=2.3.12
+- uri-template=1.3.0
+- uriparser=0.9.7
+- urllib3=2.1.0
+- validators=0.22.0
+- wcwidth=0.2.12
+- webcolors=1.13
 - webencodings=0.5.1
- websocket-client=1.6.1
- wheel=0.41.0
- widgetsnbextension=4.0.8
- wrapt=1.15.0
- xarray=2023.7.0
+- websocket-client=1.7.0
+- wheel=0.42.0
+- widgetsnbextension=4.0.9
+- wrapt=1.16.0
+- xarray=2023.12.0
 - xcb-util=0.4.0
 - xcb-util-image=0.4.0
 - xcb-util-keysyms=0.4.0
 - xcb-util-renderutil=0.3.9
 - xcb-util-wm=0.4.1
- xerces-c=3.2.4
- xkeyboard-config=2.39
+- xerces-c=3.2.5
+- xkeyboard-config=2.40
 - xlrd=2.0.1
 - xorg-fixesproto=5.0
 - xorg-inputproto=2.3.2
 - xorg-kbproto=1.0.7
 - xorg-libice=1.1.1
 - xorg-libsm=1.2.4
- xorg-libx11=1.8.6
+- xorg-libx11=1.8.7
 - xorg-libxau=1.0.11
 - xorg-libxdmcp=1.1.3
 - xorg-libxext=1.3.4
 - xorg-libxfixes=5.0.3
 - xorg-libxi=1.7.10
 - xorg-libxrender=0.9.11
+- xorg-libxt=1.3.0
 - xorg-libxtst=1.2.3
 - xorg-recordproto=1.14.2
 - xorg-renderproto=0.11.1
 - xorg-xextproto=7.3.0
 - xorg-xf86vidmodeproto=2.3.1
 - xorg-xproto=7.0.31
- xyzservices=2023.7.0
+- xyzservices=2023.10.1
 - xz=5.2.6
 - yaml=0.2.5
- yte=1.5.1
- zeromq=4.3.4
+- yte=1.5.4
+- zeromq=4.3.5
 - zict=3.0.0
- zipp=3.16.2
+- zipp=3.17.0
 - zlib=1.2.13
 - zlib-ng=2.0.7
- zstd=1.5.2
+- zstd=1.5.5
 - pip:
-  - gurobipy==10.0.2
-  - linopy==0.2.2
-  - pypsa==0.25.1
-  - tsam==2.3.0
-  - validators==0.20.0
+  - highspy==1.5.3
+  - tsam==2.3.1
--- a/graphics/workflow.png
+++ b/graphics/workflow.png
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@ -20,7 +20,7 @@ if config["enable"].get("prepare_links_p_nom", False):

 rule build_electricity_demand:
    params:
-        snapshots=config["snapshots"],
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
        countries=config["countries"],
        load=config["load"],
    input:
@ -41,6 +41,7 @@ rule build_powerplants:
    params:
        powerplants_filter=config["electricity"]["powerplants_filter"],
        custom_powerplants=config["electricity"]["custom_powerplants"],
+        everywhere_powerplants=config["electricity"]["everywhere_powerplants"],
        countries=config["countries"],
    input:
        base_network=RESOURCES + "networks/base.nc",
@ -61,7 +62,7 @@ rule build_powerplants:
 rule base_network:
    params:
        countries=config["countries"],
-        snapshots=config["snapshots"],
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
        lines=config["lines"],
        links=config["links"],
        transformers=config["transformers"],
@ -144,7 +145,7 @@ if config["enable"].get("build_cutout", False):

    rule build_cutout:
        params:
-            snapshots=config["snapshots"],
+            snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
            cutouts=config["atlite"]["cutouts"],
        input:
            regions_onshore=RESOURCES + "regions_onshore.geojson",
@ -258,6 +259,7 @@ else:

 rule build_renewable_profiles:
    params:
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
        renewable=config["renewable"],
        foresight=config["foresight"],
    input:
@ -355,6 +357,8 @@ rule build_hydro_profile:
 if config["lines"]["dynamic_line_rating"]["activate"]:

    rule build_line_rating:
+        params:
+            snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
        input:
            base_network=RESOURCES + "networks/base.nc",
            cutout="cutouts/"
@ -532,13 +536,20 @@ rule add_extra_components:

 rule prepare_network:
    params:
+        snapshots={
+            "resolution": config["snapshots"].get("resolution", False),
+            "segmentation": config["snapshots"].get("segmentation", False),
+        },
        links=config["links"],
        lines=config["lines"],
        co2base=config["electricity"]["co2base"],
+        co2limit_enable=config["electricity"].get("co2limit_enable", False),
        co2limit=config["electricity"]["co2limit"],
+        gaslimit_enable=config["electricity"].get("gaslimit_enable", False),
        gaslimit=config["electricity"].get("gaslimit"),
        max_hours=config["electricity"]["max_hours"],
        costs=config["costs"],
+        autarky=config["electricity"].get("autarky", {}),
    input:
        RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
        tech_costs=COSTS,
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@ -67,81 +67,65 @@ rule build_simplified_population_layouts:
        "../scripts/build_clustered_population_layouts.py"


-if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
-
-    rule build_gas_network:
-        input:
-            gas_network="data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson",
-        output:
-            cleaned_gas_network=RESOURCES + "gas_network.csv",
-        resources:
-            mem_mb=4000,
-        log:
-            LOGS + "build_gas_network.log",
-        conda:
-            "../envs/environment.yaml"
-        script:
-            "../scripts/build_gas_network.py"
-
-    rule build_gas_input_locations:
-        input:
-            gem=HTTP.remote(
-                "https://globalenergymonitor.org/wp-content/uploads/2023/07/Europe-Gas-Tracker-2023-03-v3.xlsx",
-                keep_local=True,
-            ),
-            entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson",
-            storage="data/gas_network/scigrid-gas/data/IGGIELGN_Storages.geojson",
-            regions_onshore=RESOURCES
-            + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
-            regions_offshore=RESOURCES
-            + "regions_offshore_elec_s{simpl}_{clusters}.geojson",
-        output:
-            gas_input_nodes=RESOURCES
-            + "gas_input_locations_s{simpl}_{clusters}.geojson",
-            gas_input_nodes_simplified=RESOURCES
-            + "gas_input_locations_s{simpl}_{clusters}_simplified.csv",
-        resources:
-            mem_mb=2000,
-        log:
-            LOGS + "build_gas_input_locations_s{simpl}_{clusters}.log",
-        conda:
-            "../envs/environment.yaml"
-        script:
-            "../scripts/build_gas_input_locations.py"
-
-    rule cluster_gas_network:
-        input:
-            cleaned_gas_network=RESOURCES + "gas_network.csv",
-            regions_onshore=RESOURCES
-            + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
-            regions_offshore=RESOURCES
-            + "regions_offshore_elec_s{simpl}_{clusters}.geojson",
-        output:
-            clustered_gas_network=RESOURCES + "gas_network_elec_s{simpl}_{clusters}.csv",
-        resources:
-            mem_mb=4000,
-        log:
-            LOGS + "cluster_gas_network_s{simpl}_{clusters}.log",
-        conda:
-            "../envs/environment.yaml"
-        script:
-            "../scripts/cluster_gas_network.py"
-
-    gas_infrastructure = {
-        **rules.cluster_gas_network.output,
-        **rules.build_gas_input_locations.output,
-    }
+rule build_gas_network:
+    input:
+        gas_network="data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson",
+    output:
+        cleaned_gas_network=RESOURCES + "gas_network.csv",
+    resources:
+        mem_mb=4000,
+    log:
+        LOGS + "build_gas_network.log",
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/build_gas_network.py"


-if not (config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]):
-    # this is effecively an `else` statement which is however not liked by snakefmt
+rule build_gas_input_locations:
+    input:
+        gem=HTTP.remote(
+            "https://globalenergymonitor.org/wp-content/uploads/2023/07/Europe-Gas-Tracker-2023-03-v3.xlsx",
+            keep_local=True,
+        ),
+        entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson",
+        storage="data/gas_network/scigrid-gas/data/IGGIELGN_Storages.geojson",
+        regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
+        regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}_{clusters}.geojson",
+    output:
+        gas_input_nodes=RESOURCES + "gas_input_locations_s{simpl}_{clusters}.geojson",
+        gas_input_nodes_simplified=RESOURCES
+        + "gas_input_locations_s{simpl}_{clusters}_simplified.csv",
+    resources:
+        mem_mb=2000,
+    log:
+        LOGS + "build_gas_input_locations_s{simpl}_{clusters}.log",
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/build_gas_input_locations.py"

-    gas_infrastructure = {}
+
+rule cluster_gas_network:
+    input:
+        cleaned_gas_network=RESOURCES + "gas_network.csv",
+        regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
+        regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}_{clusters}.geojson",
+    output:
+        clustered_gas_network=RESOURCES + "gas_network_elec_s{simpl}_{clusters}.csv",
+    resources:
+        mem_mb=4000,
+    log:
+        LOGS + "cluster_gas_network_s{simpl}_{clusters}.log",
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/cluster_gas_network.py"


 rule build_heat_demands:
    params:
-        snapshots=config["snapshots"],
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
    input:
        pop_layout=RESOURCES + "pop_layout_{scope}.nc",
        regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@ -163,7 +147,7 @@ rule build_heat_demands:

 rule build_temperature_profiles:
    params:
-        snapshots=config["snapshots"],
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
    input:
        pop_layout=RESOURCES + "pop_layout_{scope}.nc",
        regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@ -215,7 +199,7 @@ rule build_cop_profiles:

 rule build_solar_thermal_profiles:
    params:
-        snapshots=config["snapshots"],
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
        solar_thermal=config["solar_thermal"],
    input:
        pop_layout=RESOURCES + "pop_layout_{scope}.nc",
@ -678,7 +662,7 @@ rule build_shipping_demand:

 rule build_transport_demand:
    params:
-        snapshots=config["snapshots"],
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
        sector=config["sector"],
    input:
        clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
@ -722,7 +706,8 @@ rule prepare_sector_network:
    input:
        **build_retro_cost_output,
        **build_biomass_transport_costs_output,
-        **gas_infrastructure,
+        **rules.cluster_gas_network.output,
+        **rules.build_gas_input_locations.output,
        **build_sequestration_potentials_output,
        network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
        energy_totals_name=RESOURCES + "energy_totals.csv",
--- a/rules/common.smk
+++ b/rules/common.smk
@ -2,9 +2,14 @@
 #
 # SPDX-License-Identifier: MIT

-import os, sys
+import os, sys, glob
+
+helper_source_path = [match for match in glob.glob("**/_helpers.py", recursive=True)]
+
+for path in helper_source_path:
+    path = os.path.dirname(os.path.abspath(path))
+    sys.path.insert(0, os.path.abspath(path))

-sys.path.insert(0, os.path.abspath("scripts"))
 from _helpers import validate_checksum


--- a/rules/postprocess.smk
+++ b/rules/postprocess.smk
@ -85,7 +85,7 @@ rule make_summary:
    params:
        foresight=config["foresight"],
        costs=config["costs"],
-        snapshots=config["snapshots"],
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
        scenario=config["scenario"],
        RDIR=RDIR,
    input:
--- a/rules/retrieve.smk
+++ b/rules/retrieve.smk
@ -162,9 +162,7 @@ if config["enable"]["retrieve"] and config["enable"].get(
            "../scripts/retrieve_sector_databundle.py"


-if config["enable"]["retrieve"] and (
-    config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]
-):
+if config["enable"]["retrieve"]:
    datafiles = [
        "IGGIELGN_LNGs.geojson",
        "IGGIELGN_BorderPoints.geojson",
--- a/rules/validate.smk
+++ b/rules/validate.smk
@ -17,7 +17,7 @@ rule build_electricity_production:
    The data is used for validation of the optimization results.
    """
    params:
-        snapshots=config["snapshots"],
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
        countries=config["countries"],
    output:
        RESOURCES + "historical_electricity_production.csv",
@ -35,7 +35,7 @@ rule build_cross_border_flows:
    The data is used for validation of the optimization results.
    """
    params:
-        snapshots=config["snapshots"],
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
        countries=config["countries"],
    input:
        network=RESOURCES + "networks/base.nc",
@ -55,7 +55,7 @@ rule build_electricity_prices:
    The data is used for validation of the optimization results.
    """
    params:
-        snapshots=config["snapshots"],
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
        countries=config["countries"],
    output:
        RESOURCES + "historical_electricity_prices.csv",
--- a/scripts/_helpers.py
+++ b/scripts/_helpers.py
@ -7,6 +7,7 @@ import contextlib
 import hashlib
 import logging
 import os
+import re
 import urllib
 from pathlib import Path

@ -23,6 +24,35 @@ logger = logging.getLogger(__name__)
 REGION_COLS = ["geometry", "name", "x", "y", "country"]


+def get_opt(opts, expr, flags=None):
+    """
+    Return the first option matching the regular expression.
+
+    The regular expression is case-insensitive by default.
+    """
+    if flags is None:
+        flags = re.IGNORECASE
+    for o in opts:
+        match = re.match(expr, o, flags=flags)
+        if match:
+            return match.group(0)
+    return None
+
+
+def find_opt(opts, expr):
+    """
+    Return if available the float after the expression.
+    """
+    for o in opts:
+        if expr in o:
+            m = re.findall("[0-9]*\.?[0-9]+$", o)
+            if len(m) > 0:
+                return True, float(m[0])
+            else:
+                return True, None
+    return False, None
+
+
 # Define a context manager to temporarily mute print statements
@contextlib.contextmanager
 def mute_print():
--- a/scripts/add_brownfield.py
+++ b/scripts/add_brownfield.py
@ -135,9 +135,7 @@ def disable_grid_expansion_if_LV_limit_hit(n):

    # allow small numerical differences
    if lv_limit - total_expansion < 1:
-        logger.info(
-            f"LV is already reached (gap {diff} MWkm), disabling expansion and LV limit"
-        )
+        logger.info(f"LV is already reached, disabling expansion and LV limit")
        extendable_acs = n.lines.query("s_nom_extendable").index
        n.lines.loc[extendable_acs, "s_nom_extendable"] = False
        n.lines.loc[extendable_acs, "s_nom"] = n.lines.loc[extendable_acs, "s_nom_min"]
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@ -305,7 +305,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
        else:
            bus0 = vars(spatial)[carrier[generator]].nodes
            if "EU" not in vars(spatial)[carrier[generator]].locations:
-                bus0 = bus0.intersection(capacity.index + " gas")
+                bus0 = bus0.intersection(capacity.index + " " + carrier[generator])

            # check for missing bus
            missing_bus = pd.Index(bus0).difference(n.buses.index)
--- a/scripts/base_network.py
+++ b/scripts/base_network.py
@ -725,11 +725,12 @@ def base_network(
    transformers = _set_electrical_parameters_transformers(transformers, config)
    links = _set_electrical_parameters_links(links, config, links_p_nom)
    converters = _set_electrical_parameters_converters(converters, config)
+    snapshots = snakemake.params.snapshots

    n = pypsa.Network()
    n.name = "PyPSA-Eur"

-    n.set_snapshots(pd.date_range(freq="h", **config["snapshots"]))
+    n.set_snapshots(pd.date_range(freq="h", **snapshots))
    n.madd("Carrier", ["AC", "DC"])

    n.import_components_from_dataframe(buses, "Bus")
--- a/scripts/build_industrial_distribution_key.py
+++ b/scripts/build_industrial_distribution_key.py
@ -130,7 +130,7 @@ def build_nodal_distribution_key(hotmaps, regions, countries):

        if not facilities.empty:
            emissions = facilities["Emissions_ETS_2014"].fillna(
-                hotmaps["Emissions_EPRTR_2014"]
+                hotmaps["Emissions_EPRTR_2014"].dropna()
            )
            if emissions.sum() == 0:
                key = pd.Series(1 / len(facilities), facilities.index)
--- a/scripts/build_line_rating.py
+++ b/scripts/build_line_rating.py
@ -146,8 +146,10 @@ if __name__ == "__main__":
        )
    configure_logging(snakemake)

+    snapshots = snakemake.params.snapshots
+
    n = pypsa.Network(snakemake.input.base_network)
-    time = pd.date_range(freq="h", **snakemake.config["snapshots"])
+    time = pd.date_range(freq="h", **snapshots)
    cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time)

    da = calculate_line_rating(n, cutout)
--- a/scripts/build_monthly_prices.py
+++ b/scripts/build_monthly_prices.py
@ -6,11 +6,8 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-Created on Tue May 16 10:37:35 2023.
-
-This script extracts monthly fuel prices of oil, gas, coal and lignite,
-as well as CO2 prices
-
+This script extracts monthly fuel prices of oil, gas, coal and lignite, as well
+as CO2 prices.

 Inputs
 ------
--- a/scripts/build_powerplants.py
+++ b/scripts/build_powerplants.py
@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT

@ -10,6 +10,7 @@ Retrieves conventional powerplant capacities and locations from
 these to buses and creates a ``.csv`` file. It is possible to amend the
 powerplant database with custom entries provided in
 ``data/custom_powerplants.csv``.
+Lastly, for every substation, powerplants with zero-initial capacity can be added for certain fuel types automatically.

 Relevant Settings
 -----------------
@ -19,6 +20,7 @@ Relevant Settings
    electricity:
      powerplants_filter:
      custom_powerplants:
+      everywhere_powerplants:

 .. seealso::
    Documentation of the configuration file ``config/config.yaml`` at
@ -44,6 +46,7 @@ Description
 -----------

 The configuration options ``electricity: powerplants_filter`` and ``electricity: custom_powerplants`` can be used to control whether data should be retrieved from the original powerplants database or from custom amendmends. These specify `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ commands.
+In addition the configuration option ``electricity: everywhere_powerplants`` can be used to place powerplants with zero-initial capacity of certain fuel types at all substations.

 1. Adding all powerplants from custom:

@ -73,10 +76,18 @@ The configuration options ``electricity: powerplants_filter`` and ``electricity:

        powerplants_filter: Country not in ['Germany'] and YearCommissioned <= 2015
        custom_powerplants: YearCommissioned <= 2015
+
+4. Adding powerplants at all substations for 4 conventional carrier types:
+
+    .. code:: yaml
+
+        everywhere_powerplants: ['Natural Gas', 'Coal', 'nuclear', 'OCGT']
 """

+import itertools
 import logging

+import numpy as np
 import pandas as pd
 import powerplantmatching as pm
 import pypsa
@ -97,6 +108,45 @@ def add_custom_powerplants(ppl, custom_powerplants, custom_ppl_query=False):
    )


+def add_everywhere_powerplants(ppl, substations, everywhere_powerplants):
+    # Create a dataframe with "everywhere_powerplants" of stated carriers at the location of all substations
+    everywhere_ppl = (
+        pd.DataFrame(
+            itertools.product(substations.index.values, everywhere_powerplants),
+            columns=["substation_index", "Fueltype"],
+        ).merge(
+            substations[["x", "y", "country"]],
+            left_on="substation_index",
+            right_index=True,
+        )
+    ).drop(columns="substation_index")
+
+    # PPL uses different columns names compared to substations dataframe -> rename
+    everywhere_ppl = everywhere_ppl.rename(
+        columns={"x": "lon", "y": "lat", "country": "Country"}
+    )
+
+    # Add default values for the powerplants
+    everywhere_ppl["Name"] = (
+        "Automatically added everywhere-powerplant " + everywhere_ppl.Fueltype
+    )
+    everywhere_ppl["Set"] = "PP"
+    everywhere_ppl["Technology"] = everywhere_ppl["Fueltype"]
+    everywhere_ppl["Capacity"] = 0.0
+
+    # Assign plausible values for the commissioning and decommissioning years
+    # required for multi-year models
+    everywhere_ppl["DateIn"] = ppl["DateIn"].min()
+    everywhere_ppl["DateOut"] = ppl["DateOut"].max()
+
+    # NaN values for efficiency will be replaced by the generic efficiency by attach_conventional_generators(...) in add_electricity.py later
+    everywhere_ppl["Efficiency"] = np.nan
+
+    return pd.concat(
+        [ppl, everywhere_ppl], sort=False, ignore_index=True, verify_integrity=True
+    )
+
+
 def replace_natural_gas_technology(df):
    mapping = {"Steam Turbine": "CCGT", "Combustion Engine": "OCGT"}
    tech = df.Technology.replace(mapping).fillna("CCGT")
@ -149,6 +199,11 @@ if __name__ == "__main__":
    if countries_wo_ppl := set(countries) - set(ppl.Country.unique()):
        logging.warning(f"No powerplants known in: {', '.join(countries_wo_ppl)}")

+    # Add "everywhere powerplants" to all bus locations
+    ppl = add_everywhere_powerplants(
+        ppl, n.buses.query("substation_lv"), snakemake.params.everywhere_powerplants
+    )
+
    substations = n.buses.query("substation_lv")
    ppl = ppl.dropna(subset=["lat", "lon"])
    ppl = map_country_bus(ppl, substations)
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@ -221,6 +221,7 @@ if __name__ == "__main__":

    correction_factor = params.get("correction_factor", 1.0)
    capacity_per_sqkm = params["capacity_per_sqkm"]
+    snapshots = snakemake.params.snapshots

    if correction_factor != 1.0:
        logger.info(f"correction_factor is set as {correction_factor}")
@ -230,7 +231,7 @@ if __name__ == "__main__":
    else:
        client = None

-    sns = pd.date_range(freq="h", **snakemake.config["snapshots"])
+    sns = pd.date_range(freq="h", **snapshots)
    cutout = atlite.Cutout(snakemake.input.cutout).sel(time=sns)
    regions = gpd.read_file(snakemake.input.regions)
    assert not regions.empty, (
--- a/scripts/build_retro_cost.py
+++ b/scripts/build_retro_cost.py
@ -533,16 +533,16 @@ def prepare_temperature_data():
    """
    temperature = xr.open_dataarray(snakemake.input.air_temperature).to_pandas()
    d_heat = (
-        temperature.groupby(temperature.columns.str[:2], axis=1)
+        temperature.T.groupby(temperature.columns.str[:2])
        .mean()
-        .resample("1D")
+        .T.resample("1D")
        .mean()
        < t_threshold
    ).sum()
    temperature_average_d_heat = (
-        temperature.groupby(temperature.columns.str[:2], axis=1)
+        temperature.T.groupby(temperature.columns.str[:2])
        .mean()
-        .apply(
+        .T.apply(
            lambda x: get_average_temperature_during_heating_season(x, t_threshold=15)
        )
    )
@ -610,7 +610,7 @@ def calculate_costs(u_values, l, cost_retro, window_assumptions):
            cost_retro.loc[x.name[3], "cost_var"]
            * 100
            * float(l)
-            * l_weight.loc[x.name[3]][0]
+            * l_weight.loc[x.name[3]].iloc[0]
            + cost_retro.loc[x.name[3], "cost_fix"]
        )
        * x.A_element
@ -720,6 +720,7 @@ def map_to_lstrength(l_strength, df):
        .swaplevel(axis=1)
        .dropna(axis=1)
    )
+
    return pd.concat([df.drop([2, 3], axis=1, level=1), l_strength_df], axis=1)


@ -800,6 +801,7 @@ def calculate_heat_losses(u_values, data_tabula, l_strength, temperature_factor)
        * data_tabula.A_envelope
        / data_tabula.A_C_Ref
    )
+
    heat_transfer_perm2 = pd.concat(
        [
            heat_transfer_perm2,
--- a/scripts/build_ship_raster.py
+++ b/scripts/build_ship_raster.py
@ -42,8 +42,8 @@ Description
 """

 import logging
-import os
 import zipfile
+from pathlib import Path

 import rioxarray
 from _helpers import configure_logging
@ -62,11 +62,13 @@ if __name__ == "__main__":
    xs, Xs, ys, Ys = zip(*(determine_cutout_xXyY(cutout) for cutout in cutouts))

    with zipfile.ZipFile(snakemake.input.ship_density) as zip_f:
-        zip_f.extract("shipdensity_global.tif")
-        with rioxarray.open_rasterio("shipdensity_global.tif") as ship_density:
-            ship_density = ship_density.drop_vars(["band"]).sel(
-                x=slice(min(xs), max(Xs)), y=slice(max(Ys), min(ys))
-            )
-            ship_density.rio.to_raster(snakemake.output[0])
+        resources = Path(snakemake.output[0]).parent
+        fn = "shipdensity_global.tif"
+        zip_f.extract(fn, resources)
+    with rioxarray.open_rasterio(resources / fn) as ship_density:
+        ship_density = ship_density.drop_vars(["band"]).sel(
+            x=slice(min(xs), max(Xs)), y=slice(max(Ys), min(ys))
+        )
+        ship_density.rio.to_raster(snakemake.output[0])

-    os.remove("shipdensity_global.tif")
+    (resources / fn).unlink()
--- a/scripts/determine_availability_matrix_MD_UA.py
+++ b/scripts/determine_availability_matrix_MD_UA.py
@ -2,6 +2,9 @@
 # SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
+"""
+Create land elibility analysis for Ukraine and Moldova with different datasets.
+"""

 import functools
 import logging
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@ -63,7 +63,7 @@ import re
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import configure_logging
+from _helpers import configure_logging, find_opt, get_opt
 from add_electricity import load_costs, update_transmission_costs
 from pypsa.descriptors import expand_series

@ -296,42 +296,42 @@ if __name__ == "__main__":

    set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"])

-    for o in opts:
-        m = re.match(r"^\d+h$", o, re.IGNORECASE)
-        if m is not None:
-            n = average_every_nhours(n, m.group(0))
-            break
+    # temporal averaging
+    nhours_config = snakemake.params.snapshots.get("resolution", False)
+    nhours_wildcard = get_opt(opts, r"^\d+h$")
+    nhours = nhours_wildcard or nhours_config
+    if nhours:
+        n = average_every_nhours(n, nhours)

-    for o in opts:
-        m = re.match(r"^\d+seg$", o, re.IGNORECASE)
-        if m is not None:
-            solver_name = snakemake.config["solving"]["solver"]["name"]
-            n = apply_time_segmentation(n, m.group(0)[:-3], solver_name)
-            break
+    # segments with package tsam
+    time_seg_config = snakemake.params.snapshots.get("segmentation", False)
+    time_seg_wildcard = get_opt(opts, r"^\d+seg$")
+    time_seg = time_seg_wildcard or time_seg_config
+    if time_seg:
+        solver_name = snakemake.config["solving"]["solver"]["name"]
+        n = apply_time_segmentation(n, time_seg.replace("seg", ""), solver_name)

-    for o in opts:
-        if "Co2L" in o:
-            m = re.findall("[0-9]*\.?[0-9]+$", o)
-            if len(m) > 0:
-                co2limit = float(m[0]) * snakemake.params.co2base
-                add_co2limit(n, co2limit, Nyears)
-                logger.info("Setting CO2 limit according to wildcard value.")
-            else:
-                add_co2limit(n, snakemake.params.co2limit, Nyears)
-                logger.info("Setting CO2 limit according to config value.")
-            break
+    Co2L_config = snakemake.params.co2limit_enable
+    Co2L_wildcard, co2limit_wildcard = find_opt(opts, "Co2L")
+    if Co2L_wildcard or Co2L_config:
+        if co2limit_wildcard is not None:
+            co2limit = co2limit_wildcard * snakemake.params.co2base
+            add_co2limit(n, co2limit, Nyears)
+            logger.info("Setting CO2 limit according to wildcard value.")
+        else:
+            add_co2limit(n, snakemake.params.co2limit, Nyears)
+            logger.info("Setting CO2 limit according to config value.")

-    for o in opts:
-        if "CH4L" in o:
-            m = re.findall("[0-9]*\.?[0-9]+$", o)
-            if len(m) > 0:
-                limit = float(m[0]) * 1e6
-                add_gaslimit(n, limit, Nyears)
-                logger.info("Setting gas usage limit according to wildcard value.")
-            else:
-                add_gaslimit(n, snakemake.params.gaslimit, Nyears)
-                logger.info("Setting gas usage limit according to config value.")
-            break
+    CH4L_config = snakemake.params.gaslimit_enable
+    CH4L_wildcard, gaslimit_wildcard = find_opt(opts, "CH4L")
+    if CH4L_wildcard or CH4L_config:
+        if gaslimit_wildcard is not None:
+            gaslimit = gaslimit_wildcard * 1e6
+            add_gaslimit(n, gaslimit, Nyears)
+            logger.info("Setting gas usage limit according to wildcard value.")
+        else:
+            add_gaslimit(n, snakemake.params.gaslimit, Nyears)
+            logger.info("Setting gas usage limit according to config value.")

    for o in opts:
        if "+" not in o:
@ -352,21 +352,26 @@ if __name__ == "__main__":
                    sel = c.df.carrier.str.contains(carrier)
                    c.df.loc[sel, attr] *= factor

-    for o in opts:
-        if "Ept" in o:
-            logger.info(
-                "Setting time dependent emission prices according spot market price"
+    emission_prices = snakemake.params.costs["emission_prices"]
+    Ept_config = emission_prices.get("co2_monthly_prices", False)
+    Ept_wildcard = "Ept" in opts
+    Ep_config = emission_prices.get("enable", False)
+    Ep_wildcard, co2_wildcard = find_opt(opts, "Ep")
+
+    if Ept_wildcard or Ept_config:
+        logger.info(
+            "Setting time dependent emission prices according spot market price"
+        )
+        add_dynamic_emission_prices(n)
+    elif Ep_wildcard or Ep_config:
+        if co2_wildcard is not None:
+            logger.info("Setting CO2 prices according to wildcard value.")
+            add_emission_prices(n, dict(co2=co2_wildcard))
+        else:
+            logger.info("Setting CO2 prices according to config value.")
+            add_emission_prices(
+                n, dict(co2=snakemake.params.costs["emission_prices"]["co2"])
            )
-            add_dynamic_emission_prices(n)
-        elif "Ep" in o:
-            m = re.findall("[0-9]*\.?[0-9]+$", o)
-            if len(m) > 0:
-                logger.info("Setting emission prices according to wildcard value.")
-                add_emission_prices(n, dict(co2=float(m[0])))
-            else:
-                logger.info("Setting emission prices according to config value.")
-                add_emission_prices(n, snakemake.params.costs["emission_prices"])
-            break

    ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:]
    set_transmission_limit(n, ll_type, factor, costs, Nyears)
@ -379,10 +384,12 @@ if __name__ == "__main__":
        p_nom_max_ext=snakemake.params.links.get("max_extension", np.inf),
    )

-    if "ATK" in opts:
-        enforce_autarky(n)
-    elif "ATKc" in opts:
-        enforce_autarky(n, only_crossborder=True)
+    autarky_config = snakemake.params.autarky
+    if "ATK" in opts or autarky_config.get("enable", False):
+        only_crossborder = False
+        if "ATKc" in opts or autarky_config.get("by_country", False):
+            only_crossborder = True
+        enforce_autarky(n, only_crossborder=only_crossborder)

    n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
    n.export_to_netcdf(snakemake.output[0])
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@ -102,7 +102,10 @@ def define_spatial(nodes, options):
        spatial.gas.biogas = ["EU biogas"]
        spatial.gas.industry = ["gas for industry"]
        spatial.gas.biogas_to_gas = ["EU biogas to gas"]
-        spatial.gas.biogas_to_gas_cc = ["EU biogas to gas CC"]
+        if options.get("biomass_spatial", options["biomass_transport"]):
+            spatial.gas.biogas_to_gas_cc = nodes + " biogas to gas CC"
+        else:
+            spatial.gas.biogas_to_gas_cc = ["EU biogas to gas CC"]
        if options.get("co2_spatial", options["co2network"]):
            spatial.gas.industry_cc = nodes + " gas for industry CC"
        else:
@ -129,15 +132,43 @@ def define_spatial(nodes, options):
    spatial.h2.locations = nodes

    # methanol
+
+    # beware: unlike other carriers, uses locations rather than locations+carriername
+    # this allows to avoid separation between nodes and locations
+
    spatial.methanol = SimpleNamespace()
+
    spatial.methanol.nodes = ["EU methanol"]
    spatial.methanol.locations = ["EU"]

+    if options["regional_methanol_demand"]:
+        spatial.methanol.demand_locations = nodes
+        spatial.methanol.shipping = nodes + " shipping methanol"
+    else:
+        spatial.methanol.demand_locations = ["EU"]
+        spatial.methanol.shipping = ["EU shipping methanol"]
+
    # oil
    spatial.oil = SimpleNamespace()
+
    spatial.oil.nodes = ["EU oil"]
    spatial.oil.locations = ["EU"]

+    if options["regional_oil_demand"]:
+        spatial.oil.demand_locations = nodes
+        spatial.oil.naphtha = nodes + " naphtha for industry"
+        spatial.oil.kerosene = nodes + " kerosene for aviation"
+        spatial.oil.shipping = nodes + " shipping oil"
+        spatial.oil.agriculture_machinery = nodes + " agriculture machinery oil"
+        spatial.oil.land_transport = nodes + " land transport oil"
+    else:
+        spatial.oil.demand_locations = ["EU"]
+        spatial.oil.naphtha = ["EU naphtha for industry"]
+        spatial.oil.kerosene = ["EU kerosene for aviation"]
+        spatial.oil.shipping = ["EU shipping oil"]
+        spatial.oil.agriculture_machinery = ["EU agriculture machinery oil"]
+        spatial.oil.land_transport = ["EU land transport oil"]
+
    # uranium
    spatial.uranium = SimpleNamespace()
    spatial.uranium.nodes = ["EU uranium"]
@ -258,6 +289,8 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year):
    )

    planning_horizons = snakemake.params.planning_horizons
+    if not isinstance(planning_horizons, list):
+        planning_horizons = [planning_horizons]
    t_0 = planning_horizons[0]

    if "be" in o:
@ -427,7 +460,7 @@ def update_wind_solar_costs(n, costs):

            logger.info(
                "Added connection cost of {:0.0f}-{:0.0f} Eur/MW/a to {}".format(
-                    connection_cost[0].min(), connection_cost[0].max(), tech
+                    connection_cost.min(), connection_cost.max(), tech
                )
            )

@ -519,7 +552,7 @@ def patch_electricity_network(n):
    n.loads_t.p_set.rename(lambda x: x.strip(), axis=1, inplace=True)


-def add_co2_tracking(n, options):
+def add_co2_tracking(n, costs, options):
    # minus sign because opposite to how fossil fuels used:
    # CH4 burning puts CH4 down, atmosphere up
    n.add("Carrier", "co2", co2_emissions=-1.0)
@ -537,7 +570,7 @@ def add_co2_tracking(n, options):
        bus="co2 atmosphere",
    )

-    # this tracks CO2 stored, e.g. underground
+    # add CO2 tanks
    n.madd(
        "Bus",
        spatial.co2.nodes,
@ -546,6 +579,39 @@ def add_co2_tracking(n, options):
        unit="t_co2",
    )

+    n.madd(
+        "Store",
+        spatial.co2.nodes,
+        e_nom_extendable=True,
+        capital_cost=costs.at["CO2 storage tank", "fixed"],
+        carrier="co2 stored",
+        e_cyclic=True,
+        bus=spatial.co2.nodes,
+    )
+    n.add("Carrier", "co2 stored")
+
+    # this tracks CO2 sequestered, e.g. underground
+    sequestration_buses = pd.Index(spatial.co2.nodes).str.replace(
+        " stored", " sequestered"
+    )
+    n.madd(
+        "Bus",
+        sequestration_buses,
+        location=spatial.co2.locations,
+        carrier="co2 sequestered",
+        unit="t_co2",
+    )
+
+    n.madd(
+        "Link",
+        sequestration_buses,
+        bus0=spatial.co2.nodes,
+        bus1=sequestration_buses,
+        carrier="co2 sequestered",
+        efficiency=1.0,
+        p_nom_extendable=True,
+    )
+
    if options["regional_co2_sequestration_potential"]["enable"]:
        upper_limit = (
            options["regional_co2_sequestration_potential"]["max_size"] * 1e3
@ -561,22 +627,22 @@ def add_co2_tracking(n, options):
            .mul(1e6)
            / annualiser
        )  # t
-        e_nom_max = e_nom_max.rename(index=lambda x: x + " co2 stored")
+        e_nom_max = e_nom_max.rename(index=lambda x: x + " co2 sequestered")
    else:
        e_nom_max = np.inf

    n.madd(
        "Store",
-        spatial.co2.nodes,
+        sequestration_buses,
        e_nom_extendable=True,
        e_nom_max=e_nom_max,
        capital_cost=options["co2_sequestration_cost"],
-        carrier="co2 stored",
-        bus=spatial.co2.nodes,
+        bus=sequestration_buses,
        lifetime=options["co2_sequestration_lifetime"],
+        carrier="co2 sequestered",
    )

-    n.add("Carrier", "co2 stored")
+    n.add("Carrier", "co2 sequestered")

    if options["co2_vent"]:
        n.madd(
@ -605,6 +671,8 @@ def add_co2_network(n, costs):
        * co2_links.length
    )
    capital_cost = cost_onshore + cost_submarine
+    cost_factor = snakemake.config["sector"]["co2_network_cost_factor"]
+    capital_cost *= cost_factor

    n.madd(
        "Link",
@ -845,47 +913,6 @@ def add_ammonia(n, costs):
    )


-def add_wave(n, wave_cost_factor):
-    # TODO: handle in Snakefile
-    wave_fn = "data/WindWaveWEC_GLTB.xlsx"
-
-    # in kW
-    capacity = pd.Series({"Attenuator": 750, "F2HB": 1000, "MultiPA": 600})
-
-    # in EUR/MW
-    annuity_factor = calculate_annuity(25, 0.07) + 0.03
-    costs = (
-        1e6
-        * wave_cost_factor
-        * annuity_factor
-        * pd.Series({"Attenuator": 2.5, "F2HB": 2, "MultiPA": 1.5})
-    )
-
-    sheets = pd.read_excel(
-        wave_fn,
-        sheet_name=["FirthForth", "Hebrides"],
-        usecols=["Attenuator", "F2HB", "MultiPA"],
-        index_col=0,
-        skiprows=[0],
-        parse_dates=True,
-    )
-
-    wave = pd.concat(
-        [sheets[l].divide(capacity, axis=1) for l in locations], keys=locations, axis=1
-    )
-
-    for wave_type in costs.index:
-        n.add(
-            "Generator",
-            "Hebrides " + wave_type,
-            bus="GB4 0",  # TODO this location is hardcoded
-            p_nom_extendable=True,
-            carrier="wave",
-            capital_cost=costs[wave_type],
-            p_max_pu=wave["Hebrides", wave_type],
-        )
-
-
 def insert_electricity_distribution_grid(n, costs):
    # TODO pop_layout?
    # TODO options?
@ -1468,8 +1495,8 @@ def add_land_transport(n, costs):
        n.madd(
            "Bus",
            nodes,
-            location=nodes,
            suffix=" EV battery",
+            location=nodes,
            carrier="Li ion",
            unit="MWh_el",
        )
@ -1561,29 +1588,42 @@ def add_land_transport(n, costs):

        ice_efficiency = options["transport_internal_combustion_efficiency"]

-        n.madd(
-            "Load",
-            nodes,
-            suffix=" land transport oil",
-            bus=spatial.oil.nodes,
-            carrier="land transport oil",
-            p_set=ice_share / ice_efficiency * transport[nodes],
-        )
-
-        co2 = (
+        p_set_land_transport_oil = (
            ice_share
            / ice_efficiency
-            * transport[nodes].sum().sum()
-            / nhours
-            * costs.at["oil", "CO2 intensity"]
+            * transport[nodes].rename(columns=lambda x: x + " land transport oil")
        )

-        n.add(
+        if not options["regional_oil_demand"]:
+            p_set_land_transport_oil = p_set_land_transport_oil.sum(axis=1).to_frame(
+                name="EU land transport oil"
+            )
+
+        n.madd(
+            "Bus",
+            spatial.oil.land_transport,
+            location=spatial.oil.demand_locations,
+            carrier="land transport oil",
+            unit="land transport",
+        )
+
+        n.madd(
            "Load",
-            "land transport oil emissions",
-            bus="co2 atmosphere",
-            carrier="land transport oil emissions",
-            p_set=-co2,
+            spatial.oil.land_transport,
+            bus=spatial.oil.land_transport,
+            carrier="land transport oil",
+            p_set=p_set_land_transport_oil,
+        )
+
+        n.madd(
+            "Link",
+            spatial.oil.land_transport,
+            bus0=spatial.oil.nodes,
+            bus1=spatial.oil.land_transport,
+            bus2="co2 atmosphere",
+            carrier="land transport oil",
+            efficiency2=costs.at["oil", "CO2 intensity"],
+            p_nom_extendable=True,
        )


@ -1829,7 +1869,7 @@ def add_heat(n, costs):
                lifetime=costs.at[name_type + " water tank storage", "lifetime"],
            )

-        if options["boilers"]:
+        if options["resistive_heaters"]:
            key = f"{name_type} resistive heater"

            n.madd(
@ -1844,6 +1884,7 @@ def add_heat(n, costs):
                lifetime=costs.at[key, "lifetime"],
            )

+        if options["boilers"]:
            key = f"{name_type} gas boiler"

            n.madd(
@ -1981,7 +2022,7 @@ def add_heat(n, costs):
            )
        w_space["tot"] = (
            heat_demand_r["services space"] + heat_demand_r["residential space"]
-        ) / heat_demand_r.groupby(level=[1], axis=1).sum()
+        ) / heat_demand_r.T.groupby(level=[1]).sum().T

        for name in n.loads[
            n.loads.carrier.isin([x + " heat" for x in heat_systems])
@ -1990,11 +2031,21 @@ def add_heat(n, costs):
            ct = pop_layout.loc[node, "ct"]

            # weighting 'f' depending on the size of the population at the node
-            f = urban_fraction[node] if "urban" in name else (1 - urban_fraction[node])
+            if "urban central" in name:
+                f = dist_fraction[node]
+            elif "urban decentral" in name:
+                f = urban_fraction[node] - dist_fraction[node]
+            else:
+                f = 1 - urban_fraction[node]
            if f == 0:
                continue
            # get sector name ("residential"/"services"/or both "tot" for urban central)
-            sec = [x if x in name else "tot" for x in sectors][0]
+            if "urban central" in name:
+                sec = "tot"
+            if "residential" in name:
+                sec = "residential"
+            if "services" in name:
+                sec = "services"

            # get floor aread at node and region (urban/rural) in m^2
            floor_area_node = (
@ -2038,14 +2089,15 @@ def add_heat(n, costs):
                strengths = strengths.drop(s)

            # reindex normed time profile of space heat demand back to hourly resolution
-            space_pu = space_pu.reindex(index=heat_demand.index).fillna(method="ffill")
+            space_pu = space_pu.reindex(index=heat_demand.index).ffill()

            # add for each retrofitting strength a generator with heat generation profile following the profile of the heat demand
            for strength in strengths:
+                node_name = " ".join(name.split(" ")[2::])
                n.madd(
                    "Generator",
                    [node],
-                    suffix=" retrofitting " + strength + " " + name[6::],
+                    suffix=" retrofitting " + strength + " " + node_name,
                    bus=name,
                    carrier="retrofitting",
                    p_nom_extendable=True,
@ -2181,13 +2233,12 @@ def add_biomass(n, costs):
        # Assuming for costs that the CO2 from upgrading is pure, such as in amine scrubbing. I.e., with and without CC is
        # equivalent. Adding biomass CHP capture because biogas is often small-scale and decentral so further
        # from e.g. CO2 grid or buyers. This is a proxy for the added cost for e.g. a raw biogas pipeline to a central upgrading facility
-
        n.madd(
            "Link",
            spatial.gas.biogas_to_gas_cc,
            bus0=spatial.gas.biogas,
            bus1=spatial.gas.nodes,
-            bus2="co2 stored",
+            bus2=spatial.co2.nodes,
            bus3="co2 atmosphere",
            carrier="biogas to gas CC",
            capital_cost=costs.at["biogas CC", "fixed"]
@ -2254,6 +2305,14 @@ def add_biomass(n, costs):
            marginal_cost=costs.at["solid biomass", "fuel"]
            + bus_transport_costs * average_distance,
        )
+        n.add(
+            "GlobalConstraint",
+            "biomass limit",
+            carrier_attribute="solid biomass",
+            sense="<=",
+            constant=biomass_potentials["solid biomass"].sum(),
+            type="operational_limit",
+        )

    # AC buses with district heating
    urban_central = n.buses.index[n.buses.carrier == "urban central heat"]
@ -2466,9 +2525,14 @@ def add_industry(n, costs):
        efficiency=1.0,
    )

+    if len(spatial.biomass.industry_cc) <= 1 and len(spatial.co2.nodes) > 1:
+        link_names = nodes + " " + spatial.biomass.industry_cc
+    else:
+        link_names = spatial.biomass.industry_cc
+
    n.madd(
        "Link",
-        spatial.biomass.industry_cc,
+        link_names,
        bus0=spatial.biomass.nodes,
        bus1=spatial.biomass.industry,
        bus2="co2 atmosphere",
@ -2658,48 +2722,44 @@ def add_industry(n, costs):
        efficiency = (
            options["shipping_oil_efficiency"] / options["shipping_methanol_efficiency"]
        )
-        p_set_methanol = shipping_methanol_share * p_set.sum() * efficiency
+
+        p_set_methanol = (
+            shipping_methanol_share
+            * p_set.rename(lambda x: x + " shipping methanol")
+            * efficiency
+        )
+
+        if not options["regional_methanol_demand"]:
+            p_set_methanol = p_set_methanol.sum()
+
+        n.madd(
+            "Bus",
+            spatial.methanol.shipping,
+            location=spatial.methanol.demand_locations,
+            carrier="shipping methanol",
+            unit="MWh_LHV",
+        )

        n.madd(
            "Load",
-            spatial.methanol.nodes,
-            suffix=" shipping methanol",
-            bus=spatial.methanol.nodes,
+            spatial.methanol.shipping,
+            bus=spatial.methanol.shipping,
            carrier="shipping methanol",
            p_set=p_set_methanol,
        )

-        # CO2 intensity methanol based on stoichiometric calculation with 22.7 GJ/t methanol (32 g/mol), CO2 (44 g/mol), 277.78 MWh/TJ = 0.218 t/MWh
-        co2 = p_set_methanol / options["MWh_MeOH_per_tCO2"]
-
-        n.add(
-            "Load",
-            "shipping methanol emissions",
-            bus="co2 atmosphere",
-            carrier="shipping methanol emissions",
-            p_set=-co2,
-        )
-
-    if shipping_oil_share:
-        p_set_oil = shipping_oil_share * p_set.sum()
-
        n.madd(
-            "Load",
-            spatial.oil.nodes,
-            suffix=" shipping oil",
-            bus=spatial.oil.nodes,
-            carrier="shipping oil",
-            p_set=p_set_oil,
-        )
-
-        co2 = p_set_oil * costs.at["oil", "CO2 intensity"]
-
-        n.add(
-            "Load",
-            "shipping oil emissions",
-            bus="co2 atmosphere",
-            carrier="shipping oil emissions",
-            p_set=-co2,
+            "Link",
+            spatial.methanol.shipping,
+            bus0=spatial.methanol.nodes,
+            bus1=spatial.methanol.shipping,
+            bus2="co2 atmosphere",
+            carrier="shipping methanol",
+            p_nom_extendable=True,
+            efficiency2=1
+            / options[
+                "MWh_MeOH_per_tCO2"
+            ],  # CO2 intensity methanol based on stoichiometric calculation with 22.7 GJ/t methanol (32 g/mol), CO2 (44 g/mol), 277.78 MWh/TJ = 0.218 t/MWh
        )

    if "oil" not in n.buses.carrier.unique():
@ -2715,7 +2775,8 @@ def add_industry(n, costs):
        # could correct to e.g. 0.001 EUR/kWh * annuity and O&M
        n.madd(
            "Store",
-            [oil_bus + " Store" for oil_bus in spatial.oil.nodes],
+            spatial.oil.nodes,
+            suffix=" Store",
            bus=spatial.oil.nodes,
            e_nom_extendable=True,
            e_cyclic=True,
@ -2732,6 +2793,39 @@ def add_industry(n, costs):
            marginal_cost=costs.at["oil", "fuel"],
        )

+    if shipping_oil_share:
+        p_set_oil = shipping_oil_share * p_set.rename(lambda x: x + " shipping oil")
+
+        if not options["regional_oil_demand"]:
+            p_set_oil = p_set_oil.sum()
+
+        n.madd(
+            "Bus",
+            spatial.oil.shipping,
+            location=spatial.oil.demand_locations,
+            carrier="shipping oil",
+            unit="MWh_LHV",
+        )
+
+        n.madd(
+            "Load",
+            spatial.oil.shipping,
+            bus=spatial.oil.shipping,
+            carrier="shipping oil",
+            p_set=p_set_oil,
+        )
+
+        n.madd(
+            "Link",
+            spatial.oil.shipping,
+            bus0=spatial.oil.nodes,
+            bus1=spatial.oil.shipping,
+            bus2="co2 atmosphere",
+            carrier="shipping oil",
+            p_nom_extendable=True,
+            efficiency2=costs.at["oil", "CO2 intensity"],
+        )
+
    if options["oil_boilers"]:
        nodes_heat = create_nodes_for_heat_sector()[0]

@ -2775,53 +2869,101 @@ def add_industry(n, costs):
        lifetime=costs.at["Fischer-Tropsch", "lifetime"],
    )

+    # naphtha
    demand_factor = options.get("HVC_demand_factor", 1)
-    p_set = demand_factor * industrial_demand.loc[nodes, "naphtha"].sum() / nhours
    if demand_factor != 1:
        logger.warning(f"Changing HVC demand by {demand_factor*100-100:+.2f}%.")

-    n.madd(
-        "Load",
-        ["naphtha for industry"],
-        bus=spatial.oil.nodes,
-        carrier="naphtha for industry",
-        p_set=p_set,
-    )
-
-    demand_factor = options.get("aviation_demand_factor", 1)
-    all_aviation = ["total international aviation", "total domestic aviation"]
-    p_set = (
+    p_set_plastics = (
        demand_factor
-        * pop_weighted_energy_totals.loc[nodes, all_aviation].sum(axis=1).sum()
-        * 1e6
+        * industrial_demand.loc[nodes, "naphtha"].rename(
+            lambda x: x + " naphtha for industry"
+        )
        / nhours
    )
+
+    if not options["regional_oil_demand"]:
+        p_set_plastics = p_set_plastics.sum()
+
+    n.madd(
+        "Bus",
+        spatial.oil.naphtha,
+        location=spatial.oil.demand_locations,
+        carrier="naphtha for industry",
+        unit="MWh_LHV",
+    )
+
+    n.madd(
+        "Load",
+        spatial.oil.naphtha,
+        bus=spatial.oil.naphtha,
+        carrier="naphtha for industry",
+        p_set=p_set_plastics,
+    )
+
+    # some CO2 from naphtha are process emissions from steam cracker
+    # rest of CO2 released to atmosphere either in waste-to-energy or decay
+    process_co2_per_naphtha = (
+        industrial_demand.loc[nodes, "process emission from feedstock"].sum()
+        / industrial_demand.loc[nodes, "naphtha"].sum()
+    )
+    emitted_co2_per_naphtha = costs.at["oil", "CO2 intensity"] - process_co2_per_naphtha
+
+    n.madd(
+        "Link",
+        spatial.oil.naphtha,
+        bus0=spatial.oil.nodes,
+        bus1=spatial.oil.naphtha,
+        bus2="co2 atmosphere",
+        bus3=spatial.co2.process_emissions,
+        carrier="naphtha for industry",
+        p_nom_extendable=True,
+        efficiency2=emitted_co2_per_naphtha,
+        efficiency3=process_co2_per_naphtha,
+    )
+
+    # aviation
+    demand_factor = options.get("aviation_demand_factor", 1)
    if demand_factor != 1:
        logger.warning(f"Changing aviation demand by {demand_factor*100-100:+.2f}%.")

+    all_aviation = ["total international aviation", "total domestic aviation"]
+
+    p_set = (
+        demand_factor
+        * pop_weighted_energy_totals.loc[nodes, all_aviation].sum(axis=1)
+        * 1e6
+        / nhours
+    ).rename(lambda x: x + " kerosene for aviation")
+
+    if not options["regional_oil_demand"]:
+        p_set = p_set.sum()
+
+    n.madd(
+        "Bus",
+        spatial.oil.kerosene,
+        location=spatial.oil.demand_locations,
+        carrier="kerosene for aviation",
+        unit="MWh_LHV",
+    )
+
    n.madd(
        "Load",
-        ["kerosene for aviation"],
-        bus=spatial.oil.nodes,
+        spatial.oil.kerosene,
+        bus=spatial.oil.kerosene,
        carrier="kerosene for aviation",
        p_set=p_set,
    )

-    # NB: CO2 gets released again to atmosphere when plastics decay or kerosene is burned
-    # except for the process emissions when naphtha is used for petrochemicals, which can be captured with other industry process emissions
-    # tco2 per hour
-    co2_release = ["naphtha for industry", "kerosene for aviation"]
-    co2 = (
-        n.loads.loc[co2_release, "p_set"].sum() * costs.at["oil", "CO2 intensity"]
-        - industrial_demand.loc[nodes, "process emission from feedstock"].sum() / nhours
-    )
-
-    n.add(
-        "Load",
-        "oil emissions",
-        bus="co2 atmosphere",
-        carrier="oil emissions",
-        p_set=-co2,
+    n.madd(
+        "Link",
+        spatial.oil.kerosene,
+        bus0=spatial.oil.nodes,
+        bus1=spatial.oil.kerosene,
+        bus2="co2 atmosphere",
+        carrier="kerosene for aviation",
+        p_nom_extendable=True,
+        efficiency2=costs.at["oil", "CO2 intensity"],
    )

    # TODO simplify bus expression
@ -2872,19 +3014,16 @@ def add_industry(n, costs):
        unit="t_co2",
    )

-    sel = ["process emission", "process emission from feedstock"]
    if options["co2_spatial"] or options["co2network"]:
        p_set = (
-            -industrial_demand.loc[nodes, sel]
-            .sum(axis=1)
-            .rename(index=lambda x: x + " process emissions")
+            -industrial_demand.loc[nodes, "process emission"].rename(
+                index=lambda x: x + " process emissions"
+            )
            / nhours
        )
    else:
-        p_set = -industrial_demand.loc[nodes, sel].sum(axis=1).sum() / nhours
+        p_set = -industrial_demand.loc[nodes, "process emission"].sum() / nhours

-    # this should be process emissions fossil+feedstock
-    # then need load on atmosphere for feedstock emissions that are currently going to atmosphere via Link Fischer-Tropsch demand
    n.madd(
        "Load",
        spatial.co2.process_emissions,
@ -3090,9 +3229,9 @@ def add_agriculture(n, costs):
            f"Total agriculture machinery shares sum up to {total_share:.2%}, corresponding to increased or decreased demand assumptions."
        )

-    machinery_nodal_energy = pop_weighted_energy_totals.loc[
-        nodes, "total agriculture machinery"
-    ]
+    machinery_nodal_energy = (
+        pop_weighted_energy_totals.loc[nodes, "total agriculture machinery"] * 1e6
+    )

    if electric_share > 0:
        efficiency_gain = (
@ -3106,36 +3245,44 @@ def add_agriculture(n, costs):
            suffix=" agriculture machinery electric",
            bus=nodes,
            carrier="agriculture machinery electric",
-            p_set=electric_share
-            / efficiency_gain
-            * machinery_nodal_energy
-            * 1e6
-            / nhours,
+            p_set=electric_share / efficiency_gain * machinery_nodal_energy / nhours,
        )

    if oil_share > 0:
+        p_set = (
+            oil_share
+            * machinery_nodal_energy.rename(lambda x: x + " agriculture machinery oil")
+            / nhours
+        )
+
+        if not options["regional_oil_demand"]:
+            p_set = p_set.sum()
+
+        n.madd(
+            "Bus",
+            spatial.oil.agriculture_machinery,
+            location=spatial.oil.demand_locations,
+            carrier="agriculture machinery oil",
+            unit="MWh_LHV",
+        )
+
        n.madd(
            "Load",
-            ["agriculture machinery oil"],
-            bus=spatial.oil.nodes,
+            spatial.oil.agriculture_machinery,
+            bus=spatial.oil.agriculture_machinery,
            carrier="agriculture machinery oil",
-            p_set=oil_share * machinery_nodal_energy.sum() * 1e6 / nhours,
+            p_set=p_set,
        )

-        co2 = (
-            oil_share
-            * machinery_nodal_energy.sum()
-            * 1e6
-            / nhours
-            * costs.at["oil", "CO2 intensity"]
-        )
-
-        n.add(
-            "Load",
-            "agriculture machinery oil emissions",
-            bus="co2 atmosphere",
-            carrier="agriculture machinery oil emissions",
-            p_set=-co2,
+        n.madd(
+            "Link",
+            spatial.oil.agriculture_machinery,
+            bus0=spatial.oil.nodes,
+            bus1=spatial.oil.agriculture_machinery,
+            bus2="co2 atmosphere",
+            carrier="agriculture machinery oil",
+            p_nom_extendable=True,
+            efficiency2=costs.at["oil", "CO2 intensity"],
        )


@ -3158,7 +3305,8 @@ def remove_h2_network(n):

 def maybe_adjust_costs_and_potentials(n, opts):
    for o in opts:
-        if "+" not in o:
+        flags = ["+e", "+p", "+m", "+c"]
+        if all(flag not in o for flag in flags):
            continue
        oo = o.split("+")
        carrier_list = np.hstack(
@ -3172,7 +3320,12 @@ def maybe_adjust_costs_and_potentials(n, opts):
        suptechs = map(lambda c: c.split("-", 2)[0], carrier_list)
        if oo[0].startswith(tuple(suptechs)):
            carrier = oo[0]
-            attr_lookup = {"p": "p_nom_max", "e": "e_nom_max", "c": "capital_cost"}
+            attr_lookup = {
+                "p": "p_nom_max",
+                "e": "e_nom_max",
+                "c": "capital_cost",
+                "m": "marginal_cost",
+            }
            attr = attr_lookup[oo[1][0]]
            factor = float(oo[1][1:])
            # beware if factor is 0 and p_nom_max is np.inf, 0*np.inf is nan
@ -3204,24 +3357,24 @@ def limit_individual_line_extension(n, maxext):


 aggregate_dict = {
-    "p_nom": "sum",
-    "s_nom": "sum",
+    "p_nom": pd.Series.sum,
+    "s_nom": pd.Series.sum,
    "v_nom": "max",
    "v_mag_pu_max": "min",
    "v_mag_pu_min": "max",
-    "p_nom_max": "sum",
-    "s_nom_max": "sum",
-    "p_nom_min": "sum",
-    "s_nom_min": "sum",
+    "p_nom_max": pd.Series.sum,
+    "s_nom_max": pd.Series.sum,
+    "p_nom_min": pd.Series.sum,
+    "s_nom_min": pd.Series.sum,
    "v_ang_min": "max",
    "v_ang_max": "min",
    "terrain_factor": "mean",
    "num_parallel": "sum",
    "p_set": "sum",
    "e_initial": "sum",
-    "e_nom": "sum",
-    "e_nom_max": "sum",
-    "e_nom_min": "sum",
+    "e_nom": pd.Series.sum,
+    "e_nom_max": pd.Series.sum,
+    "e_nom_min": pd.Series.sum,
    "state_of_charge_initial": "sum",
    "state_of_charge_set": "sum",
    "inflow": "sum",
@ -3283,13 +3436,11 @@ def cluster_heat_buses(n):
        pnl = c.pnl
        agg = define_clustering(pd.Index(pnl.keys()), aggregate_dict)
        for k in pnl.keys():
-            pnl[k].rename(
-                columns=lambda x: x.replace("residential ", "").replace(
-                    "services ", ""
-                ),
-                inplace=True,
-            )
-            pnl[k] = pnl[k].groupby(level=0, axis=1).agg(agg[k], **agg_group_kwargs)
+
+            def renamer(s):
+                return s.replace("residential ", "").replace("services ", "")
+
+            pnl[k] = pnl[k].T.groupby(renamer).agg(agg[k], **agg_group_kwargs).T

        # remove unclustered assets of service/residential
        to_drop = c.df.index.difference(df.index)
@ -3353,6 +3504,7 @@ def apply_time_segmentation(
    sn_weightings = pd.Series(
        weightings, index=snapshots, name="weightings", dtype="float64"
    )
+    logger.info(f"Distribution of snapshot durations:\n{weightings.value_counts()}")

    n.set_snapshots(sn_weightings.index)
    n.snapshot_weightings = n.snapshot_weightings.mul(sn_weightings, axis=0)
@ -3497,7 +3649,7 @@ if __name__ == "__main__":
        for carrier in conventional:
            add_carrier_buses(n, carrier)

-    add_co2_tracking(n, options)
+    add_co2_tracking(n, costs, options)

    add_generation(n, costs)

@ -3505,12 +3657,6 @@ if __name__ == "__main__":

    # TODO merge with opts cost adjustment below
    for o in opts:
-        if o[:4] == "wave":
-            wave_cost_factor = float(o[4:].replace("p", ".").replace("m", "-"))
-            logger.info(
-                f"Including wave generators with cost factor of {wave_cost_factor}"
-            )
-            add_wave(n, wave_cost_factor)
        if o[:4] == "dist":
            options["electricity_distribution_grid"] = True
            options["electricity_distribution_grid_cost_factor"] = float(
@ -3546,7 +3692,7 @@ if __name__ == "__main__":
    if "I" in opts:
        add_industry(n, costs)

-    if "I" in opts and "H" in opts:
+    if "H" in opts:
        add_waste_heat(n)

    if "A" in opts:  # requires H and I
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@ -37,7 +37,7 @@ import pandas as pd
 import pypsa
 import xarray as xr
 from _benchmark import memory_logger
-from _helpers import configure_logging, update_config_with_sector_opts
+from _helpers import configure_logging, get_opt, update_config_with_sector_opts
 from pypsa.descriptors import get_activity_mask

 logger = logging.getLogger(__name__)
@ -182,9 +182,6 @@ def add_co2_sequestration_limit(n, config, limit=200):
    """
    Add a global constraint on the amount of Mt CO2 that can be sequestered.
    """
-    n.carriers.loc["co2 stored", "co2_absorptions"] = -1
-    n.carriers.co2_absorptions = n.carriers.co2_absorptions.fillna(0)
-
    limit = limit * 1e6
    for o in opts:
        if "seq" not in o:
@ -202,10 +199,10 @@ def add_co2_sequestration_limit(n, config, limit=200):
    n.madd(
        "GlobalConstraint",
        names,
-        sense="<=",
-        constant=limit,
-        type="primary_energy",
-        carrier_attribute="co2_absorptions",
+        sense=">=",
+        constant=-limit,
+        type="operational_limit",
+        carrier_attribute="co2 sequestered",
        investment_period=periods,
    )

@ -396,7 +393,7 @@ def prepare_network(
        if snakemake.params["sector"]["limit_max_growth"]["enable"]:
            n = add_max_growth(n, config)

-    if n.stores.carrier.eq("co2 stored").any():
+    if n.stores.carrier.eq("co2 sequestered").any():
        limit = co2_sequestration_potential
        add_co2_sequestration_limit(n, config, limit=limit)

@ -809,18 +806,30 @@ def extra_functionality(n, snapshots):
    """
    opts = n.opts
    config = n.config
-    if "BAU" in opts and n.generators.p_nom_extendable.any():
+    constraints = config["solving"].get("constraints", {})
+    if (
+        "BAU" in opts or constraints.get("BAU", False)
+    ) and n.generators.p_nom_extendable.any():
        add_BAU_constraints(n, config)
-    if "SAFE" in opts and n.generators.p_nom_extendable.any():
+    if (
+        "SAFE" in opts or constraints.get("SAFE", False)
+    ) and n.generators.p_nom_extendable.any():
        add_SAFE_constraints(n, config)
-    if "CCL" in opts and n.generators.p_nom_extendable.any():
+    if (
+        "CCL" in opts or constraints.get("CCL", False)
+    ) and n.generators.p_nom_extendable.any():
        add_CCL_constraints(n, config)
+
    reserve = config["electricity"].get("operational_reserve", {})
    if reserve.get("activate"):
        add_operational_reserve_margin(n, snapshots, config)
-    for o in opts:
-        if "EQ" in o:
-            add_EQ_constraints(n, o)
+
+    EQ_config = constraints.get("EQ", False)
+    EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)")
+    EQ_o = EQ_wildcard or EQ_config
+    if EQ_o:
+        add_EQ_constraints(n, EQ_o.replace("EQ", ""))
+
    add_battery_constraints(n)
    add_lossy_bidirectional_link_constraints(n)
    add_pipe_retrofit_constraint(n)
@ -835,7 +844,8 @@ def extra_functionality(n, snapshots):
        sys.path.append(os.path.dirname(source_path))
        module_name = os.path.splitext(os.path.basename(source_path))[0]
        module = importlib.import_module(module_name)
-        module.custom_extra_functionality(n, snapshots)
+        custom_extra_functionality = getattr(module, module_name)
+        custom_extra_functionality(n, snapshots, snakemake)


 def solve_network(n, config, solving, opts="", **kwargs):
@ -854,6 +864,9 @@ def solve_network(n, config, solving, opts="", **kwargs):
    )
    kwargs["assign_all_duals"] = cf_solving.get("assign_all_duals", False)

+    if kwargs["solver_name"] == "gurobi":
+        logging.getLogger("gurobipy").setLevel(logging.CRITICAL)
+
    rolling_horizon = cf_solving.pop("rolling_horizon", False)
    skip_iterations = cf_solving.pop("skip_iterations", False)
    if not n.lines.s_nom_extendable.any():