enhance documentation, with instructions for sector-coupled / electricity-only studies

2023-03-15 17:00:06 +01:00 · 2023-03-15 17:00:06 +01:00 · 55395c5465
commit 55395c5465
parent e0f8952218
46 changed files with 1471 additions and 1114 deletions
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@ -7,7 +7,7 @@ Closes # (if applicable).

 - [ ] I tested my contribution locally and it seems to work fine.
 - [ ] Code and workflow changes are sufficiently documented.
- [ ] Newly introduced dependencies are added to `envs/environment.yaml` and `envs/environment.docs.yaml`.
- [ ] Changes in configuration options are added in all of `config.default.yaml`, `config.tutorial.yaml`, and `test/config.test1.yaml`.
- [ ] Changes in configuration options are also documented in `doc/configtables/*.csv` and line references are adjusted in `doc/configuration.rst` and `doc/tutorial.rst`.
- [ ] A note for the release notes `doc/release_notes.rst` is amended in the format of previous release notes.
+- [ ] Changed dependencies are added to `envs/environment.yaml`.
+- [ ] Changes in configuration options are added in all of `config.default.yaml`.
+- [ ] Changes in configuration options are also documented in `doc/configtables/*.csv`.
+- [ ] A release note `doc/release_notes.rst` is added.
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@ -103,7 +103,7 @@ jobs:
      run: |
        conda activate pypsa-eur
        conda list
-        snakemake -call solve_all_elec_networks --configfile test/config.test1.yaml --rerun-triggers=mtime
+        snakemake -call solve_elec_networks --configfile test/config.electricity.yaml --rerun-triggers=mtime
        snakemake -call all --configfile test/config.overnight.yaml --rerun-triggers=mtime
        snakemake -call all --configfile test/config.myopic.yaml --rerun-triggers=mtime

--- a/3
+++ b/3
@ -67,10 +67,11 @@ if config["foresight"] == "myopic":

 rule purge:
    message:
-        "Purging generated resources and results. Downloads are kept."
+        "Purging generated resources, results and docs. Downloads are kept."
    run:
        rmtree("resources/", ignore_errors=True)
        rmtree("results/", ignore_errors=True)
+        rmtree("doc/_build", ignore_errors=True)


 rule dag:
--- a/config.default.yaml
+++ b/config.default.yaml
@ -81,13 +81,13 @@ enable:
 # this is over-ridden if CO2Lx is set in sector_opts
 # this is also over-ridden if cb is set in sector_opts
 co2_budget:
-  2020: 0.7011648746
-  2025: 0.5241935484
-  2030: 0.2970430108
-  2035: 0.1500896057
-  2040: 0.0712365591
-  2045: 0.0322580645
-  2050: 0
+  2020: 0.701
+  2025: 0.524
+  2030: 0.297
+  2035: 0.150
+  2040: 0.071
+  2045: 0.032
+  2050: 0.000

 electricity:
  voltages: [220., 300., 380.]
@ -175,7 +175,7 @@ renewable:
    # acceptance issues.
    # correction_factor: 0.93
    corine:
-      # Scholz, Y. (2012). Renewable energy based electricity supply at low costs:
+      # Scholz, Y. (2012). Renewable energy based electricity supply at low costs
      #  development of the REMix model and application for Europe. ( p.42 / p.28)
      grid_codes: [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32]
      distance: 1000
@ -236,7 +236,7 @@ renewable:
    # Correction factor determined by comparing uncorrected area-weighted full-load hours to those
    # published in Supplementary Data to
    # Pietzcker, Robert Carl, et al. "Using the sun to decarbonize the power
-    # sector: The economic potential of photovoltaics and concentrating solar
+    # sector -- The economic potential of photovoltaics and concentrating solar
    # power." Applied Energy 135 (2014): 704-720.
    # This correction factor of 0.854337 may be in order if using reanalysis data.
    # for discussion refer to https://github.com/PyPSA/pypsa-eur/pull/304
--- a/config.tutorial.yaml
+++ b/config.tutorial.yaml
@ -1,318 +0,0 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
-#
-# SPDX-License-Identifier: CC0-1.0
-
-version: 0.7.0
-tutorial: true
-
-logging:
-  level: INFO
-  format: '%(levelname)s:%(name)s:%(message)s'
-
-run:
-  name: ""
-  shared_cutouts: false
-
-scenario:
-  simpl: ['']
-  ll: ['copt']
-  clusters: [5]
-  opts: [Co2L-24H]
-
-countries: ['BE']
-
-snapshots:
-  start: "2013-03-01"
-  end: "2013-04-01"
-  inclusive: 'left' # include start, not end
-
-enable:
-  prepare_links_p_nom: false
-  retrieve_databundle: true
-  retrieve_cost_data: true
-  build_cutout: false
-  retrieve_cutout: true
-  build_natura_raster: false
-  retrieve_natura_raster: true
-  custom_busmap: false
-
-electricity:
-  voltages: [220., 300., 380.]
-  co2limit: 100.e+6
-
-  extendable_carriers:
-    Generator: [OCGT]
-    StorageUnit: [] #battery, H2
-    Store: [battery, H2]
-    Link: [] # H2 pipeline
-
-  max_hours:
-    battery: 6
-    H2: 168
-
-  # use pandas query strings here, e.g. Country not in ['Germany']
-  powerplants_filter: (DateOut >= 2022 or DateOut != DateOut)
-  # use pandas query strings here, e.g. Country in ['Germany']
-  custom_powerplants: false
-
-  conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]
-  renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro]
-
-  estimate_renewable_capacities:
-    enable: true
-    # Add capacities from OPSD data
-    from_opsd: true
-    # Renewable capacities are based on existing capacities reported by IRENA
-    year: 2020
-    # Artificially limit maximum capacities to factor * (IRENA capacities),
-    # i.e. 110% of <years>'s capacities => expansion_limit: 1.1
-    # false: Use estimated renewable potentials determine by the workflow
-    expansion_limit: false
-    technology_mapping:
-      # Wind is the Fueltype in powerplantmatching, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur
-      Offshore: [offwind-ac, offwind-dc]
-      Onshore: [onwind]
-      PV: [solar]
-
-atlite:
-  nprocesses: 4
-  show_progress: false # false saves time
-  cutouts:
-    be-03-2013-era5:
-      module: era5
-      x: [4., 15.]
-      y: [46., 56.]
-      time: ["2013-03", "2013-03"]
-
-renewable:
-  onwind:
-    cutout: be-03-2013-era5
-    resource:
-      method: wind
-      turbine: Vestas_V112_3MW
-    capacity_per_sqkm: 3 # ScholzPhd Tab 4.3.1: 10MW/km^2
-    # correction_factor: 0.93
-    corine:
-      # Scholz, Y. (2012). Renewable energy based electricity supply at low costs:
-      #  development of the REMix model and application for Europe. ( p.42 / p.28)
-      grid_codes: [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32]
-      distance: 1000
-      distance_grid_codes: [1, 2, 3, 4, 5, 6]
-    natura: true
-    excluder_resolution: 200
-    potential: simple # or conservative
-    clip_p_max_pu: 1.e-2
-  offwind-ac:
-    cutout: be-03-2013-era5
-    resource:
-      method: wind
-      turbine: NREL_ReferenceTurbine_5MW_offshore
-    capacity_per_sqkm: 3
-    # correction_factor: 0.93
-    corine: [44, 255]
-    natura: true
-    ship_threshold: 400
-    max_shore_distance: 30000
-    excluder_resolution: 200
-    potential: simple # or conservative
-    clip_p_max_pu: 1.e-2
-  offwind-dc:
-    cutout: be-03-2013-era5
-    resource:
-      method: wind
-      turbine: NREL_ReferenceTurbine_5MW_offshore
-    # ScholzPhd Tab 4.3.1: 10MW/km^2
-    capacity_per_sqkm: 3
-    # correction_factor: 0.93
-    corine: [44, 255]
-    natura: true
-    ship_threshold: 400
-    min_shore_distance: 30000
-    excluder_resolution: 200
-    potential: simple # or conservative
-    clip_p_max_pu: 1.e-2
-  solar:
-    cutout: be-03-2013-era5
-    resource:
-      method: pv
-      panel: CSi
-      orientation:
-        slope: 35.
-        azimuth: 180.
-    capacity_per_sqkm: 1.7 # ScholzPhd Tab 4.3.1: 170 MW/km^2
-    # Correction factor determined by comparing uncorrected area-weighted full-load hours to those
-    # published in Supplementary Data to
-    # Pietzcker, Robert Carl, et al. "Using the sun to decarbonize the power
-    # sector: The economic potential of photovoltaics and concentrating solar
-    # power." Applied Energy 135 (2014): 704-720.
-    # This correction factor of 0.854337 may be in order if using reanalysis data.
-    # correction_factor: 0.854337
-    corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32]
-    natura: true
-    excluder_resolution: 200
-    potential: simple # or conservative
-    clip_p_max_pu: 1.e-2
-
-lines:
-  types:
-    220.: "Al/St 240/40 2-bundle 220.0"
-    300.: "Al/St 240/40 3-bundle 300.0"
-    380.: "Al/St 240/40 4-bundle 380.0"
-  s_max_pu: 0.7
-  s_nom_max: .inf
-  length_factor: 1.25
-  under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
-
-links:
-  p_max_pu: 1.0
-  p_nom_max: .inf
-  include_tyndp: true
-  under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
-
-transformers:
-  x: 0.1
-  s_nom: 2000.
-  type: ''
-
-load:
-  power_statistics: true # only for files from <2019; set false in order to get ENTSOE transparency data
-  interpolate_limit: 3 # data gaps up until this size are interpolated linearly
-  time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from
-  manual_adjustments: true # false
-  scaling_factor: 1.0
-
-costs:
-  year: 2030
-  version: v0.5.0
-  rooftop_share: 0.14
-  fill_values:
-    FOM: 0
-    VOM: 0
-    efficiency: 1
-    fuel: 0
-    investment: 0
-    lifetime: 25
-    "CO2 intensity": 0
-    "discount rate": 0.07
-  marginal_cost:
-    solar: 0.01
-    onwind: 0.015
-    offwind: 0.015
-    H2: 0.
-    battery: 0.
-  emission_prices: # in currency per tonne emission, only used with the option Ep
-    co2: 0.
-
-clustering:
-  simplify_network:
-    to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections)
-    algorithm: kmeans # choose from: [hac, kmeans]
-    feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc.
-    exclude_carriers: []
-  cluster_network:
-    algorithm: kmeans
-    feature: solar+onwind-time
-    exclude_carriers: []
-  aggregation_strategies:
-    generators:
-      p_nom_max: sum # use "min" for more conservative assumptions
-      p_nom_min: sum
-      p_min_pu: mean
-      marginal_cost: mean
-      committable: any
-      ramp_limit_up: max
-      ramp_limit_down: max
-      efficiency: mean
-
-solving:
-  options:
-    formulation: kirchhoff
-    load_shedding: false
-    noisy_costs: true
-    min_iterations: 1
-    max_iterations: 1
-    clip_p_max_pu: 0.01
-    skip_iterations: false
-    track_iterations: false
-  solver:
-    name: cbc
-
-plotting:
-  map:
-    figsize: [7, 7]
-    boundaries: [-10.2, 29, 35, 72]
-    p_nom:
-      bus_size_factor: 5.e+4
-      linewidth_factor: 3.e+3
-
-  costs_max: 800
-  costs_threshold: 1
-
-  energy_max: 15000.
-  energy_min: -10000.
-  energy_threshold: 50.
-
-  vre_techs: ["onwind", "offwind-ac", "offwind-dc", "solar", "ror"]
-  conv_techs: ["OCGT", "CCGT", "Nuclear", "Coal"]
-  storage_techs: ["hydro+PHS", "battery", "H2"]
-  load_carriers: ["AC load"]
-  AC_carriers: ["AC line", "AC transformer"]
-  link_carriers: ["DC line", "Converter AC-DC"]
-  tech_colors:
-    "onwind": "#235ebc"
-    "onshore wind": "#235ebc"
-    'offwind': "#6895dd"
-    'offwind-ac': "#6895dd"
-    'offshore wind': "#6895dd"
-    'offshore wind ac': "#6895dd"
-    'offwind-dc': "#74c6f2"
-    'offshore wind dc': "#74c6f2"
-    "hydro": "#08ad97"
-    "hydro+PHS": "#08ad97"
-    "PHS": "#08ad97"
-    "hydro reservoir": "#08ad97"
-    'hydroelectricity': '#08ad97'
-    "ror": "#4adbc8"
-    "run of river": "#4adbc8"
-    'solar': "#f9d002"
-    'solar PV': "#f9d002"
-    'solar thermal': '#ffef60'
-    'biomass': '#0c6013'
-    'solid biomass': '#06540d'
-    'biogas': '#23932d'
-    'waste': '#68896b'
-    'geothermal': '#ba91b1'
-    "OCGT": "#d35050"
-    "gas": "#d35050"
-    "natural gas": "#d35050"
-    "CCGT": "#b20101"
-    "nuclear": "#ff9000"
-    "coal": "#707070"
-    "lignite": "#9e5a01"
-    "oil": "#262626"
-    "H2": "#ea048a"
-    "hydrogen storage": "#ea048a"
-    "battery": "#b8ea04"
-    "Electric load": "#f9d002"
-    "electricity": "#f9d002"
-    "lines": "#70af1d"
-    "transmission lines": "#70af1d"
-    "AC-AC": "#70af1d"
-    "AC line": "#70af1d"
-    "links": "#8a1caf"
-    "HVDC links": "#8a1caf"
-    "DC-DC": "#8a1caf"
-    "DC link": "#8a1caf"
-  nice_names:
-    OCGT: "Open-Cycle Gas"
-    CCGT: "Combined-Cycle Gas"
-    offwind-ac: "Offshore Wind (AC)"
-    offwind-dc: "Offshore Wind (DC)"
-    onwind: "Onshore Wind"
-    solar: "Solar"
-    PHS: "Pumped Hydro Storage"
-    hydro: "Reservoir & Dam"
-    battery: "Battery Storage"
-    H2: "Hydrogen Storage"
-    lines: "Transmission Lines"
-    ror: "Run of River"
--- a/doc/configtables/atlite.csv
+++ b/doc/configtables/atlite.csv
@ -1,4 +1,5 @@
 ,Unit,Values,Description
+default_cutout,--,str,"Defines a default cutout."
 nprocesses,--,int,"Number of parallel processes in cutout preparation"
 show_progress,bool,true/false,"Whether progressbar for atlite conversion processes should be shown. False saves time."
 cutouts,,,
--- a/doc/configtables/enable.csv
+++ b/doc/configtables/enable.csv
@ -0,0 +1,9 @@
+,Unit,Values,Description
+prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia <https://en.wikipedia.org/wiki/List_of_HVDC_projects>`_"
+retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder."
+retrieve_cost_data,bool,"{true, false}","Switch to retrieve technology cost data from `technology-data repository <https://github.com/PyPSA/technology-data>`_."
+build_cutout,bool,"{true, false}","Switch to enable the building of cutouts via the rule :mod:`build_cutout`."
+retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts from zenodo with :mod:`retrieve_cutout`."
+build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`."
+retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`."
+custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``."
--- a/doc/configtables/licenses-sector.csv
+++ b/doc/configtables/licenses-sector.csv
--- a/doc/configtables/plotting.csv
+++ b/doc/configtables/plotting.csv
@ -1,10 +1,6 @@
 ,Unit,Values,Description
 map,,,
-- figsize,--,"[width, height]; e.g. [7,7]","Figure size in inches."
 -- boundaries,°,"[x1,x2,y1,y2]","Boundaries of the map plots in degrees latitude (y) and longitude (x)"
-- p_nom,,,
-- -- bus_size_factor,--,float,"Factor by which values determining bus sizes are scaled to fit well in the plot."
-- -- linewidth_factor,--,float,"Factor by which values determining bus sizes are scaled to fit well in the plot."
 costs_max,bn Euro,float,"Upper y-axis limit in cost bar plots."
 costs_threshold,bn Euro,float,"Threshold below which technologies will not be shown in cost bar plots."
 energy_max,TWh,float,"Upper y-axis limit in energy bar plots."
--- a/doc/configtables/sector-opts.csv
+++ b/doc/configtables/sector-opts.csv
@ -0,0 +1,4 @@
+Trigger, Description, Definition, Status
+``nH``; i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L110>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L146>`__), In active use
+``Co2L``, Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L19>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L154>`__, In active use
+``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use
--- a/doc/configtables/solving-options.csv
+++ b/doc/configtables/solving-options.csv
@ -1,10 +0,0 @@
-,Unit,Values,Description
-formulation,--,"Any of {'angles', 'kirchhoff', 'cycles', 'ptdf'}","Specifies which variant of linearized power flow formulations to use in the optimisation problem. Recommended is 'kirchhoff'. Explained in `this article <https://arxiv.org/abs/1704.01881>`_."
-load_shedding,bool,"{'true','false'}","Add generators with a prohibitively high marginal cost to simulate load shedding and avoid problem infeasibilities."
-noisy_costs,bool,"{'true','false'}","Add random noise to marginal cost of generators by :math:`\mathcal{U}(0.009,0,011)` and capital cost of lines and links by :math:`\mathcal{U}(0.09,0,11)`."
-min_iterations,--,int,"Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run."
-max_iterations,--,int,"Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run."
-nhours,--,int,"Specifies the :math:`n` first snapshots to take into account. Must be less than the total number of snapshots. Rather recommended only for debugging."
-clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero."
-skip_iterations,bool,"{'true','false'}","Skip iterating, do not update impedances of branches. Defaults to true."
-track_iterations,bool,"{'true','false'}","Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration)"
--- a/doc/configtables/solving-solver.csv
+++ b/doc/configtables/solving-solver.csv
@ -1,3 +0,0 @@
-,Unit,Values,Description
-name,--,"One of {'gurobi', 'cplex', 'cbc', 'glpk', 'ipopt'}; potentially more possible","Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow."
-opts,--,"Parameter list for `Gurobi <https://www.gurobi.com/documentation/8.1/refman/parameters.html>`_ and `CPLEX <https://www.ibm.com/docs/en/icos/20.1.0?topic=cplex-topical-list-parameters>`_","Solver specific parameter settings."
--- a/doc/configtables/solving.csv
+++ b/doc/configtables/solving.csv
@ -0,0 +1,16 @@
+,Unit,Values,Description
+options,,,
+-- formulation,--,"Any of {'angles', 'kirchhoff', 'cycles', 'ptdf'}","Specifies which variant of linearized power flow formulations to use in the optimisation problem. Recommended is 'kirchhoff'. Explained in `this article <https://arxiv.org/abs/1704.01881>`_."
+-- load_shedding,bool,"{'true','false'}","Add generators with a prohibitively high marginal cost to simulate load shedding and avoid problem infeasibilities."
+-- noisy_costs,bool,"{'true','false'}","Add random noise to marginal cost of generators by :math:`\mathcal{U}(0.009,0,011)` and capital cost of lines and links by :math:`\mathcal{U}(0.09,0,11)`."
+-- min_iterations,--,int,"Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run."
+-- max_iterations,--,int,"Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run."
+-- nhours,--,int,"Specifies the :math:`n` first snapshots to take into account. Must be less than the total number of snapshots. Rather recommended only for debugging."
+-- clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero."
+-- skip_iterations,bool,"{'true','false'}","Skip iterating, do not update impedances of branches. Defaults to true."
+-- track_iterations,bool,"{'true','false'}","Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration)"
+solver,,,
+-- name,--,"One of {'gurobi', 'cplex', 'cbc', 'glpk', 'ipopt'}; potentially more possible","Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow."
+-- options,--,"Key listed under ``solver_options``.","Link to specific parameter settings."
+solver_options,,"dict","Dictionaries with solver-specific parameter settings."
+mem,MB,"int","Estimated maximum memory requirement for solving networks."
--- a/doc/configtables/toplevel.csv
+++ b/doc/configtables/toplevel.csv
@ -9,12 +9,3 @@ run,,,
 -- shared_cutouts,bool,"{true, false}","Switch to select whether cutouts should be shared across runs."
 countries,--,"Subset of {'AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'}","European countries defined by their `Two-letter country codes (ISO 3166-1) <https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2>`_ which should be included in the energy system model."
 focus_weights,--,"Keys should be two-digit country codes (e.g. DE) and values should range between 0 and 1","Ratio of total clusters for particular countries. the remaining weight is distributed according to mean load. An example: ``focus_weights: 'DE': 0.6 'FR': 0.2``."
-enable,,,
-- prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia <https://en.wikipedia.org/wiki/List_of_HVDC_projects>`_"
-- retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder."
-- retrieve_cost_data,bool,"{true, false}","Switch to retrieve technology cost data from `technology-data repository <https://github.com/PyPSA/technology-data>`_."
-- build_cutout,bool,"{true, false}","Switch to enable the building of cutouts via the rule :mod:`build_cutout`."
-- retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts from zenodo with :mod:`retrieve_cutout`."
-- build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`."
-- retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`."
-- custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``."
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@ -18,7 +18,7 @@ Top-level configuration

 .. literalinclude:: ../config.default.yaml
   :language: yaml
-   :lines: 5-11,23,30-38
+   :lines: 5-11,18-19,62,80-90


 .. csv-table::
@ -40,7 +40,7 @@ The ``run`` section is used for running and storing scenarios with different con
 .. literalinclude:: ../config.default.yaml
   :language: yaml
   :start-at: run:
-   :end-before: scenario:
+   :end-before: foresight:


 ``scenario``
@ -52,9 +52,21 @@ facilitate running multiple scenarios through a single command

 .. code:: bash

-    snakemake -call solve_all_networks
+   # for electricity-only studies
+   snakemake -call solve_elec_networks

-For each wildcard, a **list of values** is provided. The rule ``solve_all_networks`` will trigger the rules for creating ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc`` for **all combinations** of the provided wildcard values as defined by Python's `itertools.product(...) <https://docs.python.org/2/library/itertools.html#itertools.product>`_ function that snakemake's `expand(...) function <https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#targets>`_ uses.
+   # for sector-coupling studies
+   snakemake -call solve_all_sector_networks
+
+For each wildcard, a **list of values** is provided. The rule
+``solve_all_elec_networks`` will trigger the rules for creating
+``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc`` for **all
+combinations** of the provided wildcard values as defined by Python's
+`itertools.product(...)
+<https://docs.python.org/2/library/itertools.html#itertools.product>`_ function
+that snakemake's `expand(...) function
+<https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#targets>`_
+uses.

 An exemplary dependency graph (starting from the simplification rules) then looks like this:

@ -87,6 +99,18 @@ Specifies the temporal range to build an energy system model for as arguments to
   :widths: 25,7,22,30
   :file: configtables/snapshots.csv

+.. _enable_cf:
+
+``enable``
+==========
+
+Switches for some rules and optional features.
+
+.. literalinclude:: ../config.default.yaml
+   :language: yaml
+   :start-at: enable:
+   :end-before: co2_budget:
+
 .. _electricity_cf:

 ``electricity``
@ -194,7 +218,12 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
 ``conventional``
 ================

-Define additional generator attribute for conventional carrier types. If a scalar value is given it is applied to all generators. However if a string starting with "data/" is given, the value is interpreted as a path to a csv file with country specific values. Then, the values are read in and applied to all generators of the given carrier in the given country. Note that the value(s) overwrite the existing values in the corresponding section of the ``generators`` dataframe.
+Define additional generator attribute for conventional carrier types. If a
+scalar value is given it is applied to all generators. However if a string
+starting with "data/" is given, the value is interpreted as a path to a csv file
+with country specific values. Then, the values are read in and applied to all
+generators of the given carrier in the given country. Note that the value(s)
+overwrite the existing values.

 .. literalinclude:: ../config.default.yaml
   :language: yaml
@ -252,8 +281,8 @@ Define additional generator attribute for conventional carrier types. If a scala

 .. literalinclude:: ../config.default.yaml
   :language: yaml
-   :start-at: load:
-   :end-before: costs:
+   :start-after:   type:
+   :end-at:   scaling_factor:

 .. csv-table::
   :header-rows: 1
@ -267,7 +296,7 @@ Define additional generator attribute for conventional carrier types. If a scala

 .. literalinclude:: ../config.default.yaml
   :language: yaml
-   :start-after: scaling_factor:
+   :start-at: costs:
   :end-before: clustering:

 .. csv-table::
@ -275,10 +304,6 @@ Define additional generator attribute for conventional carrier types. If a scala
   :widths: 25,7,22,30
   :file: configtables/costs.csv

-.. note::
-    To change cost assumptions in more detail (i.e. other than ``marginal_cost`` and ``capital_cost``), consider modifying cost assumptions directly in ``resources/costs.csv`` as this is not yet supported through the config file.
-    You can also build multiple different cost databases. Make a renamed copy of ``resources/costs.csv`` (e.g. ``data/costs-optimistic.csv``) and set the variable ``COSTS=data/costs-optimistic.csv`` in the ``Snakefile``.
-

 .. _clustering_cf:

@ -287,7 +312,7 @@ Define additional generator attribute for conventional carrier types. If a scala

 .. literalinclude:: ../config.default.yaml
   :language: yaml
-   :start-after:     co2:
+   :start-at: clustering:
   :end-before: solving:

 .. csv-table::
@ -296,31 +321,117 @@ Define additional generator attribute for conventional carrier types. If a scala
   :file: configtables/clustering.csv


+.. _energy_cf:
+
+``energy``
+=======================
+
+.. note::
+   Only used for sector-coupling studies.
+
+.. warning::
+   More comprehensive documentation for this segment will be released soon.
+
+.. literalinclude:: ../config.default.yaml
+   :language: yaml
+   :start-at: energy:
+   :end-before: biomass:
+
+
+.. _biomass_cf:
+
+``biomass``
+=======================
+
+.. note::
+   Only used for sector-coupling studies.
+
+.. warning::
+   More comprehensive documentation for this segment will be released soon.
+
+.. literalinclude:: ../config.default.yaml
+   :language: yaml
+   :start-at: biomass:
+   :end-before: solar_thermal:
+
+.. _solar_thermal_cf:
+
+``solar_thermal``
+=======================
+
+.. note::
+   Only used for sector-coupling studies.
+
+.. warning::
+   More comprehensive documentation for this segment will be released soon.
+
+.. literalinclude:: ../config.default.yaml
+   :language: yaml
+   :start-at: solar_thermal:
+   :end-before: existing_capacities:
+
+.. _existing_capacities_cf:
+
+``existing_capacities``
+=======================
+
+.. note::
+   Only used for sector-coupling studies.
+
+.. warning::
+   More comprehensive documentation for this segment will be released soon.
+
+.. literalinclude:: ../config.default.yaml
+   :language: yaml
+   :start-at: existing_capacities:
+   :end-before: sector:
+
+.. _sector_cf:
+
+``sector``
+=======================
+
+.. note::
+   Only used for sector-coupling studies.
+
+.. warning::
+   More comprehensive documentation for this segment will be released soon.
+
+.. literalinclude:: ../config.default.yaml
+   :language: yaml
+   :start-at: sector:
+   :end-before: industry:
+
+.. _industry_cf:
+
+``industry``
+=======================
+
+.. note::
+   Only used for sector-coupling studies.
+
+.. warning::
+   More comprehensive documentation for this segment will be released soon.
+
+.. literalinclude:: ../config.default.yaml
+   :language: yaml
+   :start-at: industry:
+   :end-before: costs:
+
 .. _solving_cf:

 ``solving``
 =============

-``options``
-----------
-
 .. literalinclude:: ../config.default.yaml
   :language: yaml
   :start-at: solving:
-   :end-before:   solver:
+   :end-before: plotting:

 .. csv-table::
   :header-rows: 1
   :widths: 25,7,22,30
-   :file: configtables/solving-options.csv
-
-``solver``
----------
-
-.. literalinclude:: ../config.default.yaml
-   :language: yaml
-   :start-at:   solver:
-   :end-before: plotting:
+   :file: configtables/solving.csv

 .. csv-table::
   :header-rows: 1
@ -332,6 +443,9 @@ Define additional generator attribute for conventional carrier types. If a scala
 ``plotting``
 =============

+.. warning::
+   More comprehensive documentation for this segment will be released soon.
+
 .. literalinclude:: ../config.default.yaml
   :language: yaml
   :start-at: plotting:
--- a/doc/contributing.rst
+++ b/doc/contributing.rst
@ -21,16 +21,17 @@ For linting, formatting and checking your code contributions
 against our guidelines (e.g. we use `Black <https://github.com/psf/black>`_ as code style
 use `pre-commit <https://pre-commit.com/index.html>`_:

-1. Installation ``conda install -c conda-forge pre-commit`` or ``pip install pre-commit``
+1. Installation ``mamba install -c conda-forge pre-commit`` or ``pip install pre-commit``
 2. Usage:
    * To automatically activate ``pre-commit`` on every ``git commit``: Run ``pre-commit install``
    * To manually run it: ``pre-commit run --all``

-Note that installing `pre-commit` locally is not strictly necessary. If you create a Pull Request the `pre-commit CI` will be triggered automatically and take care of the checks.
+.. note::
+  Note that installing ``pre-commit`` locally is not strictly necessary. If you create a Pull Request the ``pre-commit CI`` will be triggered automatically and take care of the checks.

 For all code contributions we follow the four eyes principle (two person principle), i.e. all suggested code
 including our own are reviewed by a second person before they are incorporated into our repository.

 If you are unfamiliar with pull requests, the GitHub help pages have a nice `guide <https://help.github.com/en/articles/about-pull-requests>`_.

-To ask and answer general usage questions, join the `PyPSA and PyPSA-Eur mailing list <https://groups.google.com/forum/#!forum/pypsa>`_.
+To ask and answer general usage questions, join the `PyPSA mailing list <https://groups.google.com/forum/#!forum/pypsa>`_.
--- a/doc/costs.rst
+++ b/doc/costs.rst
@ -3,18 +3,22 @@

  SPDX-License-Identifier: CC-BY-4.0

-##################
-Cost Assumptions
-##################
+############################
+Techno-Economic Assumptions
+############################

 The database of cost assumptions is retrieved from the repository
 `PyPSA/technology-data <https://github.com/pypsa/technology-data>`_ and then
-saved to ``resources/costs.csv``.
+saved to a file ``data/costs_{year}.csv``. The ``config.yaml`` provides options
+to choose a reference year and use a specific version of the repository.

-The ``config.yaml`` provides options to choose a reference year (``costs: year:``) and use a specific version of the repository ``costs: version:``.
+.. literalinclude:: ../config.default.yaml
+   :language: yaml
+   :start-at: costs:
+   :end-at:   version:

-It includes cost assumptions for all included technologies for specific
-years from various sources, namely for
+The file includes cost assumptions for all included technologies for specific
+years compiled from various sources, namely for

 - discount rate,
 - lifetime,
@ -25,6 +29,10 @@ years from various sources, namely for
 - efficiency, and
 - carbon-dioxide intensity.

+Many values are taken from a database published by the Danish Energy Agency (`DEA
+<https://ens.dk/en/our-services/projections-and-models/technology-data>`_).
+
+
 The given overnight capital costs are annualised to net present costs
 with a discount rate of :math:`r` over the economic lifetime :math:`n` using the annuity factor

@ -32,14 +40,19 @@ with a discount rate of :math:`r` over the economic lifetime :math:`n` using the

    a = \frac{1-(1+r)^{-n}}{r}.

-Based on the parameters above the ``marginal_cost`` and ``capital_cost`` of the system components are calculated.
+Based on the parameters above the ``marginal_cost`` and ``capital_cost`` of the
+system components are automatically calculated.


-Modifying Cost Assumptions
-==========================
+Modifying Assumptions
+=====================

-Some cost assumptions (e.g. marginal cost and capital cost) can be directly overwritten in the ``config.yaml`` (cf. Section  :ref:`costs_cf`  in :ref:`config`).
+Some cost assumptions (e.g. marginal cost and capital cost) can be directly
+set in the ``config.yaml`` (cf. Section  :ref:`costs_cf`  in
+:ref:`config`). To change cost assumptions in more detail, make a copy of
+``data/costs_{year}.csv`` and reference the new cost file in the ``Snakefile``:

-To change cost assumptions in more detail, modify cost assumptions directly in ``resources/costs.csv`` as this is not yet supported through the config file.
+.. literalinclude:: ../Snakefile
+   :start-at: COSTS
+   :end-at: COSTS

-You can also build multiple different cost databases. Make a renamed copy of ``resources/costs.csv`` (e.g. ``data/costs-optimistic.csv``) and set the variable ``COSTS=data/costs-optimistic.csv`` in the ``Snakefile``.
--- a/doc/foresight.rst
+++ b/doc/foresight.rst
@ -0,0 +1,271 @@
+..
+  SPDX-FileCopyrightText: 2021-2023 The PyPSA-Eur Authors
+
+  SPDX-License-Identifier: CC-BY-4.0
+
+.. _foresight:
+
+#####################
+Foresight Options
+#####################
+
+.. _overnight:
+
+Overnight (greenfield) scenarios
+================================
+
+The default is to calculate a rebuilding of the energy system to meet demand, a so-called overnight or greenfield approach.
+
+In this case, the ``planning_horizons`` parameter specifies the reference year for exogenously given transition paths (e.g. the level of steel recycling).
+It does not affect the year for cost and technology assumptions, which is set separately in the config.
+
+.. code:: yaml
+
+  scenario:
+    planning_horizons:
+    - 2050
+
+  costs:
+    year: 2030
+
+For running overnight scenarios, use in the ``config.yaml``:
+
+.. code:: yaml
+
+  foresight: overnight
+
+.. _perfect:
+
+Perfect foresight scenarios
+===========================
+
+.. warning::
+
+  Perfect foresight is currently under development and not yet implemented.
+
+For running perfect foresight scenarios, in future versions you will be able to
+set in the ``config.yaml``:
+
+.. code:: yaml
+
+  foresight: perfect
+
+
+.. _myopic:
+
+Myopic foresight scenarios
+=============================
+
+The myopic code can be used to investigate progressive changes in a network, for
+instance, those taking place throughout a transition path. The capacities
+installed in a certain time step are maintained in the network until their
+operational lifetime expires.
+
+The myopic approach was initially developed and used in the paper `Early
+decarbonisation of the European Energy system pays off (2020)
+<https://www.nature.com/articles/s41467-020-20015-4>`__ and later further
+extended in `Speed of technological transformations required in Europe to
+achieve different climate goals (2022)
+<https://doi.org/10.1016/j.joule.2022.04.016>`__. The current implementation
+complies with the PyPSA-Eur-Sec standard working flow and is compatible with
+using the higher resolution electricity transmission model `PyPSA-Eur
+<https://github.com/PyPSA/pypsa-eur>`__ rather than a one-node-per-country
+model.
+
+The current code applies the myopic approach to generators, storage technologies
+and links in the power sector. It furthermore applies it to the space and water
+heating sector (e.g., the share of district heating and reduced space heat
+demand), industry processes (e.g., steel, direct reduced iron, and aluminum
+production via primary route), the share of fuel cell and battery electric
+vehicles in land transport, and the hydrogen share in shipping (see
+:doc:`supply_demand` for further information).
+
+The following subjects within the land transport and biomass currently do not
+evolve with the myopic approach:
+
+- The percentage of electric vehicles that allow demand-side management and
+  vehicle-to-grid services.
+
+- The annual biomass potential (default year and scenario for which potential is
+  taken is 2030, defined `here
+  <https://github.com/PyPSA/pypsa-eur-sec/blob/413254e241fb37f55b41caba7264644805ad8e97/config.default.yaml#L109>`_)
+
+
+Configuration
+--------------
+
+For running myopic foresight transition scenarios, set in ``config.yaml``:
+
+.. code:: yaml
+
+  foresight: myopic
+
+The following options included in the config.yaml file  are relevant for the
+myopic code.
+
+The ``{planning_horizons}`` wildcard indicates the year in which the network is
+optimized. For a myopic optimization, this is equivalent to the investment year.
+To set the investment years which are sequentially simulated for the myopic
+investment planning, select for example:
+
+.. literalinclude:: ../test/config.myopic.yaml
+   :language: yaml
+   :start-at:   planning_horizons:
+   :end-before: countries:
+
+
+**existing capacities**
+
+Grouping years indicates the bins limits for grouping the existing capacities of
+different technologies. Note that separate bins are defined for the power and
+heating plants due to different data sources.
+
+``grouping_years_power: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020,
+2025, 2030]``
+
+``grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019]``
+
+
+
+
+
+**threshold capacity**
+
+If for a technology, node, and grouping bin, the capacity is lower than
+threshold_capacity, it is ignored.
+
+``threshold_capacity: 10``
+
+
+
+
+**conventional carriers**
+
+Conventional carriers indicate carriers used in the existing conventional
+technologies.
+
+    conventional_carriers:
+
+    \- lignite
+
+    \- coal
+
+    \- oil
+
+    \- uranium
+
+
+
+
+Options
+--------------
+
+The total carbon budget for the entire transition path can be indicated in the
+`sector_opts
+<https://github.com/PyPSA/pypsa-eur-sec/blob/f13902510010b734c510c38c4cae99356f683058/config.default.yaml#L25>`_
+in ``config.yaml``. The carbon budget can be split among the
+``planning_horizons`` following an exponential or beta decay. E.g. ``'cb40ex0'``
+splits a carbon budget equal to 40 Gt :math:`_{CO_2}` following an exponential
+decay whose initial linear growth rate r is zero. They can also follow some
+user-specified path, if defined `here
+<https://github.com/PyPSA/pypsa-eur-sec/blob/413254e241fb37f55b41caba7264644805ad8e97/config.default.yaml#L56>`_.
+The paper `Speed of technological transformations required in Europe to achieve
+different climate goals (2022) <https://doi.org/10.1016/j.joule.2022.04.016>`__
+defines CO_2 budgets corresponding to global temperature increases (1.5C – 2C)
+as response to the emissions. Here, global carbon budgets are converted to
+European budgets assuming equal-per capita distribution which translates into a
+6.43% share for Europe. The carbon budgets are in this paper distributed
+throughout the transition paths assuming an exponential decay. Emissions e(t) in
+every year t are limited by
+
+.. math::
+  e(t) = e_0 (1+ (r+m)t) e^{-mt}
+
+where r is the initial linear growth rate, which here is assumed to be r=0, and
+the decay parameter m is determined by imposing the integral of the path to be
+equal to the budget for Europe. Following this approach, the CO_2 budget is
+defined. Following the same approach as in this paper, add the following to the
+``scenario.sector_opts`` E.g.  ``-cb25.7ex0`` (1.5C increase) Or ``cb73.9ex0``
+(2C increase). See details in Supplemental Note S1 `Speed of technological
+transformations required in Europe to achieve different climate goals (2022)
+<https://doi.org/10.1016/j.joule.2022.04.016>`__.
+
+
+General myopic code structure
+---------------------------------
+
+The myopic code solves the network for the time steps included in
+``planning_horizons`` in a recursive loop, so that:
+
+1. The existing capacities (those installed before the base year are added as
+   fixed capacities with p_nom=value, p_nom_extendable=False). E.g. for
+   baseyear=2020, capacities installed before 2020 are added. In addition, the
+   network comprises additional generator, storage, and link capacities with
+   p_nom_extendable=True. The non-solved network is saved in
+   ``results/run_name/networks/prenetworks-brownfield``.
+The base year is the first element in ``planning_horizons``. Step 1 is
+implemented with the rule add_baseyear for the base year and with the rule
+add_brownfield for the remaining planning_horizons.
+
+2. The 2020 network is optimized. The solved network is saved in
+   ``results/run_name/networks/postnetworks``
+
+3. For the next planning horizon, e.g. 2030, the capacities from a previous time
+   step are added if they are still in operation (i.e., if they fulfil planning
+   horizon <= commissioned year + lifetime). In addition, the network comprises
+   additional generator, storage, and link capacities with
+   p_nom_extendable=True. The non-solved network is saved in
+   ``results/run_name/networks/prenetworks-brownfield``.
+
+Steps 2 and 3 are solved recursively for all the planning_horizons included in
+``config.yaml``.
+
+Rule overview
+--------------
+
+- rule add_existing baseyear
+
+  The rule add_existing_baseyear loads the network in
+  ‘results/run_name/networks/prenetworks’ and performs the following operations:
+
+  1. Add the conventional, wind and solar power generators that were installed
+     before the base year.
+
+  2. Add the heating capacities that were installed before the base year.
+
+  The existing conventional generators are retrieved from the `powerplants.csv
+  file
+  <https://pypsa-eur.readthedocs.io/en/latest/preparation/build_powerplants.html?highlight=powerplants>`__
+  generated by pypsa-eur which, in turn, is based on the `powerplantmatching
+  <https://github.com/FRESNA/powerplantmatching>`__ database.
+
+  Existing wind and solar capacities are retrieved from `IRENA annual statistics
+  <https://www.irena.org/Statistics/Download-Data>`__ and distributed among the
+  nodes in a country proportional to capacity factor. (This will be updated to
+  include capacity distributions closer to reality.)
+
+  Existing heating capacities are retrieved from the report `Mapping and
+  analyses of the current and future (2020 - 2030) heating/cooling fuel
+  deployment (fossil/renewables)
+  <https://ec.europa.eu/energy/studies/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment_en?redir=1>`__.
+
+  The heating capacities are assumed to have a lifetime indicated by the
+  parameter lifetime in the configuration file, e.g 25 years. They are assumed
+  to be decommissioned linearly starting on the base year, e.g., from 2020 to
+  2045.
+
+  Then, the resulting network is saved in
+  ``results/run_name/networks/prenetworks-brownfield``.
+
+- rule add_brownfield
+
+  The rule add_brownfield loads the network in
+  ``results/run_name/networks/prenetworks`` and performs the following
+  operation:
+
+  1. Read the capacities optimized in the previous time step and add them to the
+     network if they are still in operation (i.e., if they fulfill planning
+     horizon < commissioned year + lifetime)
+
+  Then, the resulting network is saved in
+  ``results/run_name/networks/prenetworks_brownfield``.
--- a/doc/img/intro-workflow.png
+++ b/doc/img/intro-workflow.png
--- a/doc/img/workflow.png
+++ b/doc/img/workflow.png
--- a/doc/index.rst
+++ b/doc/index.rst
@ -3,8 +3,9 @@

  SPDX-License-Identifier: CC-BY-4.0

+##################################################################################
 PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy System
-=================================================================================
+##################################################################################

 .. image:: https://img.shields.io/github/v/release/pypsa/pypsa-eur?include_prereleases
    :alt: GitHub release (latest by date including pre-releases)
@ -22,7 +23,7 @@ PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy Syste
 .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3520874.svg
    :target: https://doi.org/10.5281/zenodo.3520874

-.. image:: https://img.shields.io/badge/snakemake-≥5.0.0-brightgreen.svg?style=flat
+.. image:: https://img.shields.io/badge/snakemake-≥7.19-brightgreen.svg?style=flat
    :target: https://snakemake.readthedocs.io
    :alt: Snakemake

@ -30,22 +31,19 @@ PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy Syste
    :target: https://api.reuse.software/info/github.com/pypsa/pypsa-eur
    :alt: REUSE

+|

 PyPSA-Eur is an open model dataset of the European energy system at the
 transmission network level that covers the full ENTSO-E area.

-It contains alternating current lines at and above 220 kV voltage level and all
-high voltage direct current lines, substations, an open database of conventional
-power plants, time series for electrical demand and variable renewable generator
-availability, and geographic potentials for the expansion of wind and solar
-power.
+Electricity System
+==================

-A sector-coupled extension (previously known as PyPSA-Eur-Sec) adds demand and
-supply for the following sectors: transport, space and water heating, biomass,
-energy consumption in the agriculture, industry and industrial feedstocks,
-carbon management, carbon capture and usage/sequestration. This completes the
-energy system and includes all greenhouse gas emitters except waste management,
-agriculture, forestry and land use.
+The electricity system representation contains alternating current lines at
+and above 220 kV voltage level and all high voltage direct current lines,
+substations, an open database of conventional power plants, time series for
+electrical demand and variable renewable generator availability, geographic
+potentials for the expansion of wind and solar power.

 The model is suitable both for operational studies and generation and
 transmission expansion planning studies. The continental scope and highly
@ -56,15 +54,59 @@ effects for renewable power generation and their varying resource availability.
    :width: 70%
    :align: center

+|
+
+Sector-Coupled Energy System
+============================
+
+A sector-coupled extension (previously known as **PyPSA-Eur-Sec**) adds demand
+and supply for the following sectors: transport, space and water heating,
+biomass, energy consumption in the agriculture, industry and industrial
+feedstocks, carbon management, carbon capture and usage/sequestration. This
+completes the energy system and includes all greenhouse gas emitters except
+waste management, agriculture, forestry and land use. The diagram below gives an
+overview of the sectors and the links between them:
+
+.. image:: ../graphics/multisector_figure.png
+    :width: 70%
+    :align: center
+
+.. note::
+    You can find showcases of the model's capabilities in the Supplementary Materials of the
+    preprint `Benefits of a Hydrogen Network in Europe
+    <https://arxiv.org/abs/2207.05816>`_, the Supplementary Materials of the `paper in Joule with a
+    description of the industry sector
+    <https://arxiv.org/abs/2109.09563>`_, or in `a 2021 presentation
+    at EMP-E <https://nworbmot.org/energy/brown-empe.pdf>`_.
+    The sector-coupled extension of PyPSA-Eur was
+    initially described in the paper `Synergies of sector coupling and transmission
+    reinforcement in a cost-optimised, highly renewable European energy system
+    <https://arxiv.org/abs/1801.05290>`_ (2018) but it differs by being based on the
+    higher resolution electricity transmission model `PyPSA-Eur
+    <https://github.com/PyPSA/pypsa-eur>`_ rather than a one-node-per-country model,
+    and by including biomass, industry, industrial feedstocks, aviation, shipping,
+    better carbon management, carbon capture and usage/sequestration, and gas
+    networks.
+
+About
+=====
+
+PyPSA-Eur is designed to be imported into the open energy system modelling
+framework `PyPSA <https://www.pypsa.org>`_ for which `documentation
+<https://pypsa.readthedocs.io>`_ is available as well.
+
 The restriction to freely available and open data encourages the open exchange
 of model data developments and eases the comparison of model results. It
 provides a full, automated software pipeline to assemble the load-flow-ready
 model from the original datasets, which enables easy replacement and improvement
 of the individual parts.

-PyPSA-Eur is designed to be imported into the open toolbox `PyPSA
-<https://www.pypsa.org>`_ for which `documentation <https://pypsa.org/doc>`_ is
-available as well.
+.. warning::
+    PyPSA-Eur is under active development and has several
+    :doc:`limitations` which
+    you should understand before using the model. The Github repository
+    `issues <https://github.com/PyPSA/pypsa-eur/issues>`_ collect known
+    topics we are working on. Please feel free to help or make suggestions.

 This project is currently maintained by the `Department of Digital
 Transformation in Energy Systems <https:/www.ensys.tu-berlin.de>`_ at the
@ -80,186 +122,38 @@ Ministry for Education and Research (BMBF) <https://www.bmbf.de/en/index.html>`_
 as part of the `Stromnetze Research Initiative
 <http://forschung-stromnetze.info/projekte/grundlagen-und-konzepte-fuer-effiziente-dezentrale-stromnetze/>`_.

-**WARNING**: PyPSA-Eur is under active development and has several
-`limitations <https://pypsa-eur.readthedocs.io/en/latest/limitations.html>`_ which
-you should understand before using the model. The github repository
-`issues <https://github.com/PyPSA/pypsa-eur/issues>`_ collect known
-topics we are working on (please feel free to help or make suggestions).

-.. note::
-    You can find showcases of the model's capabilities in the Supplementary Materials of the
-    preprint `Benefits of a Hydrogen Network in Europe
-    <https://arxiv.org/abs/2207.05816>`_, the Supplementary Materials of the `paper in Joule with a
-    description of the industry sector
-    <https://arxiv.org/abs/2109.09563>`_, or in `a 2021 presentation
-    at EMP-E <https://nworbmot.org/energy/brown-empe.pdf>`_.
-
-
-This diagram gives an overview of the sectors and the links between
-them:
-
-.. image:: ../graphics/multisector_figure.png
-    :width: 70%
-    :align: center
-
-The sector-coupled extension of PyPSA-Eur (previously PyPSA-Eur-Sec) was
-initially described in the paper `Synergies of sector coupling and transmission
-reinforcement in a cost-optimised, highly renewable European energy system
-<https://arxiv.org/abs/1801.05290>`_ (2018) but it differs by being based on the
-higher resolution electricity transmission model `PyPSA-Eur
-<https://github.com/PyPSA/pypsa-eur>`_ rather than a one-node-per-country model,
-and by including biomass, industry, industrial feedstocks, aviation, shipping,
-better carbon management, carbon capture and usage/sequestration, and gas
-networks.
-
-
-
-Workflow Outline
-================
+Workflow
+========

 .. image:: ../graphics/workflow.png
+    :class: full-width
+    :align: center

 .. note::
    The graph above was generated using
    ``snakemake --rulegraph -F | sed -n "/digraph/,/}/p" | dot -Tpng -o workflow.png``


-Documentation
-=============
-
-**Getting Started**
-
-* :doc:`introduction`
-* :doc:`installation`
-* :doc:`tutorial`
-
-.. toctree::
-   :hidden:
-   :maxdepth: 1
-   :caption: Getting Started
-
-   introduction
-   installation
-   tutorial
-
-**Configuration**
-
-* :doc:`wildcards`
-* :doc:`configuration`
-* :doc:`costs`
-
-.. toctree::
-   :hidden:
-   :maxdepth: 1
-   :caption: Configuration
-
-   wildcards
-   configuration
-   costs
-
-**Rules Overview**
-
-* :doc:`retrieve`
-* :doc:`preparation`
-* :doc:`simplification`
-* :doc:`sector`
-* :doc:`solving`
-* :doc:`plotting`
-
-.. toctree::
-   :hidden:
-   :maxdepth: 1
-   :caption: Rules Overview
-
-   retrieve
-   preparation
-   simplification
-   sector
-   solving
-   plotting
-
-**Foresight options**
-
-* :doc:`overnight`
-* :doc:`myopic`
-* :doc:`perfect`
-
-.. toctree::
-   :hidden:
-   :maxdepth: 1
-   :caption: Foresight options
-
-   overnight
-   myopic
-   perfect
-
-
-**Implementation details**
-
-* :doc:`spatial_resolution`
-* :doc:`supply_demand`
-* :doc:`technology_assumptions`
-
-.. toctree::
-   :hidden:
-   :maxdepth: 1
-   :caption: Implementation details
-
-   spatial_resolution
-   supply_demand
-   technology_assumptions
-
-**References**
-
-* :doc:`release_notes`
-* :doc:`limitations`
-* :doc:`contributing`
-
-.. toctree::
-   :hidden:
-   :maxdepth: 1
-   :caption: References
-
-   release_notes
-   limitations
-   contributing
-
-
-Warnings
-========
-
-Please read the `limitations <https://pypsa-eur.readthedocs.io/en/latest/limitations.html>`_ section of the
-documentation and paper carefully before using the model. We do not
-recommend to use the full resolution network model for simulations. At
-high granularity the assignment of loads and generators to the nearest
-network node may not be a correct assumption, depending on the topology of the underlying distribution grid,
-and local grid
-bottlenecks may cause unrealistic load-shedding or generator
-curtailment. We recommend to cluster the network to a couple of
-hundred nodes to remove these local inconsistencies.
-
 Learning Energy System Modelling
 ================================

-If you are (relatively) new to energy system modelling and optimisation
-and plan to use PyPSA-Eur, the following resources are *one way* to get started
-in addition to reading this documentation.
+If you are (relatively) new to energy system modelling and optimisation and plan
+to use PyPSA-Eur, the following resources are one way to get started in addition
+to reading this documentation.

 - Documentation of `PyPSA <https://pypsa.readthedocs.io>`__, the package for
-  simulating and optimising modern power systems which PyPSA-Eur uses under the hood.
- Course on `Energy Systems <https://nworbmot.org/courses/es-22/>`_,
-  Technical University of Berlin (TUB), `Prof. Dr. Tom Brown <https://nworbmot.org>`_
- Course on `Data Science for Energy System Modelling <https://fneum.github.io/data-science-for-esm/intro.html>`_,
-  Technical University of Berlin (TUB), `Dr. Fabian Neumann <https://neumann.fyi>`_
+  modelling energy systems which PyPSA-Eur uses under the hood.
+- Course on `Energy Systems <https://nworbmot.org/courses/es-22/>`_ given at
+  Technical University of Berlin by `Prof. Dr. Tom Brown <https://nworbmot.org>`_.
+- Course on `Data Science for Energy System Modelling <https://fneum.github.io/data-science-for-esm/intro.html>`_
+  given at Technical University of Berlin by `Dr. Fabian Neumann <https://neumann.fyi>`_.
+

 Citing PyPSA-Eur
 ================

-If you use PyPSA-Eur for your research, we would appreciate it if you would cite the following paper:
-
- Jonas Hörsch, Fabian Hofmann, David Schlachtberger, and Tom Brown. `PyPSA-Eur: An open optimisation model of the European transmission system <https://arxiv.org/abs/1806.01613>`_. Energy Strategy Reviews, 22:207-215, 2018. `arXiv:1806.01613 <https://arxiv.org/abs/1806.01613>`_, `doi:10.1016/j.esr.2018.08.012 <https://doi.org/10.1016/j.esr.2018.08.012>`_.
-
-Please use the following BibTeX: ::
+If you use PyPSA-Eur for your research, we would appreciate it if you would cite the following paper: ::

    @article{PyPSAEur,
        author = "Jonas Hoersch and Fabian Hofmann and David Schlachtberger and Tom Brown",
@ -274,12 +168,12 @@ Please use the following BibTeX: ::
    }


-If you want to cite a specific PyPSA-Eur version, each release of PyPSA-Eur is stored on Zenodo with a release-specific DOI.
-This can be found linked from the overall PyPSA-Eur Zenodo DOI:
+If you want to cite a specific PyPSA-Eur version, each release of PyPSA-Eur is stored on Zenodo with a release-specific DOI:

 .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3520874.svg
   :target: https://doi.org/10.5281/zenodo.3520874

+
 Pre-Built Networks as a Dataset
 ===============================

@ -293,30 +187,59 @@ The included ``.nc`` files are PyPSA network files which can be imported with Py
 .. code:: python

    import pypsa
-
    filename = "elec_s_1024_ec.nc"  # example
    n = pypsa.Network(filename)

-Licence
-=======

-PyPSA-Eur work is released under multiple licenses:

-* All original source code is licensed as free software under `MIT <LICENSES/MIT.txt>`_.
-* The documentation is licensed under `CC-BY-4.0 <LICENSES/CC-BY-4.0.txt>`_.
-* Configuration files are mostly licensed under `CC0-1.0 <LICENSES/CC0-1.0.txt>`_.
-* Data files are licensed under `CC-BY-4.0 <LICENSES/CC-BY-4.0.txt>`_.

-See the individual files and the `dep5 <.reuse/dep5>`_ file for license details.

-Additionally, different licenses and terms of use also apply to the various input data, which are summarised below.
-More details are included in
-`the description of the data bundles on zenodo <https://zenodo.org/record/3517935#.XbGeXvzRZGo>`_.
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+   :caption: Getting Started

-.. csv-table::
-   :header-rows: 1
-   :file: configtables/licenses.csv
+   introduction
+   installation
+   tutorial
+   tutorial_sector

-* BY: Attribute Source
-* NC: Non-Commercial Use Only
-* SA: Share Alike
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+   :caption: Configuration
+
+   wildcards
+   configuration
+   foresight
+   costs
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+   :caption: Rules Overview
+
+   retrieve
+   preparation
+   simplification
+   sector
+   solving
+   plotting
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+   :caption: Implementation details for sector-coupled systems
+
+   spatial_resolution
+   supply_demand
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+   :caption: References
+
+   release_notes
+   licenses
+   limitations
+   contributing
--- a/doc/installation.rst
+++ b/doc/installation.rst
@ -15,13 +15,13 @@ directory in which the commands following the ``%`` should be entered.
 Clone the Repository
 ====================

-First of all, clone the `PyPSA-Eur repository <https://github.com/PyPSA/pypsa-eur>`_ using the version control system ``git``.
+First of all, clone the `PyPSA-Eur repository <https://github.com/PyPSA/pypsa-eur>`_ using the version control system ``git`` in the command line.

 .. code:: bash

-    /some/other/path % cd /some/path/without/spaces
+    /some/other/path % cd /some/path

-    /some/path/without/spaces % git clone https://github.com/PyPSA/pypsa-eur.git
+    /some/path % git clone https://github.com/PyPSA/pypsa-eur.git


 .. _deps:
@ -30,99 +30,108 @@ Install Python Dependencies
 ===============================

 PyPSA-Eur relies on a set of other Python packages to function.
-We recommend using the package manager and environment management system ``conda`` to install them.
-Install `miniconda <https://docs.conda.io/en/latest/miniconda.html>`_, which is a mini version of `Anaconda <https://www.anaconda.com/>`_ that includes only ``conda`` and its dependencies or make sure ``conda`` is already installed on your system.
-For instructions for your operating system follow the ``conda`` `installation guide <https://docs.conda.io/projects/conda/en/latest/user-guide/install/>`_.
+We recommend using the package manager `mamba <https://mamba.readthedocs.io/en/latest/>`_ to install them and manage your environments.
+For instructions for your operating system follow the ``mamba`` `installation guide <https://mamba.readthedocs.io/en/latest/installation.html>`_.
+You can also use ``conda`` equivalently.

-The python package requirements are curated in the `envs/environment.yaml <https://github.com/PyPSA/pypsa-eur/blob/master/envs/environment.yaml>`_ file.
+The package requirements are curated in the `envs/environment.yaml <https://github.com/PyPSA/pypsa-eur/blob/master/envs/environment.yaml>`_ file.
 The environment can be installed and activated using

 .. code:: bash

-    .../pypsa-eur % conda env create -f envs/environment.yaml
+    .../pypsa-eur % mamba create -f envs/environment.yaml

-    .../pypsa-eur % conda activate pypsa-eur
-
-
-Note that activation is local to the currently open shell!
-After opening a new terminal window, one needs to reissue the second command!
+    .../pypsa-eur % mamba activate pypsa-eur

 .. note::
-    If you have troubles with a slow ``conda`` installation, we recommend to install
-    `mamba <https://github.com/QuantStack/mamba>`_ as a fast drop-in replacement via
+    The equivalent commands for ``conda`` would be

    .. code:: bash

-        conda install -c conda-forge mamba
+        .../pypsa-eur % conda env create -f envs/environment.yaml

-    and then install the environment with
+        .../pypsa-eur % conda activate pypsa-eur

-    .. code:: bash
-
-        mamba env create -f envs/environment.yaml

 Install a Solver
 ================

-PyPSA passes the PyPSA-Eur network model to an external solver for performing a total annual system cost minimization with optimal power flow.
+PyPSA passes the PyPSA-Eur network model to an external solver for performing the optimisation.
 PyPSA is known to work with the free software

- `Ipopt <https://coin-or.github.io/Ipopt/INSTALL.html>`_
+- `HiGHS <https://highs.dev/>`_
 - `Cbc <https://projects.coin-or.org/Cbc#DownloadandInstall>`_
 - `GLPK <https://www.gnu.org/software/glpk/>`_ (`WinGLKP <http://winglpk.sourceforge.net/>`_)
- `HiGHS <https://highs.dev/>`_
+- `Ipopt <https://coin-or.github.io/Ipopt/INSTALL.html>`_

 and the non-free, commercial software (for some of which free academic licenses are available)

 - `Gurobi <https://www.gurobi.com/documentation/quickstart.html>`_
 - `CPLEX <https://www.ibm.com/products/ilog-cplex-optimization-studio>`_
- `FICO® Xpress Solver <https://www.fico.com/de/products/fico-xpress-solver>`_
+- `FICO Xpress Solver <https://www.fico.com/de/products/fico-xpress-solver>`_

 For installation instructions of these solvers for your operating system, follow the links above.
-Commercial solvers such as Gurobi and CPLEX currently significantly outperform open-source solvers for large-scale problems.
-It might be the case that you can only retrieve solutions by using a commercial solver.
+Commercial solvers such as Gurobi and CPLEX currently significantly outperform open-source solvers for large-scale problems, and
+it might be the case that you can only retrieve solutions by using a commercial solver.
+Nevertheless, you can still use open-source solvers for smaller problems.

 .. seealso::
-    `Getting a solver in the PyPSA documentation <https://pypsa.readthedocs.io/en/latest/installation.html#getting-a-solver-for-linear-optimisation>`_
+    `Instructions how to install a solver in the documentation of PyPSA <https://pypsa.readthedocs.io/en/latest/installation.html#getting-a-solver-for-linear-optimisation>`_

 .. note::
    The rules :mod:`cluster_network` and :mod:`simplify_network` solve a quadratic optimisation problem for clustering.
    The open-source solvers Cbc and GlPK cannot handle this. A fallback to Ipopt is implemented in this case, but requires
-    also Ipopt to be installed. For an open-source solver setup install in your ``conda`` environment on OSX/Linux
+    it to be installed. For an open-source solver setup install in your ``conda`` environment on OSX/Linux

    .. code:: bash

-        conda activate pypsa-eur
-        conda install -c conda-forge ipopt coincbc
+        mamba activate pypsa-eur
+        mamba install -c conda-forge ipopt coincbc

    and on Windows

    .. code:: bash

-        conda activate pypsa-eur
-        conda install -c conda-forge ipopt glpk
+        mamba activate pypsa-eur
+        mamba install -c conda-forge ipopt glpk

-    or
+    For HiGHS, run

    .. code:: bash

-        conda activate pypsa-eur
+        mamba activate pypsa-eur
+        mamba install -c conda-forge ipopt
        pip install highspy

+    For Gurobi, run
+
+    .. code:: bash
+
+        mamba activate pypsa-eur
+        mamba install -c gurobi gurobi
+
+    Additionally, you need to setup your `Gurobi license <https://www.gurobi.com/solutions/licensing/>`_.
+
+
 .. _defaultconfig:

-Set Up the Default Configuration
-================================
+Handling Configuration Files
+============================

-PyPSA-Eur has several configuration options that must be specified in a ``config.yaml`` file located in the root directory.
-An example configuration ``config.default.yaml`` is maintained in the repository.
-More details on the configuration options are in :ref:`config`.
+PyPSA-Eur has several configuration options that must be specified in a
+``config.yaml`` file located in the root directory. An example configuration
+``config.default.yaml`` is maintained in the repository, which will be used to
+automatically create your customisable ``config.yaml`` on first use. More
+details on the configuration options are in :ref:`config`.

-Before first use, create a ``config.yaml`` by copying the example.
+You can also use ``snakemake`` to specify another file, e.g.
+``config.mymodifications.yaml``, to update the settings of the ``config.yaml``.

 .. code:: bash

-    .../pypsa-eur % cp config.default.yaml config.yaml
+    .../pypsa-eur % snakemake -call --configfile config.mymodifications.yaml

-Users are advised to regularly check their own ``config.yaml`` against changes in the ``config.default.yaml``
-when pulling a new version from the remote repository.
+.. warning::
+    Users are advised to regularly check their own ``config.yaml`` against changes
+    in the ``config.default.yaml`` when pulling a new version from the remote
+    repository.
--- a/doc/introduction.rst
+++ b/doc/introduction.rst
@ -13,58 +13,86 @@

    <iframe width="832" height="468" src="https://www.youtube.com/embed/ty47YU1_eeQ" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>

-Find the introductory slides `here <https://docs.google.com/presentation/d/e/2PACX-1vQGQZD7KIVdocRZzRVu8Uk-JC_ltEow5zjtIarhyws46IMJpaqGuux695yincmJA_i5bVEibEs7z2eo/pub?start=false&loop=true&delayms=3000>`_.
+.. note::
+    Find the introductory slides `here <https://docs.google.com/presentation/d/e/2PACX-1vQGQZD7KIVdocRZzRVu8Uk-JC_ltEow5zjtIarhyws46IMJpaqGuux695yincmJA_i5bVEibEs7z2eo/pub?start=false&loop=true&delayms=3000>`_.

 Workflow
 =========

-The generation of the model is controlled by the workflow management system
-`Snakemake <https://snakemake.bitbucket.io/>`_.
-In a nutshell, the ``Snakefile`` declares for each python script in the ``scripts`` directory a rule which describes which files the scripts consume and produce (their corresponding input and output files).
-The ``snakemake`` tool then runs the scripts in the correct order according to the rules' input/output dependencies.
-Moreover, it is able to track, what parts of the workflow have to be regenerated, when a data file or a script is modified/updated.
+The generation of the model is controlled by the open workflow management system
+`Snakemake <https://snakemake.github.io/>`_. In a nutshell, the ``Snakefile``
+declares for each script in the ``scripts`` directory a rule which describes
+which files the scripts consume and produce (their corresponding input and
+output files). The ``snakemake`` tool then runs the scripts in the correct order
+according to the rules' input and output dependencies. Moreover, ``snakemake``
+will track what parts of the workflow have to be regenerated when files or
+scripts were modified.

-For instance an invocation to
+For instance, an invocation to

 .. code:: bash

    .../pypsa-eur % snakemake -call results/networks/elec_s_128_ec_lvopt_Co2L-3H.nc

-follows this dependency graph:
+follows this dependency graph

-.. image:: img/workflow.png
+.. image:: img/intro-workflow.png
    :class: full-width

-The **blocks** represent the individual rules which are required to create the file ``networks/elec_s_128.nc``. The **arrows** indicate the outputs from preceding rules which a particular rule takes as input data.
+to solve an electricity system model.
+
+The **blocks** represent the individual rules which are required to create the
+file referenced in the command above. The **arrows** indicate the outputs from
+preceding rules which another rule takes as input data.

 .. note::
-    The dependency graph shown above was generated using
-    ``snakemake --dag results/networks/elec_s_128_ec_lvopt_Co2L-3H.nc -F | sed -n "/digraph/,/}/p" | dot -Tpng -o workflow.png``
+    The dependency graph was generated using
+    ``snakemake --dag results/networks/elec_s_128_ec_lvopt_Co2L-3H.nc -F | sed -n "/digraph/,/}/p" | dot -Tpng -o doc/img/intro-workflow.png``

-For the use of ``snakemake``, it makes sense to familiarize oneself quickly with its `basic tutorial <https://snakemake.readthedocs.io/en/stable/tutorial/basics.html>`_ and then read carefully through the section `Executing Snakemake <https://snakemake.readthedocs.io/en/stable/executable.html>`_, noting the arguments ``-j``, ``-n``, ``-r``, but also ``--dag``, ``-R`` and ``-t``.
+For the use of ``snakemake``, it makes sense to familiarize yourself quickly
+with the `basic tutorial
+<https://snakemake.readthedocs.io/en/stable/tutorial/basics.html>`_ and then
+read carefully through the documentation of the `command line interface
+<https://snakemake.readthedocs.io/en/stable/executing/cli.html>`_, noting the
+arguments ``-j``, ``-c``, ``-f``, ``-F``, ``-n``, ``-r``, ``--dag`` and ``-t``
+in particular.

 Scenarios, Configuration and Modification
 =========================================

-It is easy to run PyPSA-Eur for multiple scenarios using the `wildcards feature <https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#wildcards>`_ of ``snakemake``. Wildcards allow to generalise a rule to produce all files that follow a `regular expression <https://en.wikipedia.org/wiki/Regular_expression>`_ pattern, which e.g. defines one particular scenario. One can think of a wildcard as a parameter that shows up in the input/output file names and thereby determines which rules to run, what data to retrieve and what files to produce. **Details are explained in** :ref:`wildcards` **and** :ref:`scenario`.
+It is easy to run PyPSA-Eur for multiple scenarios using the `wildcards feature
+<https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#wildcards>`_
+of ``snakemake``. Wildcards allow to generalise a rule to produce all files that
+follow a `regular expression
+<https://en.wikipedia.org/wiki/Regular_expression>`_ pattern, which defines
+a particular scenario. One can think of a wildcard as a parameter that shows
+up in the input/output file names and thereby determines which rules to run,
+what data to retrieve and what files to produce. Details are explained in
+:ref:`wildcards` and :ref:`scenario`.

-The model also has several further configuration options collected in the ``config.yaml`` file
-located in the root directory, which that are not part of the scenarios. **All options are explained in detail in** :ref:`config`.
+The model also has several further configuration options collected in the
+``config.yaml`` file located in the root directory, which that are not part of
+the scenarios. Options are explained in :ref:`config`.

 Folder Structure
 ================

- ``data``: Includes input data that is not produced by any ``snakemake`` rule.
 - ``scripts``: Includes all the Python scripts executed by the ``snakemake`` rules.
+- ``rules``: Includes all the ``snakemake`` rules loaded in the ``Snakefile``.
+- ``envs``: Includes all the ``conda`` environment specifications to run the workflow.
+- ``data``: Includes input data that is not produced by any ``snakemake`` rule.
+- ``cutouts``: Stores raw weather data cutouts from ``atlite``.
 - ``resources``: Stores intermediate results of the workflow which can be picked up again by subsequent rules.
- ``networks``: Stores intermediate, unsolved stages of the PyPSA network that describes the energy system model.
 - ``results``: Stores the solved PyPSA network data, summary files and plots.
+- ``logs``: Stores log files.
 - ``benchmarks``: Stores ``snakemake`` benchmarks.
- ``logs``: Stores log files about solving, including the solver output, console output and the output of a memory logger.
+- ``test``: Includes the test configuration files used for continuous integration.
+- ``doc``: Includes the documentation of PyPSA-Eur.

 System Requirements
 ===================

-Building the model with the scripts in this repository runs on a normal computer.
-But computing optimal investment and operation scenarios requires a strong interior-point solver
+Building the model with the scripts in this repository runs on a regular computer.
+But optimising for investment and operation decisions across many scenarios requires a strong interior-point solver
 like `Gurobi <http://www.gurobi.com/>`_ or `CPLEX <https://www.ibm.com/analytics/cplex-optimizer>`_ with more memory.
+Open-source solvers like `HiGHS <https://highs.dev>` can also be used for smaller problems.
--- a/doc/licenses.rst
+++ b/doc/licenses.rst
@ -0,0 +1,44 @@
+..
+  SPDX-FileCopyrightText: 2023 The PyPSA-Eur Authors
+
+  SPDX-License-Identifier: CC-BY-4.0
+
+##########################################
+Licenses
+##########################################
+
+
+PyPSA-Eur is released under multiple licenses:
+
+* All original source code is licensed as free software under `MIT <LICENSES/MIT.txt>`_.
+* The documentation is licensed under `CC-BY-4.0 <LICENSES/CC-BY-4.0.txt>`_.
+* Configuration files are mostly licensed under `CC0-1.0 <LICENSES/CC0-1.0.txt>`_.
+* Data files are licensed under `CC-BY-4.0 <LICENSES/CC-BY-4.0.txt>`_.
+
+See the individual files and the `dep5 <.reuse/dep5>`_ file for license details.
+
+Additionally, different licenses and terms of use also apply to the various
+input data for both electricity-only and sector-coupled modelling exercises,
+which are summarised below.
+
+Electricity Systems Databundle
+==============================
+
+.. note::
+  More details are included in `the description of the
+  data bundles on zenodo <https://zenodo.org/record/3517935#.XbGeXvzRZGo>`_.
+
+.. csv-table::
+   :header-rows: 1
+   :file: configtables/licenses.csv
+
+* BY: Attribute Source
+* NC: Non-Commercial Use Only
+* SA: Share Alike
+
+Sector-Coupled Systems Databundle
+=================================
+
+.. csv-table::
+   :header-rows: 1
+   :file: configtables/licenses-sector.csv
--- a/doc/limitations.rst
+++ b/doc/limitations.rst
@ -14,9 +14,12 @@ The limitations of the dataset are listed below,
 both as a warning to the user and as an encouragement to assist in
 improving the approximations.

-This list of limitations is incomplete and will be added to over time.
+.. warning::

-See also the `GitHub repository issues <https://github.com/PyPSA/pypsa-eur/issues>`_.
+  This list of limitations is incomplete and will be added to over time.
+
+.. seealso::
+  See also the `GitHub repository issues <https://github.com/PyPSA/pypsa-eur/issues>`_.

 - **Electricity transmission network topology:**
  The grid data is based on a map of the ENTSO-E area that is known
@ -75,3 +78,4 @@ See also the `GitHub repository issues <https://github.com/PyPSA/pypsa-eur/issue

 - **Demand sufficiency:** Further measures of demand reduction may be
  possible beyond the assumptions made here.
+
--- a/doc/myopic.rst
+++ b/doc/myopic.rst
@ -1,144 +0,0 @@
-..
-  SPDX-FileCopyrightText: 2022-2023 The PyPSA-Eur Authors
-
-  SPDX-License-Identifier: CC-BY-4.0
-
-.. _myopic:
-
-##########################################
-Myopic transition path
-##########################################
-
-The myopic code can be used to investigate progressive changes in a network, for instance, those taking place throughout a transition path. The capacities installed in a certain time step are maintained in the network until their operational lifetime expires.
-
-The myopic approach was initially developed and used in the paper `Early decarbonisation of the European Energy system pays off (2020) <https://www.nature.com/articles/s41467-020-20015-4>`__ and later further extended in `Speed of technological transformations required in Europe to achieve different climate goals (2022) <https://doi.org/10.1016/j.joule.2022.04.016>`__. The current implementation complies with the PyPSA-Eur-Sec standard working flow and is compatible with using the higher resolution electricity transmission model `PyPSA-Eur <https://github.com/PyPSA/pypsa-eur>`__ rather than a one-node-per-country model.
-
-The current code applies the myopic approach to generators, storage technologies and links in the power sector. It furthermore applies it to the space and water heating sector (e.g., the share of district heating and reduced space heat demand), industry processes (e.g., steel, direct reduced iron, and aluminum production via primary route), the share of fuel cell and battery electric vehicles in land transport, and the hydrogen share in shipping (see :doc:`supply_demand` for further information).
-
-The following subjects within the land transport and biomass currently do not evolve with the myopic approach:
-
- The percentage of electric vehicles that allow demand-side management and vehicle-to-grid services.
-
- The annual biomass potential (default year and scenario for which potential is taken is 2030, defined `here <https://github.com/PyPSA/pypsa-eur-sec/blob/413254e241fb37f55b41caba7264644805ad8e97/config.default.yaml#L109>`_)
-
-Configuration
-=================
-
-PyPSA-Eur-Sec has several configuration options which are collected in a config.yaml file located in the root directory. For myopic optimization, users should copy the provided default configuration ``config.default.yaml`` and make their own modifications and assumptions in the user-specific configuration file (``config.yaml``).
-
-The following options included in the config.yaml file  are relevant for the myopic code.
-
-To activate the myopic option select ``foresight: 'myopic'`` in ``config.yaml``.
-
-The {planning_horizons} wildcard indicates the year in which the network is optimized. For a myopic optimization, this is equivalent to the investment year. To set the investment years which are sequentially simulated for the myopic investment planning, select for example:
-
-    planning_horizons:
-
-    \- 2020
-
-    \- 2030
-
-    \- 2040
-
-    \- 2050
-
-in ``config.yaml``.
-
-
-**existing capacities**
-
-Grouping years indicates the bins limits for grouping the existing capacities of different technologies. Note that separate bins are defined for the power and heating plants due to different data sources.
-
-``grouping_years_power: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030]``
-
-``grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019]``
-
-
-
-
-
-**threshold capacity**
-
-If for a technology, node, and grouping bin, the capacity is lower than threshold_capacity, it is ignored.
-
-``threshold_capacity: 10``
-
-
-
-
-**conventional carriers**
-
-Conventional carriers indicate carriers used in the existing conventional technologies.
-
-    conventional_carriers:
-
-    \- lignite
-
-    \- coal
-
-    \- oil
-
-    \- uranium
-
-
-
-
-Options
-=============
-The total carbon budget for the entire transition path can be indicated in the `sector_opts <https://github.com/PyPSA/pypsa-eur-sec/blob/f13902510010b734c510c38c4cae99356f683058/config.default.yaml#L25>`_ in ``config.yaml``. The carbon budget can be split among the ``planning_horizons`` following an exponential or beta decay.
-E.g. ``'cb40ex0'`` splits a carbon budget equal to 40 Gt :math:`_{CO_2}` following an exponential decay whose initial linear growth rate r is zero.
-They can also follow some user-specified path, if defined `here <https://github.com/PyPSA/pypsa-eur-sec/blob/413254e241fb37f55b41caba7264644805ad8e97/config.default.yaml#L56>`_.
-The paper `Speed of technological transformations required in Europe to achieve different climate goals (2022) <https://doi.org/10.1016/j.joule.2022.04.016>`__ defines CO_2 budgets corresponding to global temperature increases (1.5C – 2C) as response to the emissions. Here, global carbon budgets are converted to European budgets assuming equal-per capita distribution which translates into a 6.43% share for Europe. The carbon budgets are in this paper distributed throughout the transition paths assuming an exponential decay. Emissions e(t) in every year t are limited by
-
-.. math::
-  e(t) = e_0 (1+ (r+m)t) e^{-mt}
-
-where r is the initial linear growth rate, which here is assumed to be r=0, and the decay parameter m is determined by imposing the integral of the path to be equal to the budget for Europe. Following this approach, the CO_2 budget is defined. Following the same approach as in this paper, add the following to the ``scenario.sector_opts``
-E.g.  ``-cb25.7ex0`` (1.5C increase)
-Or ``cb73.9ex0`` (2C increase).
-See details in Supplemental Note S1 `Speed of technological transformations required in Europe to achieve different climate goals (2022) <https://doi.org/10.1016/j.joule.2022.04.016>`__.
-
-
-General myopic code structure
-===============================
-
-The myopic code solves the network for the time steps included in ``planning_horizons`` in a recursive loop, so that:
-
-1. The existing capacities (those installed before the base year are added as fixed capacities with p_nom=value, p_nom_extendable=False). E.g. for baseyear=2020, capacities installed before 2020 are added. In addition, the network comprises additional generator, storage, and link capacities with p_nom_extendable=True. The non-solved network is saved in ``results/run_name/networks/prenetworks-brownfield``.
-The base year is the first element in ``planning_horizons``. Step 1 is implemented with the rule add_baseyear for the base year and with the rule add_brownfield for the remaining planning_horizons.
-
-2. The 2020 network is optimized. The solved network is saved in ``results/run_name/networks/postnetworks``
-
-3. For the next planning horizon, e.g. 2030, the capacities from a previous time step are added if they are still in operation (i.e., if they fulfil planning horizon <= commissioned year + lifetime). In addition, the network comprises additional generator, storage, and link capacities with p_nom_extendable=True. The non-solved network is saved in ``results/run_name/networks/prenetworks-brownfield``.
-
-Steps 2 and 3 are solved recursively for all the planning_horizons included in ``config.yaml``.
-
-Rule overview
-===============================
-
- rule add_existing baseyear
-
-  The rule add_existing_baseyear loads the network in ‘results/run_name/networks/prenetworks’ and performs the following operations:
-
-  1. Add the conventional, wind and solar power generators that were installed before the base year.
-
-  2. Add the heating capacities that were installed before the base year.
-
-  The existing conventional generators are retrieved from the `powerplants.csv file <https://pypsa-eur.readthedocs.io/en/latest/preparation/build_powerplants.html?highlight=powerplants>`__ generated by pypsa-eur which, in turn, is based on the `powerplantmatching <https://github.com/FRESNA/powerplantmatching>`__ database.
-
-  Existing wind and solar capacities are retrieved from `IRENA annual statistics <https://www.irena.org/Statistics/Download-Data>`__ and distributed among the nodes in a country proportional to capacity factor. (This will be updated to include capacity distributions closer to reality.)
-
-  Existing heating capacities are retrieved from the report `Mapping and analyses of the current and future (2020 - 2030) heating/cooling fuel deployment (fossil/renewables)
-  <https://ec.europa.eu/energy/studies/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment_en?redir=1>`__.
-
-  The heating capacities are assumed to have a lifetime indicated by the parameter lifetime in the configuration file, e.g 25 years. They are assumed to be decommissioned linearly starting on the base year, e.g., from 2020 to 2045.
-
-  Then, the resulting network is saved in ``results/run_name/networks/prenetworks-brownfield``.
-
- rule add_brownfield
-
-  The rule add_brownfield loads the network in ``results/run_name/networks/prenetworks`` and performs the following operation:
-
-  1. Read the capacities optimized in the previous time step and add them to the network if they are still in operation (i.e., if they fulfill planning horizon < commissioned year + lifetime)
-
-  Then, the resulting network is saved in ``results/run_name/networks/prenetworks_brownfield``.
--- a/doc/overnight.rst
+++ b/doc/overnight.rst
@ -1,16 +0,0 @@
-..
-  SPDX-FileCopyrightText: 2021-2023 The PyPSA-Eur Authors
-
-  SPDX-License-Identifier: CC-BY-4.0
-
-.. _overnight:
-
-##########################################
-Overnight (greenfield) scenarios
-##########################################
-
-The default is to calculate a rebuilding of the energy system to meet demand, a so-called overnight or greenfield approach.
-
-For this, use ``foresight : 'overnight'`` in ``config.yaml``, like the example in ``config.default.yaml``.
-
-In this case, the ``planning_horizons : [2030]`` scenario parameter can be set to use the year from which cost and other technology assumptions are set (forecasts for 2030 in this case).
--- a/doc/perfect.rst
+++ b/doc/perfect.rst
@ -1,14 +0,0 @@
-..
-  SPDX-FileCopyrightText: 2021-2023 The PyPSA-Eur Authors
-
-  SPDX-License-Identifier: CC-BY-4.0
-
-.. _perfect:
-
-##########################################
-Perfect foresight scenarios
-##########################################
-
-Perfect foresight is currently under development but it is not yet implemented.
-
-For this, use ``foresight : 'perfect'`` in ``config.yaml``.
--- a/doc/preparation.rst
+++ b/doc/preparation.rst
@ -4,7 +4,7 @@
  SPDX-License-Identifier: CC-BY-4.0

 ##########################################
-Preparing Networks
+Building Electricity Networks
 ##########################################

 The preparation process of the PyPSA-Eur energy system model consists of a group of ``snakemake``
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@ -10,7 +10,21 @@ Release Notes
 Upcoming Release
 ================

-* The solver configuration in `config.default.yaml` are now modularized. To change the set of solver options, change to value in `solving`: `solver`: `options` to one of the keys `solving`: `solver_options`.
+* new feature or bugfix
+
+
+PyPSA-Eur 0.8.0 (TBA)
+=====================
+
+.. note:: 
+  This is the first release of PyPSA-Eur which incorporates its sector-coupled extension PyPSA-Eur-Sec.
+
+* The solver configurations in ``config.default.yaml`` are now modularized. To
+  change the set of solver options, change to value in ``solving: solver:
+  options:`` to one of the keys in ``solving: solver_options:``.
+
+* The ``Snakefile`` has been modularised. Rules are now organised in the
+  ``rules`` directory.

 PyPSA-Eur Releases (pre-merge)
 ==============================
@ -445,7 +459,7 @@ PyPSA-Eur 0.3.0 (7th December 2020)

 **New Features**

-Using the ``{opts}`` wildcard for scenarios:
+Using the ``{opts}`` wildcard for scenario:

 * An option is introduced which adds constraints such that each country or node produces on average a minimal share of its total consumption itself.
  For example ``EQ0.5c`` set in the ``{opts}`` wildcard requires each country to produce on average at least 50% of its consumption. Additionally,
@ -1186,7 +1200,7 @@ Release Process

 * Make a `GitHub release <https://github.com/PyPSA/pypsa-eur-sec/releases>`_, which automatically triggers archiving to the `zenodo code repository <https://doi.org/10.5281/zenodo.3520874>`_ with `MIT license <https://opensource.org/licenses/MIT>`_.

-* Create pre-built networks for ``config.default.yaml`` by running ``snakemake -call extra_components_all_networks``.
+* Create pre-built networks for ``config.default.yaml`` by running ``snakemake -call extra_components_networks``.

 * Upload pre-built networks to `zenodo data repository <https://doi.org/10.5281/zenodo.3601881>`_ with `CC BY 4.0 <https://creativecommons.org/licenses/by/4.0/>`_ license.

--- a/doc/retrieve.rst
+++ b/doc/retrieve.rst
@ -135,14 +135,11 @@ None.
 Rule ``retrieve_sector_databundle``
 ====================================

+.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.5546516.svg
+   :target: https://doi.org/10.5281/zenodo.5546516

-Small data files are included directly in the git repository, while
-larger ones are archived in a data bundle on zenodo (`10.5281/zenodo.5824485 <https://doi.org/10.5281/zenodo.5824485>`_).
-The data bundle's size is around 640 MB.
-
-The data licences and sources are given in the following table.
+In addition to the databundle required for electricity-only studies,
+another databundle is required for modelling sector-coupled systems.
+The size of this data bundle is around 640 MB.


-.. csv-table::
-   :header-rows: 1
-   :file: data.csv
--- a/doc/sector.rst
+++ b/doc/sector.rst
@ -7,6 +7,9 @@
 Building Sector-Coupled Networks
 ##########################################

+.. warning::
+  This part of the documentation is under development.
+
 Rule ``add_brownfield``
 ==============================================================================

--- a/doc/simplification.rst
+++ b/doc/simplification.rst
@ -6,7 +6,7 @@
  SPDX-License-Identifier: CC-BY-4.0

 ##########################################
-Simplifying Networks
+Simplifying Electricity Networks
 ##########################################

 The simplification ``snakemake`` rules prepare **approximations** of the full model, for which it is computationally viable to co-optimize generation, storage and transmission capacities.
--- a/doc/solving.rst
+++ b/doc/solving.rst
@ -7,7 +7,10 @@
 Solving Networks
 ##########################################

-After generating and simplifying the networks they can be solved through the rule :mod:`solve_network`  by using the collection rule :mod:`solve_all_networks`. Moreover, networks can be solved for another focus with the derivative rules :mod:`solve_network`  by using the collection rule :mod:`solve_operations_network` for dispatch-only analyses on an already solved network.
+After generating and simplifying the networks they can be solved through the
+rule :mod:`solve_network`  by using the collection rules ``solve_elec_networks``
+or ``solve_sector_networks``. Moreover, networks can be solved for dispatch-only
+analyses on an already solved network with :mod:`solve_operations_network`.

 .. _solve:

@ -26,4 +29,5 @@ Rule ``solve_operations_network``
 Rule ``solve_sector_network``
 =============================

-.. automodule:: solve_sector_network
+.. warning::
+   More comprehensive documentation for this rule will be released soon.
--- a/doc/technology_assumptions.rst
+++ b/doc/technology_assumptions.rst
@ -1,14 +0,0 @@
-..
-  SPDX-FileCopyrightText: 2021-2023 The PyPSA-Eur Authors
-
-  SPDX-License-Identifier: CC-BY-4.0
-
-.. _technology_assumptions:
-
-##########################################
-Technology and cost assumptions
-##########################################
-
-*Techno-Economic Assumptions*
-
-For the technological assumptions (cost, efficiency, lifetime, etc.), we take estimates for the investment year specified in the `config <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L43>`_. Many of those come from a database published by the Danish Energy Agency (`DEA <https://ens.dk/en/our-services/projections-and-models/technology-data>`_). Assumptions are maintained at the `technology data repository <https://github.com/PyPSA/technology-data>`_.
--- a/doc/tutorial.rst
+++ b/doc/tutorial.rst
@ -5,130 +5,121 @@

 .. _tutorial:

-#####################
-Tutorial
-#####################
+###############################
+Tutorial: Electricity-Only
+###############################

 .. raw:: html

    <iframe width="832" height="468" src="https://www.youtube.com/embed/mAwhQnNRIvs" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>

-Before getting started with **PyPSA-Eur** it makes sense to be familiar
+.. note::
+    If you have not done it yet, follow the :ref:`installation` steps first.
+
+In this tutorial, we will build a heavily simplified power system model for
+Belgium. But before getting started with **PyPSA-Eur** it makes sense to be familiar
 with its general modelling framework `PyPSA <https://pypsa.readthedocs.io>`__.

-Running the tutorial requires limited computational resources compared to the full model,
-which allows the user to explore most of its functionalities on a local machine.
-It takes approximately five minutes to complete and
-requires 3 GB of memory along with 1 GB free disk space.
-
-If not yet completed, follow the :ref:`installation` steps first.
-
-The tutorial will cover examples on how to
-
- configure and customise the PyPSA-Eur model and
- run the ``snakemake`` workflow step by step from network creation to the solved network.
-
-The configuration of the tutorial is included in the ``config.tutorial.yaml``.
-To run the tutorial, use this as your configuration file ``config.yaml``.
+Running the tutorial requires limited computational resources compared to the
+full model, which allows the user to explore most of its functionalities on a
+local machine. The tutorial will cover examples on how to configure and
+customise the PyPSA-Eur model and run the ``snakemake`` workflow step by step
+from network creation to the solved network. The configuration for the tutorial
+is located at ``test/config.electricity.yaml``. It includes parts deviating from
+the default config file ``config.default.yaml``. To run the tutorial with this
+configuration, execute

 .. code:: bash
+    :class: full-width

-    .../pypsa-eur % cp config.tutorial.yaml config.yaml
+    snakemake -call --configfile test/config.electricity.yaml results/networks/elec_s_6_ec_lcopt_Co2L-24H.nc

 This configuration is set to download a reduced data set via the rules :mod:`retrieve_databundle`,
-:mod:`retrieve_natura_raster`, :mod:`retrieve_cutout` totalling at less than 250 MB.
-The full set of data dependencies would take 5.3 GB.
+:mod:`retrieve_natura_raster`, :mod:`retrieve_cutout`.
 For more information on the data dependencies of PyPSA-Eur, continue reading :ref:`data`.

-How to customise PyPSA-Eur?
+How to configure runs?
 ===========================

 The model can be adapted to only include selected countries (e.g. Belgium) instead of all European countries to limit the spatial scope.

-.. literalinclude:: ../config.tutorial.yaml
+.. literalinclude:: ../test/config.electricity.yaml
   :language: yaml
   :start-at: countries:
   :end-before: snapshots:

-Likewise, the example's temporal scope can be restricted (e.g. to a single month).
+Likewise, the example's temporal scope can be restricted (e.g. to a single week).

-.. literalinclude:: ../config.tutorial.yaml
+.. literalinclude:: ../test/config.electricity.yaml
   :language: yaml
   :start-at: snapshots:
-   :end-before: enable:
+   :end-before: electricity:

-It is also possible to allow less or more carbon-dioxide emissions. Here, we limit the emissions of Germany 100 Megatonnes per year.
+It is also possible to allow less or more carbon-dioxide emissions. Here, we limit the emissions of Belgium to 100 Mt per year.

-.. literalinclude:: ../config.tutorial.yaml
+.. literalinclude:: ../test/config.electricity.yaml
   :language: yaml
   :start-at: electricity:
   :end-before: extendable_carriers:

 PyPSA-Eur also includes a database of existing conventional powerplants.
-We can select which types of powerplants we like to be included:
+We can select which types of existing powerplants we like to be extendable:

-.. literalinclude:: ../config.tutorial.yaml
+.. literalinclude:: ../test/config.electricity.yaml
   :language: yaml
   :start-at: extendable_carriers:
-   :end-before: max_hours:
+   :end-before: renewable_carriers:

-To accurately model the temporal and spatial availability of renewables such as wind and solar energy, we rely on historical weather data.
-It is advisable to adapt the required range of coordinates to the selection of countries.
+To accurately model the temporal and spatial availability of renewables such as
+wind and solar energy, we rely on historical weather data. It is advisable to
+adapt the required range of coordinates to the selection of countries.

-.. literalinclude:: ../config.tutorial.yaml
+.. literalinclude:: ../test/config.electricity.yaml
   :language: yaml
   :start-at: atlite:
   :end-before: renewable:

-We can also decide which weather data source should be used to calculate potentials and capacity factor time-series for each carrier.
-For example, we may want to use the ERA-5 dataset for solar and not the default SARAH-2 dataset.
+We can also decide which weather data source should be used to calculate
+potentials and capacity factor time-series for each carrier. For example, we may
+want to use the ERA-5 dataset for solar and not the default SARAH-2 dataset.

-.. literalinclude:: ../config.tutorial.yaml
-   :language: yaml
-   :start-at: be-03-2013-era5:
-   :end-at: module:
-
-.. literalinclude:: ../config.tutorial.yaml
+.. literalinclude:: ../test/config.electricity.yaml
   :language: yaml
   :start-at: solar:
   :end-at: cutout:

-Finally, it is possible to pick a solver. For instance, this tutorial uses the open-source solvers CBC and Ipopt and does not rely
-on the commercial solvers Gurobi or CPLEX (for which free academic licenses are available).
+Finally, it is possible to pick a solver. For instance, this tutorial uses the
+open-source solver GLPK.

-.. literalinclude:: ../config.tutorial.yaml
+.. literalinclude:: ../test/config.electricity.yaml
   :language: yaml
   :start-at: solver:
   :end-before: plotting:

-.. note::
+Note, that ``test/config.electricity.yaml`` only includes changes relative to
+the default configuration. There are many more configuration options, which are
+documented at :ref:`config`.

-    To run the tutorial, either install CBC and Ipopt (see instructions for :ref:`installation`).

-    Alternatively, choose another installed solver in the ``config.yaml`` at ``solving: solver:``.
-
-Note, that we only focus on changes relative to the default configuration.
-There are many more configuration options, which are documented at :ref:`config`.
-
-How to use the ``snakemake`` rules?
+How to use ``snakemake`` rules?
 ===================================

 Open a terminal, go into the PyPSA-Eur directory, and activate the ``pypsa-eur`` environment with

 .. code:: bash

-    .../pypsa-eur % conda activate pypsa-eur
+    mamba activate pypsa-eur

 Let's say based on the modifications above we would like to solve a very simplified model
 clustered down to 6 buses and every 24 hours aggregated to one snapshot. The command

 .. code:: bash

-    .../pypsa-eur % snakemake -call results/networks/elec_s_6_ec_lcopt_Co2L-24H.nc
+    snakemake -call --configfile test/config.electricity.yaml results/networks/elec_s_6_ec_lcopt_Co2L-24H.nc

-orders ``snakemake`` to run the script ``solve_network`` that produces the solved network and stores it in ``.../pypsa-eur/results/networks`` with the name ``elec_s_6_ec_lcopt_Co2L-24H.nc``:
+orders ``snakemake`` to run the rule :mod:`solve_network` that produces the solved network and stores it in ``results/networks`` with the name ``elec_s_6_ec_lcopt_Co2L-24H.nc``:

-.. literalinclude:: ../Snakefile
+.. literalinclude:: ../rules/solve_electricity.smk
   :start-at: rule solve_network:
   :end-before: rule solve_operations_network:

@ -234,7 +225,6 @@ In the terminal, this will show up as a list of jobs to be run:
 .. code:: bash

    Building DAG of jobs...
-    Job stats:
    job                         count    min threads    max threads
    ------------------------  -------  -------------  -------------
    add_electricity                 1              1              1
@ -282,38 +272,51 @@ You can produce any output file occurring in the ``Snakefile`` by running

 .. code:: bash

-    .../pypsa-eur % snakemake -call <output file>
+    snakemake -call <output file>

 For example, you can explore the evolution of the PyPSA networks by running

-#. ``.../pypsa-eur % snakemake -call networks/base.nc``
-#. ``.../pypsa-eur % snakemake -call networks/elec.nc``
-#. ``.../pypsa-eur % snakemake -call networks/elec_s.nc``
-#. ``.../pypsa-eur % snakemake -call networks/elec_s_6.nc``
-#. ``.../pypsa-eur % snakemake -call networks/elec_s_6_ec_lcopt_Co2L-24H.nc``
+#. ``snakemake -call --configfile test/config.electricity.yaml resources/networks/base.nc``
+#. ``snakemake -call --configfile test/config.electricity.yaml resources/networks/elec.nc``
+#. ``snakemake -call --configfile test/config.electricity.yaml resources/networks/elec_s.nc``
+#. ``snakemake -call --configfile test/config.electricity.yaml resources/networks/elec_s_6.nc``
+#. ``snakemake -call --configfile test/config.electricity.yaml resources/networks/elec_s_6_ec_lcopt_Co2L-24H.nc``

-There's a special rule: If you simply run
+To run all combinations of wildcard values provided in the ``config.yaml`` under ``scenario:``,
+you can use the collection rule ``solve_elec_networks``.

 .. code:: bash

-    .../pypsa-eur % snakemake
+    snakemake -call --configfile test/config.electricity.yaml solve_elec_networks

-the wildcards given in ``scenario`` in the configuration file ``config.yaml`` are used:
+If you now feel confident and want to tackle runs with larger temporal and
+spatial scope, clean-up the repository and after modifying the ``config.yaml`` file
+target the collection rule ``solve_elec_networks`` again without providing the test
+configuration file.

-.. literalinclude:: ../config.tutorial.yaml
-   :language: yaml
-   :start-at: scenario:
-   :end-before: countries:
+.. code:: bash

-How to analyse solved networks?
+    snakemake -call purge
+    snakemake -call solve_elec_networks
+
+.. note::
+
+    It is good practice to perform a dry-run using the option `-n`, before you
+    commit to a run:
+
+    .. code:: bash
+        snakemake -call solve_elec_networks -n
+
+How to analyse results?
 ===============================

-The solved networks can be analysed just like any other PyPSA network (e.g. in Jupyter Notebooks).
+The solved networks can be analysed just like any other PyPSA network (e.g. in
+Jupyter Notebooks).

 .. code:: python

    import pypsa

-    network = pypsa.Network("results/networks/elec_s_6_ec_lcopt_Co2L-24H.nc")
+    n = pypsa.Network("results/networks/elec_s_6_ec_lcopt_Co2L-24H.nc")

 For inspiration, read the `examples section in the PyPSA documentation <https://pypsa.readthedocs.io/en/latest/examples-basic.html>`_.
--- a/doc/tutorial_sector.rst
+++ b/doc/tutorial_sector.rst
@ -0,0 +1,524 @@
+..
+  SPDX-FileCopyrightText: 2023 The PyPSA-Eur Authors
+
+  SPDX-License-Identifier: CC-BY-4.0
+
+.. _tutorial_sector:
+
+###############################
+Tutorial: Sector-Coupled
+###############################
+
+.. note::
+    If you have not done it yet, follow the :ref:`installation` steps first.
+
+    Also, checkout the tutorial for electricity-only systems first at :ref:`tutorial`.
+
+In this tutorial, we will add further sectors to the electricity-only model from
+:ref:`tutorial`, namely industry, transport, and buildings. This
+requires processing of a few more raw data sources.
+
+The sector-coupling code can be run as an overnight / greenfield scenario or
+with multi-horizon investment with myopic foresight. Pathway analysis with
+perfect foresight is under development. See also the documentation on
+:ref:`foresight`.
+
+Overnight Scenarios
+===========================
+
+Configuration
+-------------
+
+The default configuration file (``config.default.yaml``) is set up for running
+overnight scenarios. Running a sector-coupled model unlocks many further
+configuration options. In the example below, we say that the gas network should
+be added and spatially resolved. We also say that the existing gas network may
+be retrofitted to transport hydrogen instead.
+
+.. literalinclude:: ../test/config.overnight.yaml
+   :language: yaml
+   :start-at: sector:
+   :end-before: solving:
+
+Documentation for all options will be added successively to :ref:`config`.
+
+Scenarios can be defined like for electricity-only studies, but with additional
+wildcard options.
+
+.. literalinclude:: ../test/config.overnight.yaml
+   :language: yaml
+   :start-at: scenario:
+   :end-before: countries:
+
+For allowed wildcard values, refer to :ref:`wildcards`.
+
+Execution
+---------
+
+To run an overnight / greenfiled scenario with the specifications above, run
+
+.. code:: bash
+
+    snakemake -call --configfile test/config.overnight.yaml all
+
+which will result in the following *additional* jobs ``snakemake`` wants to run
+on top of those already included in the electricity-only tutorial:
+
+.. code:: bash
+
+    job                                                 count    min threads    max threads
+    ------------------------------------------------  -------  -------------  -------------
+    all                                                     1              1              1
+    build_ammonia_production                                1              1              1
+    build_biomass_potentials                                1              1              1
+    build_clustered_population_layouts                      1              1              1
+    build_cop_profiles                                      1              1              1
+    build_gas_input_locations                               1              1              1
+    build_gas_network                                       1              1              1
+    build_heat_demands                                      3              1              1
+    build_industrial_distribution_key                       1              1              1
+    build_industrial_energy_demand_per_country_today        1              1              1
+    build_industrial_energy_demand_per_node                 1              1              1
+    build_industrial_energy_demand_per_node_today           1              1              1
+    build_industrial_production_per_country                 1              1              1
+    build_industrial_production_per_country_tomorrow        1              1              1
+    build_industrial_production_per_node                    1              1              1
+    build_industry_sector_ratios                            1              1              1
+    build_population_weighted_energy_totals                 1              1              1
+    build_salt_cavern_potentials                            1              1              1
+    build_shipping_demand                                   1              1              1
+    build_simplified_population_layouts                     1              1              1
+    build_solar_thermal_profiles                            3              1              1
+    build_temperature_profiles                              3              1              1
+    build_transport_demand                                  1              1              1
+    cluster_gas_network                                     1              1              1
+    cluster_network                                         1              1              1
+    copy_config                                             1              1              1
+    make_summary                                            1              1              1
+    plot_network                                            1              1              1
+    plot_summary                                            1              1              1
+    prepare_sector_network                                  1              1              1
+    retrieve_gas_infrastructure_data                        1              1              1
+    retrieve_sector_databundle                              1              1              1
+    solve_sector_network                                    1              1              1
+
+This covers the retrieval of additional raw data from online resources and
+preprocessing data about the transport, industry, and heating sectors as well as
+additional rules about geological storage and sequestration potentials, gas
+infrastructure, and biomass potentials. The collection rule ``all`` will also
+generate summary CSV files and plots after the network has been solved
+successfully.
+
+
+
+.. graphviz::
+    :class: full-width
+    :align: center
+
+    digraph snakemake_dag {
+        graph[bgcolor=white, margin=0];
+        node[shape=box, style=rounded, fontname=sans,                 fontsize=10, penwidth=2];
+        edge[penwidth=2, color=grey];
+        0[label = "all", color = "0.51 0.6 0.85", style="rounded"];
+        1[label = "plot_summary", color = "0.54 0.6 0.85", style="rounded"];
+        2[label = "make_summary", color = "0.44 0.6 0.85", style="rounded"];
+        3[label = "solve_sector_network", color = "0.46 0.6 0.85", style="rounded"];
+        4[label = "prepare_sector_network", color = "0.09 0.6 0.85", style="rounded"];
+        5[label = "cluster_gas_network", color = "0.38 0.6 0.85", style="rounded"];
+        6[label = "build_gas_network", color = "0.00 0.6 0.85", style="rounded"];
+        7[label = "retrieve_gas_infrastructure_data", color = "0.33 0.6 0.85", style="rounded"];
+        8[label = "cluster_network", color = "0.26 0.6 0.85", style="rounded"];
+        9[label = "simplify_network", color = "0.03 0.6 0.85", style="rounded"];
+        10[label = "add_electricity", color = "0.25 0.6 0.85", style="rounded"];
+        11[label = "build_renewable_profiles", color = "0.07 0.6 0.85", style="rounded"];
+        12[label = "base_network", color = "0.16 0.6 0.85", style="rounded"];
+        13[label = "build_shapes", color = "0.65 0.6 0.85", style="rounded"];
+        14[label = "retrieve_databundle", color = "0.20 0.6 0.85", style="rounded"];
+        15[label = "retrieve_natura_raster", color = "0.10 0.6 0.85", style="rounded"];
+        16[label = "build_bus_regions", color = "0.11 0.6 0.85", style="rounded"];
+        17[label = "build_ship_raster", color = "0.56 0.6 0.85", style="rounded"];
+        18[label = "retrieve_ship_raster", color = "0.15 0.6 0.85", style="rounded"];
+        19[label = "retrieve_cost_data", color = "0.50 0.6 0.85", style="rounded"];
+        20[label = "build_powerplants", color = "0.49 0.6 0.85", style="rounded"];
+        21[label = "build_load_data", color = "0.39 0.6 0.85", style="rounded"];
+        22[label = "retrieve_load_data", color = "0.05 0.6 0.85", style="rounded"];
+        23[label = "build_gas_input_locations", color = "0.45 0.6 0.85", style="rounded"];
+        24[label = "prepare_network", color = "0.31 0.6 0.85", style="rounded"];
+        25[label = "add_extra_components", color = "0.23 0.6 0.85", style="rounded"];
+        26[label = "build_energy_totals", color = "0.19 0.6 0.85", style="rounded"];
+        27[label = "build_population_weighted_energy_totals", color = "0.27 0.6 0.85", style="rounded"];
+        28[label = "build_clustered_population_layouts", color = "0.64 0.6 0.85", style="rounded"];
+        29[label = "build_population_layouts", color = "0.43 0.6 0.85", style="rounded"];
+        30[label = "build_shipping_demand", color = "0.57 0.6 0.85", style="rounded"];
+        31[label = "build_transport_demand", color = "0.53 0.6 0.85", style="rounded"];
+        32[label = "build_temperature_profiles", color = "0.58 0.6 0.85", style="rounded"];
+        33[label = "build_biomass_potentials", color = "0.30 0.6 0.85", style="rounded"];
+        34[label = "build_salt_cavern_potentials", color = "0.47 0.6 0.85", style="rounded"];
+        35[label = "build_simplified_population_layouts", color = "0.32 0.6 0.85", style="rounded"];
+        36[label = "build_industrial_energy_demand_per_node", color = "0.14 0.6 0.85", style="rounded"];
+        37[label = "build_industry_sector_ratios", color = "0.18 0.6 0.85", style="rounded"];
+        38[label = "build_ammonia_production", color = "0.48 0.6 0.85", style="rounded"];
+        39[label = "build_industrial_production_per_node", color = "0.12 0.6 0.85", style="rounded"];
+        40[label = "build_industrial_distribution_key", color = "0.61 0.6 0.85", style="rounded"];
+        41[label = "build_industrial_production_per_country_tomorrow", color = "0.22 0.6 0.85", style="rounded"];
+        42[label = "build_industrial_production_per_country", color = "0.59 0.6 0.85", style="rounded"];
+        43[label = "build_industrial_energy_demand_per_node_today", color = "0.62 0.6 0.85", style="rounded"];
+        44[label = "build_industrial_energy_demand_per_country_today", color = "0.41 0.6 0.85", style="rounded"];
+        45[label = "build_heat_demands", color = "0.08 0.6 0.85", style="rounded"];
+        46[label = "build_cop_profiles", color = "0.52 0.6 0.85", style="rounded"];
+        47[label = "build_solar_thermal_profiles", color = "0.17 0.6 0.85", style="rounded"];
+        48[label = "copy_config", color = "0.40 0.6 0.85", style="rounded"];
+        49[label = "plot_network", color = "0.60 0.6 0.85", style="rounded"];
+        1 -> 0
+        2 -> 1
+        49 -> 2
+        19 -> 2
+        3 -> 2
+        48 -> 3
+        4 -> 3
+        19 -> 3
+        9 -> 4
+        11 -> 4
+        45 -> 4
+        36 -> 4
+        47 -> 4
+        26 -> 4
+        27 -> 4
+        8 -> 4
+        33 -> 4
+        24 -> 4
+        35 -> 4
+        5 -> 4
+        23 -> 4
+        34 -> 4
+        19 -> 4
+        31 -> 4
+        46 -> 4
+        30 -> 4
+        32 -> 4
+        28 -> 4
+        6 -> 5
+        8 -> 5
+        7 -> 6
+        19 -> 8
+        9 -> 8
+        19 -> 9
+        10 -> 9
+        16 -> 9
+        14 -> 10
+        21 -> 10
+        20 -> 10
+        19 -> 10
+        11 -> 10
+        16 -> 10
+        13 -> 10
+        12 -> 10
+        14 -> 11
+        17 -> 11
+        15 -> 11
+        16 -> 11
+        12 -> 11
+        13 -> 11
+        13 -> 12
+        14 -> 13
+        12 -> 16
+        13 -> 16
+        18 -> 17
+        12 -> 20
+        22 -> 21
+        8 -> 23
+        7 -> 23
+        25 -> 24
+        19 -> 24
+        19 -> 25
+        8 -> 25
+        13 -> 26
+        28 -> 27
+        26 -> 27
+        8 -> 28
+        29 -> 28
+        13 -> 29
+        13 -> 30
+        8 -> 30
+        26 -> 30
+        32 -> 31
+        28 -> 31
+        27 -> 31
+        26 -> 31
+        8 -> 32
+        29 -> 32
+        13 -> 33
+        14 -> 33
+        8 -> 33
+        8 -> 34
+        9 -> 35
+        29 -> 35
+        37 -> 36
+        39 -> 36
+        43 -> 36
+        38 -> 37
+        41 -> 39
+        40 -> 39
+        28 -> 40
+        8 -> 40
+        42 -> 41
+        38 -> 42
+        44 -> 43
+        40 -> 43
+        38 -> 44
+        42 -> 44
+        8 -> 45
+        29 -> 45
+        32 -> 46
+        8 -> 47
+        29 -> 47
+        8 -> 49
+        3 -> 49
+    } 
+
+|
+
+Myopic Foresight Scenarios
+===================================
+
+Configuration
+-------------
+
+In the myopic foresight mode, 
+
+.. literalinclude:: ../test/config.myopic.yaml
+   :language: yaml
+   :start-at: industry:
+   :end-before: solving:
+
+Documentation for all options will be added successively to :ref:`config`.
+
+Scenarios can be defined like for electricity-only studies, but with additional
+wildcard options. For the myopic foresight mode, the ``{planning_horizons}`` wildcard
+defines the sequence of investment horizons.
+
+.. literalinclude:: ../test/config.myopic.yaml
+   :language: yaml
+   :start-at: scenario:
+   :end-before: countries:
+
+For allowed wildcard values, refer to :ref:`wildcards`.
+
+Execution
+---------
+
+To run a myopic foresight scenario with the specifications above, run
+
+.. code:: bash
+
+    snakemake -call --configfile test/config.myopic.yaml all
+
+which will result in the following *additional* jobs ``snakemake`` wants to run:
+
+.. code:: bash
+
+    job                                                 count    min threads    max threads
+    ------------------------------------------------  -------  -------------  -------------
+    all                                                     1              1              1
+    add_brownfield                                          2              1              1
+    add_existing_baseyear                                   1              1              1
+    plot_network                                            3              1              1
+    plot_summary                                            1              1              1
+    prepare_sector_network                                  3              1              1
+    solve_sector_network_myopic                             3              1              1
+
+which translates to the following workflow diagram which nicely outlines
+how the sequential pathway optimisation with myopic foresight is
+implemented in the workflow:
+
+.. graphviz::
+    :class: full-width
+    :align: center
+
+    digraph snakemake_dag {
+        graph[bgcolor=white, margin=0];
+        node[shape=box, style=rounded, fontname=sans,                 fontsize=10, penwidth=2];
+        edge[penwidth=2, color=grey];
+        0[label = "all", color = "0.38 0.6 0.85", style="rounded"];
+        1[label = "plot_summary", color = "0.61 0.6 0.85", style="rounded"];
+        2[label = "make_summary", color = "0.51 0.6 0.85", style="rounded"];
+        3[label = "solve_sector_network_myopic", color = "0.32 0.6 0.85", style="rounded"];
+        4[label = "add_existing_baseyear", color = "0.20 0.6 0.85", style="rounded"];
+        5[label = "prepare_sector_network", color = "0.14 0.6 0.85", style="rounded"];
+        6[label = "prepare_network", color = "0.06 0.6 0.85", style="rounded"];
+        7[label = "add_extra_components", color = "0.00 0.6 0.85", style="rounded"];
+        8[label = "cluster_network", color = "0.18 0.6 0.85", style="rounded"];
+        9[label = "simplify_network", color = "0.30 0.6 0.85", style="rounded"];
+        10[label = "add_electricity", color = "0.24 0.6 0.85", style="rounded"];
+        11[label = "build_renewable_profiles", color = "0.40 0.6 0.85", style="rounded"];
+        12[label = "base_network", color = "0.11 0.6 0.85", style="rounded"];
+        13[label = "build_shapes", color = "0.29 0.6 0.85", style="rounded"];
+        14[label = "retrieve_databundle", color = "0.58 0.6 0.85", style="rounded"];
+        15[label = "retrieve_natura_raster", color = "0.39 0.6 0.85", style="rounded"];
+        16[label = "build_bus_regions", color = "0.60 0.6 0.85", style="rounded"];
+        17[label = "build_ship_raster", color = "0.65 0.6 0.85", style="rounded"];
+        18[label = "retrieve_ship_raster", color = "0.09 0.6 0.85", style="rounded"];
+        19[label = "retrieve_cost_data", color = "0.04 0.6 0.85", style="rounded"];
+        20[label = "build_powerplants", color = "0.28 0.6 0.85", style="rounded"];
+        21[label = "build_load_data", color = "0.46 0.6 0.85", style="rounded"];
+        22[label = "retrieve_load_data", color = "0.44 0.6 0.85", style="rounded"];
+        23[label = "build_energy_totals", color = "0.53 0.6 0.85", style="rounded"];
+        24[label = "build_population_weighted_energy_totals", color = "0.03 0.6 0.85", style="rounded"];
+        25[label = "build_clustered_population_layouts", color = "0.34 0.6 0.85", style="rounded"];
+        26[label = "build_population_layouts", color = "0.63 0.6 0.85", style="rounded"];
+        27[label = "build_shipping_demand", color = "0.05 0.6 0.85", style="rounded"];
+        28[label = "build_transport_demand", color = "0.52 0.6 0.85", style="rounded"];
+        29[label = "build_temperature_profiles", color = "0.16 0.6 0.85", style="rounded"];
+        30[label = "build_biomass_potentials", color = "0.47 0.6 0.85", style="rounded"];
+        31[label = "build_salt_cavern_potentials", color = "0.48 0.6 0.85", style="rounded"];
+        32[label = "build_simplified_population_layouts", color = "0.08 0.6 0.85", style="rounded"];
+        33[label = "build_industrial_energy_demand_per_node", color = "0.22 0.6 0.85", style="rounded"];
+        34[label = "build_industry_sector_ratios", color = "0.56 0.6 0.85", style="rounded"];
+        35[label = "build_ammonia_production", color = "0.57 0.6 0.85", style="rounded"];
+        36[label = "build_industrial_production_per_node", color = "0.66 0.6 0.85", style="rounded"];
+        37[label = "build_industrial_distribution_key", color = "0.41 0.6 0.85", style="rounded"];
+        38[label = "build_industrial_production_per_country_tomorrow", color = "0.54 0.6 0.85", style="rounded"];
+        39[label = "build_industrial_production_per_country", color = "0.10 0.6 0.85", style="rounded"];
+        40[label = "build_industrial_energy_demand_per_node_today", color = "0.55 0.6 0.85", style="rounded"];
+        41[label = "build_industrial_energy_demand_per_country_today", color = "0.35 0.6 0.85", style="rounded"];
+        42[label = "build_heat_demands", color = "0.49 0.6 0.85", style="rounded"];
+        43[label = "build_cop_profiles", color = "0.01 0.6 0.85", style="rounded"];
+        44[label = "build_solar_thermal_profiles", color = "0.45 0.6 0.85", style="rounded"];
+        45[label = "copy_config", color = "0.33 0.6 0.85", style="rounded"];
+        46[label = "add_brownfield", color = "0.59 0.6 0.85", style="rounded"];
+        47[label = "plot_network", color = "0.15 0.6 0.85", style="rounded"];
+        1 -> 0
+        2 -> 1
+        3 -> 2
+        19 -> 2
+        47 -> 2
+        46 -> 3
+        19 -> 3
+        4 -> 3
+        45 -> 3
+        43 -> 4
+        19 -> 4
+        20 -> 4
+        9 -> 4
+        5 -> 4
+        25 -> 4
+        8 -> 4
+        28 -> 5
+        23 -> 5
+        11 -> 5
+        33 -> 5
+        24 -> 5
+        43 -> 5
+        19 -> 5
+        27 -> 5
+        6 -> 5
+        31 -> 5
+        32 -> 5
+        44 -> 5
+        9 -> 5
+        30 -> 5
+        25 -> 5
+        29 -> 5
+        42 -> 5
+        8 -> 5
+        7 -> 6
+        19 -> 6
+        19 -> 7
+        8 -> 7
+        9 -> 8
+        19 -> 8
+        10 -> 9
+        19 -> 9
+        16 -> 9
+        11 -> 10
+        19 -> 10
+        14 -> 10
+        20 -> 10
+        12 -> 10
+        21 -> 10
+        16 -> 10
+        13 -> 10
+        15 -> 11
+        14 -> 11
+        13 -> 11
+        12 -> 11
+        16 -> 11
+        17 -> 11
+        13 -> 12
+        14 -> 13
+        13 -> 16
+        12 -> 16
+        18 -> 17
+        12 -> 20
+        22 -> 21
+        13 -> 23
+        25 -> 24
+        23 -> 24
+        8 -> 25
+        26 -> 25
+        13 -> 26
+        13 -> 27
+        23 -> 27
+        8 -> 27
+        24 -> 28
+        25 -> 28
+        29 -> 28
+        23 -> 28
+        8 -> 29
+        26 -> 29
+        13 -> 30
+        14 -> 30
+        8 -> 30
+        8 -> 31
+        9 -> 32
+        26 -> 32
+        34 -> 33
+        36 -> 33
+        40 -> 33
+        35 -> 34
+        37 -> 36
+        38 -> 36
+        25 -> 37
+        8 -> 37
+        39 -> 38
+        35 -> 39
+        41 -> 40
+        37 -> 40
+        39 -> 41
+        35 -> 41
+        8 -> 42
+        26 -> 42
+        29 -> 43
+        8 -> 44
+        26 -> 44
+        3 -> 46
+        19 -> 46
+        5 -> 46
+        43 -> 46
+        3 -> 47
+        8 -> 47
+    }  
+
+|
+
+
+Scaling-Up
+==========
+
+If you now feel confident and want to tackle runs with larger temporal, technological and
+spatial scope, clean-up the repository and after modifying the ``config.yaml`` file
+target the collection rule ``all`` again without providing the test
+configuration file.
+
+.. code:: bash
+
+    snakemake -call purge
+    snakemake -call all
+
+.. note::
+
+    It is good practice to perform a dry-run using the option `-n`, before you
+    commit to a run:
+
+    .. code:: bash
+        snakemake -call all -n
--- a/doc/wildcards.rst
+++ b/doc/wildcards.rst
@ -15,8 +15,28 @@ which e.g. defines one particular scenario. One can think of a wildcard as a par
 up in the input/output file names of the ``Snakefile`` and thereby determines which rules to run,
 what data to retrieve and what files to produce.

-Detailed explanations of how wildcards work in ``snakemake`` can be found in the
-`relevant section of the documentation <https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#wildcards>`_.
+.. note::
+    Detailed explanations of how wildcards work in ``snakemake`` can be found in the
+    `relevant section of the documentation <https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#wildcards>`_.
+
+.. _cutout_wc:
+
+The ``{cutout}`` wildcard
+=========================
+
+The ``{cutout}`` wildcard facilitates running the rule :mod:`build_cutout`
+for all cutout configurations specified under ``atlite: cutouts:``.
+These cutouts will be stored in a folder specified by ``{cutout}``.
+
+.. _technology:
+
+The ``{technology}`` wildcard
+=============================
+
+The ``{technology}`` wildcard specifies for which renewable energy technology to produce availability time
+series and potentials using the rule :mod:`build_renewable_profiles`.
+It can take the values ``onwind``, ``offwind-ac``, ``offwind-dc``, and ``solar`` but **not** ``hydro``
+(since hydroelectric plant profiles are created by a different rule).

 .. _simpl:

@ -78,78 +98,47 @@ The wildcard, in general, consists of two parts:
 The ``{opts}`` wildcard
 =======================

-The ``{opts}`` wildcard triggers optional constraints, which are activated in either
-:mod:`prepare_network` or the :mod:`solve_network` step.
-It may hold multiple triggers separated by ``-``, i.e. ``Co2L-3H`` contains the
-``Co2L`` trigger and the ``3H`` switch. There are currently:
+The ``{opts}`` wildcard is used for electricity-only studies. It triggers
+optional constraints, which are activated in either :mod:`prepare_network` or
+the :mod:`solve_network` step. It may hold multiple triggers separated by ``-``,
+i.e. ``Co2L-3H`` contains the ``Co2L`` trigger and the ``3H`` switch. There are
+currently:


 .. csv-table::
   :header-rows: 1
-   :widths: 10,20,10,10
+   :widths: 10,20,10,10 
   :file: configtables/opts.csv

-.. _country:
+.. _sector_opts:

-The ``{country}`` wildcard
-==========================
+The ``{sector_opts}`` wildcard
+==============================

-The rules :mod:`make_summary` and :mod:`plot_summary` (generating summaries of all or a subselection
-of the solved networks) as well as :mod:`plot_p_nom_map` (for plotting the cumulative
-generation potentials for renewable technologies) can be narrowed to
-individual countries using the ``{country}`` wildcard.
+.. warning::
+    More comprehensive documentation for this wildcard will be added soon.

-If ``country=all``, then the rule acts on the network for all countries
-defined in ``config.yaml``. If otherwise ``country=DE`` or another 2-letter
-country code, then the network is narrowed to buses of this country
-for the rule. For example to get a summary of the energy generated
-in Germany (in the solution for Europe) use:
+The ``{sector_opts}`` wildcard is only used for sector-coupling studies.

-.. code:: bash
+.. csv-table::
+   :header-rows: 1
+   :widths: 10,20,10,10 
+   :file: configtables/sector-opts.csv

-    snakemake -call results/summaries/elec_s_all_lall_Co2L-3H_DE
+.. _scope:

-.. _cutout_wc:
+The ``{scope}`` wildcard
+========================

-The ``{cutout}`` wildcard
-=========================
+Takes values ``residential``, ``urban``, ``total``.

-The ``{cutout}`` wildcard facilitates running the rule :mod:`build_cutout`
-for all cutout configurations specified under ``atlite: cutouts:``.
-These cutouts will be stored in a folder specified by ``{cutout}``.
+.. _planning_horizons:

-.. _technology:
+The ``{planning_horizons}`` wildcard
+====================================

-The ``{technology}`` wildcard
-=============================
+.. warning::
+    More comprehensive documentation for this wildcard will be added soon.

-The ``{technology}`` wildcard specifies for which renewable energy technology to produce availability time
-series and potentials using the rule :mod:`build_renewable_profiles`.
-It can take the values ``onwind``, ``offwind-ac``, ``offwind-dc``, and ``solar`` but **not** ``hydro``
-(since hydroelectric plant profiles are created by a different rule).
-
-.. _attr:
-
-The ``{attr}`` wildcard
-=======================
-
-The ``{attr}`` wildcard specifies which attribute is used for size
-representations of network components on a map plot produced by the rule
-:mod:`plot_network`. While it might be extended in the future, ``{attr}``
-currently only supports plotting of ``p_nom``.
-
-.. _ext:
-
-The ``{ext}`` wildcard
-======================
-
-The ``{ext}`` wildcard specifies the file type of the figures the
-rule :mod:`plot_network` and :mod:`plot_summary` produce.
-Typical examples are ``pdf`` and ``png``. The list of supported file
-formats depends on the used backend. To query the supported file types on your system, issue:
-
-.. code:: python
-
-    import matplotlib.pyplot as plt
-
-    plt.gcf().canvas.get_supported_filetypes()
+The ``{planning_horizons}`` wildcard is only used for sector-coupling studies.
+It takes years as values, e.g. 2020, 2030, 2040, 2050.
--- a/rules/collect.smk
+++ b/rules/collect.smk
@ -5,13 +5,13 @@

 localrules:
    all,
-    cluster_all_networks,
-    extra_components_all_networks,
-    prepare_all_networks,
+    cluster_networks,
+    extra_components_networks,
+    prepare_elec_networks,
    prepare_sector_networks,
-    solve_all_elec_networks,
-    solve_all_networks,
-    plot_all_networks,
+    solve_elec_networks,
+    solve_sector_networks,
+    plot_networks,


 rule all:
@ -20,19 +20,19 @@ rule all:
    default_target: True


-rule cluster_all_networks:
+rule cluster_networks:
    input:
        expand(RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", **config["scenario"]),


-rule extra_components_all_networks:
+rule extra_components_networks:
    input:
        expand(
            RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", **config["scenario"]
        ),


-rule prepare_all_networks:
+rule prepare_elec_networks:
    input:
        expand(
            RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
@ -49,7 +49,7 @@ rule prepare_sector_networks:
        ),


-rule solve_all_elec_networks:
+rule solve_elec_networks:
    input:
        expand(
            RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
@ -57,7 +57,7 @@ rule solve_all_elec_networks:
        ),


-rule solve_all_networks:
+rule solve_sector_networks:
    input:
        expand(
            RESULTS
@ -66,7 +66,7 @@ rule solve_all_networks:
        ),


-rule plot_all_networks:
+rule plot_networks:
    input:
        expand(
            RESULTS
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@ -93,7 +93,7 @@ Description
        do not work reliably with multiple voltage levels and transformers.

 .. tip::
-    The rule :mod:`cluster_all_networks` runs
+    The rule :mod:`cluster_networks` runs
    for all ``scenario`` s in the configuration file
    the rule :mod:`cluster_network`.

--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@ -52,7 +52,7 @@ Description
 -----------

 .. tip::
-    The rule :mod:`prepare_all_networks` runs
+    The rule :mod:`prepare_elec_networks` runs
    for all ``scenario`` s in the configuration file
    the rule :mod:`prepare_network`.
 """
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@ -10,43 +10,6 @@ iteratively optimize while updating line reactances.
 This script is used for optimizing the electrical network as well as the
 sector coupled network.

-Relevant Settings
-----------------
-
-.. code:: yaml
-
-    solving:
-        options:
-            formulation:
-            clip_p_max_pu:
-            load_shedding:
-            noisy_costs:
-            nhours:
-            min_iterations:
-            max_iterations:
-            skip_iterations:
-            track_iterations:
-        solver:
-            name:
-            options:
-
-.. seealso::
-    Documentation of the configuration file ``config.yaml`` at
-    :ref:`electricity_cf`, :ref:`solving_cf`, :ref:`plotting_cf`
-
-Inputs
------
-
- ``networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: confer :ref:`prepare`
-
-Outputs
-------
-
- ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: Solved PyPSA network including optimisation results
-
-    .. image:: img/results.png
-        :scale: 40 %
-
 Description
 -----------

@ -55,31 +18,14 @@ linear optimal power flow (plus investment planning
 is provided in the
 `documentation of PyPSA <https://pypsa.readthedocs.io/en/latest/optimal_power_flow.html#linear-optimal-power-flow>`_.

-The optimization is based on the ``pyomo=False`` setting in the :func:`network.lopf` and  :func:`pypsa.linopf.ilopf` function.
-Additionally, some extra constraints specified in :mod:`prepare_network` are added.
+The optimization is based on the :func:`network.optimize` function.
+Additionally, some extra constraints specified in :mod:`solve_network` are added.

-Solving the network in multiple iterations is motivated through the dependence of transmission line capacities and impedances.
-As lines are expanded their electrical parameters change, which renders the optimisation bilinear even if the power flow
-equations are linearized.
-To retain the computational advantage of continuous linear programming, a sequential linear programming technique
-is used, where in between iterations the line impedances are updated.
-Details (and errors made through this heuristic) are discussed in the paper
+.. note::

- Fabian Neumann and Tom Brown. `Heuristics for Transmission Expansion Planning in Low-Carbon Energy System Models <https://arxiv.org/abs/1907.10548>`_), *16th International Conference on the European Energy Market*, 2019. `arXiv:1907.10548 <https://arxiv.org/abs/1907.10548>`_.
-
-.. warning::
-
-    Capital costs of existing network components are not included in the objective function,
-    since for the optimisation problem they are just a constant term (no influence on optimal result).
-
-    Therefore, these capital costs are not included in ``network.objective``!
-    If you want to calculate the full total annual system costs add these to the objective value.
-
-.. tip::
-
-    The rule :mod:`solve_all_networks` runs
-    for all ``scenario`` s in the configuration file
-    the rule :mod:`solve_network`.
+    The rules ``solve_elec_networks`` and ``solve_sector_networks`` run
+    the workflow for all scenarios in the configuration file (``scenario:``)
+    based on the rule :mod:`solve_network`.
 """
 import logging
 import re
--- a/scripts/solve_operations_network.py
+++ b/scripts/solve_operations_network.py
@ -6,43 +6,6 @@
 """
 Solves linear optimal dispatch in hourly resolution using the capacities of
 previous capacity expansion in rule :mod:`solve_network`.
-
-Relevant Settings
-----------------
-
-.. code:: yaml
-
-    solving:
-        tmpdir:
-        options:
-            formulation:
-            clip_p_max_pu:
-            load_shedding:
-            noisy_costs:
-            nhours:
-            min_iterations:
-            max_iterations:
-        solver:
-            name:
-            (solveroptions):
-
-.. seealso::
-    Documentation of the configuration file ``config.yaml`` at
-    :ref:`solving_cf`
-
-Inputs
------
-
- ``networks/elec_s{simpl}_{clusters}.nc``: confer :ref:`cluster`
- ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc``: confer :ref:`solve`
-
-Outputs
-------
-
- ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc``: Solved PyPSA network for optimal dispatch including optimisation results
-
-Description
-----------
 """

 import logging
@ -66,7 +29,7 @@ if __name__ == "__main__":

        snakemake = mock_snakemake(
            "solve_operations_network",
-            configfiles="test/config.test1.yaml",
+            configfiles="test/config.electricity.yaml",
            simpl="",
            opts="",
            clusters="5",
--- a/test/config.electricity.yaml
+++ b/test/config.electricity.yaml
--- a/test/config.myopic.yaml
+++ b/test/config.myopic.yaml
@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: CC0-1.0

+tutorial: true

 run:
  name: "test-sector-myopic"
@ -53,6 +54,12 @@ renewable:
  solar:
    cutout: be-03-2013-era5

+industry:
+  St_primary_fraction:
+    2030: 0.6
+    2040: 0.5
+    2050: 0.4
+
 solving:
  solver:
    name: glpk
--- a/test/config.overnight.yaml
+++ b/test/config.overnight.yaml
@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: CC0-1.0

+tutorial: true

 run:
  name: "test-sector-overnight"