Merge branch 'master' into fneum/ariadne2

.pre-commit-config.yaml

@@ -87,6 +87,6 @@ repos:
 
   # Check for FSFE REUSE compliance (licensing)
 - repo: https://github.com/fsfe/reuse-tool
-  rev: v2.1.0
+  rev: v3.0.1
   hooks:
   - id: reuse

Snakefile

@@ -13,9 +13,10 @@ from snakemake.utils import min_version
 
 min_version("7.7")
 
-
+conf_file = os.path.join(workflow.current_basedir, "config/config.yaml")
-if not exists("config/config.yaml") and exists("config/config.default.yaml"):
+conf_default_file = os.path.join(workflow.current_basedir, "config/config.default.yaml")
-    copyfile("config/config.default.yaml", "config/config.yaml")
+if not exists(conf_file) and exists(conf_default_file):
+    copyfile(conf_default_file, conf_file)
 
 
 configfile: "config/config.yaml"

config/config.default.yaml

@@ -44,7 +44,7 @@ scenario:
   opts:
   - ''
   sector_opts:
-  - Co2L0-3H-T-H-B-I-A-solar+p3-dist1
+  - Co2L0-3H-T-H-B-I-A-dist1
   planning_horizons:
   # - 2020
   # - 2030
@@ -687,6 +687,7 @@ solving:
     rolling_horizon: false
     seed: 123
     custom_extra_functionality: "../data/custom_extra_functionality.py"
+    # io_api: "direct"  # Increases performance but only supported for the highs and gurobi solvers
     # options that go into the optimize function
     track_iterations: false
     min_iterations: 4
@@ -772,6 +773,13 @@ plotting:
     color_geomap:
       ocean: white
       land: white
+  projection:
+    name: "EqualEarth"
+    # See https://scitools.org.uk/cartopy/docs/latest/reference/projections.html for alternatives, for example:
+    # name: "LambertConformal"
+    # central_longitude: 10.
+    # central_latitude: 50.
+    # standard_parallels: [35, 65]
   eu_node_location:
     x: -5.5
     y: 46.

config/config.perfect.yaml

@@ -19,9 +19,9 @@ scenario:
   opts:
   - ''
   sector_opts:
-  - 1p5-4380H-T-H-B-I-A-solar+p3-dist1
+  - 1p5-4380H-T-H-B-I-A-dist1
-  - 1p7-4380H-T-H-B-I-A-solar+p3-dist1
+  - 1p7-4380H-T-H-B-I-A-dist1
-  - 2p0-4380H-T-H-B-I-A-solar+p3-dist1
+  - 2p0-4380H-T-H-B-I-A-dist1
   planning_horizons:
   - 2020
   - 2025

config/test/config.myopic.yaml

@@ -18,7 +18,7 @@ scenario:
   clusters:
   - 5
   sector_opts:
-  - 24H-T-H-B-I-A-solar+p3-dist1
+  - 24H-T-H-B-I-A-dist1
   planning_horizons:
   - 2030
   - 2040

config/test/config.overnight.yaml

@@ -17,7 +17,7 @@ scenario:
   clusters:
   - 5
   sector_opts:
-  - CO2L0-24H-T-H-B-I-A-solar+p3-dist1
+  - CO2L0-24H-T-H-B-I-A-dist1
   planning_horizons:
   - 2030
 

config/test/config.perfect.yaml

@@ -18,7 +18,7 @@ scenario:
   clusters:
   - 5
   sector_opts:
-  - 8760H-T-H-B-I-A-solar+p3-dist1
+  - 8760H-T-H-B-I-A-dist1
   planning_horizons:
   - 2030
   - 2040

doc/configtables/plotting.csv

@@ -1,6 +1,9 @@
 ,Unit,Values,Description
 map,,,
 -- boundaries,°,"[x1,x2,y1,y2]",Boundaries of the map plots in degrees latitude (y) and longitude (x)
+projection,,,,
+-- name,--,"Valid Cartopy projection name","See https://scitools.org.uk/cartopy/docs/latest/reference/projections.html for list of available projections."
+-- args,--,--,"Other entries under 'projection' are passed as keyword arguments to the projection constructor, e.g. ``central_longitude: 10.``."
 costs_max,bn Euro,float,Upper y-axis limit in cost bar plots.
 costs_threshold,bn Euro,float,Threshold below which technologies will not be shown in cost bar plots.
 energy_max,TWh,float,Upper y-axis limit in energy bar plots.

doc/configtables/snapshots.csv

@@ -2,5 +2,5 @@
 start,--,str or datetime-like; e.g. YYYY-MM-DD,Left bound of date range
 end,--,str or datetime-like; e.g. YYYY-MM-DD,Right bound of date range
 inclusive,--,"One of {'neither', 'both', ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``both`` or neither side ``None``."
-resolution ,--,"{false,``nH``; i.e. ``2H``-``6H``}",Resample the time-resolution by averaging over every ``n`` snapshots
+resolution ,--,"{false,``nH``; i.e. ``2H``-``6H``}","Resample the time-resolution by averaging over every ``n`` snapshots in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks."
-segmentation,--,"{false,``n``; e.g. ``4380``}","Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load."
+segmentation,--,"{false,``n``; e.g. ``4380``}","Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks."

doc/configtables/solving.csv

@@ -7,6 +7,7 @@ options,,,
 -- rolling_horizon,bool,"{'true','false'}","Whether to optimize the network in a rolling horizon manner, where the snapshot range is split into slices of size `horizon` which are solved consecutively."
 -- seed,--,int,Random seed for increased deterministic behaviour.
 -- custom_extra_functionality,--,str,Path to a Python file with custom extra functionality code to be injected into the solving rules of the workflow relative to ``rules`` directory.
+-- io_api,string,"{'lp','mps','direct'}",Passed to linopy and determines the API used to communicate with the solver. With the ``'lp'`` and ``'mps'`` options linopy passes a file to the solver; with the ``'direct'`` option (only supported for HIGHS and Gurobi) linopy uses an in-memory python API resulting in better performance.
 -- track_iterations,bool,"{'true','false'}",Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration)
 -- min_iterations,--,int,Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.
 -- max_iterations,--,int,Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.

doc/release_notes.rst

@@ -48,6 +48,15 @@ Upcoming Release
 
 * Bugfix: Correctly read out number of solver threads from configuration file.
 
+* Air-sourced heat pumps can now also be built in rural areas. Previously, only
+  ground-sourced heat pumps were considered for this category.
+
+* Bugfix: Correctly read out number of solver threads from configuration file.
+
+* Add support for the linopy ``io_api`` option; set to ``"direct"`` to increase model reading and writing performance for the highs and gurobi solvers.
+
+* Add the option to customise map projection in plotting config.
+
 
 PyPSA-Eur 0.9.0 (5th January 2024)
 ==================================

envs/environment.yaml

@@ -26,7 +26,7 @@ dependencies:
 - yaml
 - pytables
 - lxml
-- powerplantmatching>=0.5.5
+- powerplantmatching>=0.5.5,!=0.5.9
 - numpy
 - pandas>=2.1
 - geopandas>=0.11.0

envs/retrieve.yaml

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+name: pypsa-eur-retrieve
+channels:
+- conda-forge
+- bioconda
+dependencies:
+- python>=3.8
+- snakemake-minimal>=7.7.0,<8.0.0
+- pandas>=2.1
+- tqdm

rules/common.smk

@@ -43,7 +43,7 @@ def memory(w):
 def input_custom_extra_functionality(w):
     path = config["solving"]["options"].get("custom_extra_functionality", False)
     if path:
-        return workflow.source_path(path)
+        return os.path.join(os.path.dirname(workflow.snakefile), path)
     return []
 
 

rules/retrieve.smk

@@ -37,7 +37,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle",
             mem_mb=1000,
         retries: 2
         conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
         script:
             "../scripts/retrieve_databundle.py"
 
@@ -55,7 +55,7 @@ if config["enable"].get("retrieve_irena"):
             mem_mb=1000,
         retries: 2
         conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
         script:
             "../scripts/retrieve_irena.py"
 
@@ -157,7 +157,7 @@ if config["enable"]["retrieve"] and config["enable"].get(
             LOGS + "retrieve_sector_databundle.log",
         retries: 2
         conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
         script:
             "../scripts/retrieve_sector_databundle.py"
 
@@ -180,7 +180,7 @@ if config["enable"]["retrieve"]:
             LOGS + "retrieve_gas_infrastructure_data.log",
         retries: 2
         conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
         script:
             "../scripts/retrieve_gas_infrastructure_data.py"
 
@@ -376,6 +376,6 @@ if config["enable"]["retrieve"]:
             mem_mb=5000,
         retries: 2
         conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
         script:
             "../scripts/retrieve_monthly_fuel_prices.py"

scripts/add_brownfield.py

@@ -153,7 +153,7 @@ if __name__ == "__main__":
             clusters="37",
             opts="",
             ll="v1.0",
-            sector_opts="168H-T-H-B-I-solar+p3-dist1",
+            sector_opts="168H-T-H-B-I-dist1",
             planning_horizons=2030,
         )
 

scripts/add_existing_baseyear.py

@@ -561,7 +561,7 @@ if __name__ == "__main__":
             clusters="37",
             ll="v1.0",
             opts="",
-            sector_opts="1p7-4380H-T-H-B-I-A-solar+p3-dist1",
+            sector_opts="1p7-4380H-T-H-B-I-A-dist1",
             planning_horizons=2020,
         )
 

scripts/cluster_gas_network.py

@@ -75,10 +75,10 @@ def build_clustered_gas_network(df, bus_regions, length_factor=1.25):
     return df
 
 
-def reindex_pipes(df):
+def reindex_pipes(df, prefix="gas pipeline"):
     def make_index(x):
         connector = " <-> " if x.bidirectional else " -> "
-        return "gas pipeline " + x.bus0 + connector + x.bus1
+        return prefix + " " + x.bus0 + connector + x.bus1
 
     df.index = df.apply(make_index, axis=1)
 

scripts/plot_network.py

@@ -170,7 +170,7 @@ def plot_map(
     line_widths = line_widths.replace(line_lower_threshold, 0)
     link_widths = link_widths.replace(line_lower_threshold, 0)
 
-    fig, ax = plt.subplots(subplot_kw={"projection": ccrs.EqualEarth()})
+    fig, ax = plt.subplots(subplot_kw={"projection": proj})
     fig.set_size_inches(7, 6)
 
     n.plot(
@@ -358,7 +358,6 @@ def plot_h2_map(network, regions):
     n.links.bus0 = n.links.bus0.str.replace(" H2", "")
     n.links.bus1 = n.links.bus1.str.replace(" H2", "")
 
-    proj = ccrs.EqualEarth()
     regions = regions.to_crs(proj.proj4_init)
 
     fig, ax = plt.subplots(figsize=(7, 6), subplot_kw={"projection": proj})
@@ -568,7 +567,7 @@ def plot_ch4_map(network):
         "biogas": "seagreen",
     }
 
-    fig, ax = plt.subplots(figsize=(7, 6), subplot_kw={"projection": ccrs.EqualEarth()})
+    fig, ax = plt.subplots(figsize=(7, 6), subplot_kw={"projection": proj})
 
     n.plot(
         bus_sizes=bus_sizes,
@@ -679,7 +678,7 @@ def plot_map_without(network):
     # Drop non-electric buses so they don't clutter the plot
     n.buses.drop(n.buses.index[n.buses.carrier != "AC"], inplace=True)
 
-    fig, ax = plt.subplots(figsize=(7, 6), subplot_kw={"projection": ccrs.EqualEarth()})
+    fig, ax = plt.subplots(figsize=(7, 6), subplot_kw={"projection": proj})
 
     # PDF has minimum width, so set these to zero
     line_lower_threshold = 200.0
@@ -993,7 +992,7 @@ def plot_map_perfect(
     link_widths[link_widths > line_upper_threshold] = line_upper_threshold
 
     for year in costs.columns:
-        fig, ax = plt.subplots(subplot_kw={"projection": ccrs.PlateCarree()})
+        fig, ax = plt.subplots(subplot_kw={"projection": proj})
         fig.set_size_inches(7, 6)
         fig.suptitle(year)
 
@@ -1068,7 +1067,7 @@ if __name__ == "__main__":
             opts="",
             clusters="37",
             ll="v1.0",
-            sector_opts="4380H-T-H-B-I-A-solar+p3-dist1",
+            sector_opts="4380H-T-H-B-I-A-dist1",
         )
 
     logging.basicConfig(level=snakemake.config["logging"]["level"])
@@ -1082,6 +1081,10 @@ if __name__ == "__main__":
     if map_opts["boundaries"] is None:
         map_opts["boundaries"] = regions.total_bounds[[0, 2, 1, 3]] + [-1, 1, -1, 1]
 
+    proj_kwargs = snakemake.params.plotting.get("projection", dict(name="EqualEarth"))
+    proj_func = getattr(ccrs, proj_kwargs.pop("name"))
+    proj = proj_func(**proj_kwargs)
+
     if snakemake.params["foresight"] == "perfect":
         plot_map_perfect(
             n,

scripts/plot_summary.py

@@ -282,7 +282,10 @@ def plot_balances():
         # remove trailing link ports
         df.index = [
             i[:-1]
-            if ((i not in ["co2", "NH3", "H2"]) and (i[-1:] in ["0", "1", "2", "3"]))
+            if (
+                (i not in ["co2", "NH3", "H2"])
+                and (i[-1:] in ["0", "1", "2", "3", "4"])
+            )
             else i
             for i in df.index
         ]

scripts/prepare_perfect_foresight.py

@@ -503,7 +503,7 @@ if __name__ == "__main__":
             opts="",
             clusters="37",
             ll="v1.5",
-            sector_opts="1p7-4380H-T-H-B-I-A-solar+p3-dist1",
+            sector_opts="1p7-4380H-T-H-B-I-A-dist1",
         )
 
     update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)

scripts/prepare_sector_network.py

@@ -1321,7 +1321,7 @@ def add_storage_and_grids(n, costs):
 
         h2_pipes["p_nom"] = 0.0
 
-        if "custom_h2_pipelines" in snakemake.input:
+        if snakemake.input.get("custom_h2_pipelines"):
             fn = snakemake.input.custom_h2_pipelines
             custom_pipes = pd.read_csv(fn, index_col=0)
 
@@ -1795,8 +1795,9 @@ def add_heat(n, costs):
 
         ## Add heat pumps
 
-        heat_pump_type = "air" if "urban" in name else "ground"
+        heat_pump_types = ["air"] if "urban" in name else ["ground", "air"]
 
+        for heat_pump_type in heat_pump_types:
             costs_name = f"{name_type} {heat_pump_type}-sourced heat pump"
             efficiency = (
                 cop[heat_pump_type][nodes]
@@ -3568,7 +3569,7 @@ if __name__ == "__main__":
             opts="",
             clusters="37",
             ll="v1.0",
-            sector_opts="CO2L0-24H-T-H-B-I-A-solar+p3-dist1",
+            sector_opts="CO2L0-24H-T-H-B-I-A-dist1",
             planning_horizons="2030",
         )
 

scripts/solve_network.py

@@ -890,6 +890,7 @@ def solve_network(n, config, solving, opts="", **kwargs):
         "linearized_unit_commitment", False
     )
     kwargs["assign_all_duals"] = cf_solving.get("assign_all_duals", False)
+    kwargs["io_api"] = cf_solving.get("io_api", None)
 
     if kwargs["solver_name"] == "gurobi":
         logging.getLogger("gurobipy").setLevel(logging.CRITICAL)
@@ -943,7 +944,7 @@ if __name__ == "__main__":
             opts="",
             clusters="37",
             ll="v1.0",
-            sector_opts="CO2L0-1H-T-H-B-I-A-solar+p3-dist1",
+            sector_opts="CO2L0-1H-T-H-B-I-A-dist1",
             planning_horizons="2030",
         )
     configure_logging(snakemake)