Merge branch 'master' into dedicated-shared-resource

.pre-commit-config.yaml

@@ -51,7 +51,7 @@ repos:
 
   # Formatting with "black" coding style
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 24.2.0
+  rev: 24.3.0
   hooks:
       # Format Python files
   - id: black

Snakefile

@@ -56,9 +56,9 @@ localrules:
 wildcard_constraints:
     simpl="[a-zA-Z0-9]*",
     clusters="[0-9]+(m|c)?|all",
-    ll="(v|c)([0-9\.]+|opt)",
+    ll=r"(v|c)([0-9\.]+|opt)",
-    opts="[-+a-zA-Z0-9\.]*",
+    opts=r"[-+a-zA-Z0-9\.]*",
-    sector_opts="[-+a-zA-Z0-9\.\s]*",
+    sector_opts=r"[-+a-zA-Z0-9\.\s]*",
 
 
 include: "rules/common.smk"
@@ -127,7 +127,7 @@ rule dag:
     conda:
         "envs/environment.yaml"
     shell:
-        """
+        r"""
         snakemake --rulegraph all | sed -n "/digraph/,\$p" > {output.dot}
         dot -Tpdf -o {output.pdf} {output.dot}
         dot -Tpng -o {output.png} {output.dot}

doc/configtables/costs.csv

@@ -1,6 +1,6 @@
 ,Unit,Values,Description
 year,--,YYYY; e.g. '2030',Year for which to retrieve cost assumptions of ``resources/costs.csv``.
-version,--,vX.X.X; e.g. 'v0.5.0',Version of ``technology-data`` repository to use.
+version,--,vX.X.X or <user>/<repo>/vX.X.X; e.g. 'v0.5.0',Version of ``technology-data`` repository to use. If this string is of the form <user>/<repo>/<version> then costs are instead retrieved from ``github.com/<user>/<repo>`` at the <version> tag.
 rooftop_share,--,float,Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV).
 social_discountrate,p.u.,float,Social discount rate to compare costs in different investment periods. 0.02 corresponds to a social discount rate of 2%.
 fill_values,--,float,Default values if not specified for a technology in ``resources/costs.csv``.

doc/release_notes.rst

@@ -82,6 +82,8 @@ Upcoming Release
 
 * Upgrade default techno-economic assumptions to ``technology-data`` v0.8.1.
 
+* Add possibility to download cost data from custom fork of ``technology-data``.
+
 * Linearly interpolate missing investment periods in year-dependent
   configuration options.
 

rules/retrieve.smk

@@ -50,7 +50,7 @@ if config["enable"].get("retrieve_irena"):
             onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv",
             solar="data/existing_infrastructure/solar_capacity_IRENA.csv",
         log:
-            logs("retrieve_irena.log"),
+            "logs/retrieve_irena.log",
         resources:
             mem_mb=1000,
         retries: 2
@@ -115,7 +115,7 @@ if config["enable"]["retrieve"] and config["enable"].get(
             mem_mb=5000,
         retries: 2
         run:
-            move(input[0], output[0])
+            copyfile(input[0], output[0])
             validate_checksum(output[0], input[0])
 
 

rules/solve_electricity.smk

@@ -31,7 +31,7 @@ rule solve_network:
         mem_mb=memory,
         runtime=config_provider("solving", "runtime", default="6h"),
     shadow:
-        "minimal"
+        "shallow"
     conda:
         "../envs/environment.yaml"
     script:
@@ -62,7 +62,7 @@ rule solve_operations_network:
         mem_mb=(lambda w: 10000 + 372 * int(w.clusters)),
         runtime=config_provider("solving", "runtime", default="6h"),
     shadow:
-        "minimal"
+        "shallow"
     conda:
         "../envs/environment.yaml"
     script:

scripts/add_brownfield.py

@@ -145,7 +145,7 @@ def disable_grid_expansion_if_limit_hit(n):
         for name, glc in glcs.iterrows():
             total_expansion = (
                 (
-                    n.lines.query("p_nom_extendable")
+                    n.lines.query("s_nom_extendable")
                     .eval(f"s_nom_min * {cols[limit_type]}")
                     .sum()
                 )

scripts/add_electricity.py

@@ -184,12 +184,13 @@ def sanitize_carriers(n, config):
 
 
 def sanitize_locations(n):
-    n.buses["x"] = n.buses.x.where(n.buses.x != 0, n.buses.location.map(n.buses.x))
+    if "location" in n.buses.columns:
-    n.buses["y"] = n.buses.y.where(n.buses.y != 0, n.buses.location.map(n.buses.y))
+        n.buses["x"] = n.buses.x.where(n.buses.x != 0, n.buses.location.map(n.buses.x))
-    n.buses["country"] = n.buses.country.where(
+        n.buses["y"] = n.buses.y.where(n.buses.y != 0, n.buses.location.map(n.buses.y))
-        n.buses.country.ne("") & n.buses.country.notnull(),
+        n.buses["country"] = n.buses.country.where(
-        n.buses.location.map(n.buses.country),
+            n.buses.country.ne("") & n.buses.country.notnull(),
-    )
+            n.buses.location.map(n.buses.country),
+        )
 
 
 def add_co2_emissions(n, costs, carriers):

scripts/build_hydro_profile.py

@@ -202,9 +202,10 @@ if __name__ == "__main__":
 
     contained_years = pd.date_range(freq="YE", **snakemake.params.snapshots).year
     norm_year = config_hydro.get("eia_norm_year")
+    missing_years = contained_years.difference(eia_stats.index)
     if norm_year:
         eia_stats.loc[contained_years] = eia_stats.loc[norm_year]
-    elif missing_years := eia_stats.index.difference(contained_years):
+    elif missing_years.any():
         eia_stats.loc[missing_years] = eia_stats.median()
 
     inflow = cutout.runoff(

scripts/prepare_perfect_foresight.py

@@ -537,5 +537,8 @@ if __name__ == "__main__":
     # set carbon constraints
     n = set_carbon_constraints(n)
 
+    # update meta
+    n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
+
     # export network
     n.export_to_netcdf(snakemake.output[0])

scripts/retrieve_cost_data.py

@@ -25,9 +25,10 @@ if __name__ == "__main__":
     set_scenario_config(snakemake)
 
     version = snakemake.params.version
-    baseurl = (
+    if "/" in version:
-        f"https://raw.githubusercontent.com/PyPSA/technology-data/{version}/outputs/"
+        baseurl = f"https://raw.githubusercontent.com/{version}/outputs/"
-    )
+    else:
+        baseurl = f"https://raw.githubusercontent.com/PyPSA/technology-data/{version}/outputs/"
     filepath = Path(snakemake.output[0])
     url = baseurl + filepath.name
 

test.sh

@@ -2,8 +2,12 @@
 #
 # SPDX-License-Identifier: CC0-1.0
 
+set -x && \
+
 snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime && \
 snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime && \
 snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime && \
 snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime && \
-snakemake -call all --configfile config/test/config.scenarios.yaml --rerun-triggers=mtime -n
+snakemake -call all --configfile config/test/config.scenarios.yaml --rerun-triggers=mtime -n && \
+
+set +x