Merge branch 'master' into fill-missing-domestic-aviation
This commit is contained in:
commit
f774a1628d
2
.github/workflows/ci.yaml
vendored
2
.github/workflows/ci.yaml
vendored
@ -31,7 +31,7 @@ jobs:
|
|||||||
os:
|
os:
|
||||||
- ubuntu-latest
|
- ubuntu-latest
|
||||||
- macos-latest
|
- macos-latest
|
||||||
- windows-latest
|
# - windows-latest
|
||||||
inhouse:
|
inhouse:
|
||||||
- stable
|
- stable
|
||||||
- master
|
- master
|
||||||
|
39
Snakefile
39
Snakefile
@ -2,26 +2,18 @@
|
|||||||
#
|
#
|
||||||
# SPDX-License-Identifier: MIT
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
from os.path import normpath, exists
|
|
||||||
from shutil import copyfile, move, rmtree
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import yaml
|
import yaml
|
||||||
|
from os.path import normpath, exists
|
||||||
|
from shutil import copyfile, move, rmtree
|
||||||
from snakemake.utils import min_version
|
from snakemake.utils import min_version
|
||||||
|
|
||||||
min_version("8.5")
|
min_version("8.5")
|
||||||
|
|
||||||
from scripts._helpers import path_provider
|
from scripts._helpers import path_provider, copy_default_files, get_scenarios, get_rdir
|
||||||
|
|
||||||
default_files = {
|
|
||||||
"config/config.default.yaml": "config/config.yaml",
|
copy_default_files(workflow)
|
||||||
"config/scenarios.template.yaml": "config/scenarios.yaml",
|
|
||||||
}
|
|
||||||
for template, target in default_files.items():
|
|
||||||
target = os.path.join(workflow.current_basedir, target)
|
|
||||||
template = os.path.join(workflow.current_basedir, template)
|
|
||||||
if not exists(target) and exists(template):
|
|
||||||
copyfile(template, target)
|
|
||||||
|
|
||||||
|
|
||||||
configfile: "config/config.default.yaml"
|
configfile: "config/config.default.yaml"
|
||||||
@ -29,17 +21,8 @@ configfile: "config/config.yaml"
|
|||||||
|
|
||||||
|
|
||||||
run = config["run"]
|
run = config["run"]
|
||||||
scenarios = run.get("scenarios", {})
|
scenarios = get_scenarios(run)
|
||||||
if run["name"] and scenarios.get("enable"):
|
RDIR = get_rdir(run)
|
||||||
fn = Path(scenarios["file"])
|
|
||||||
scenarios = yaml.safe_load(fn.read_text())
|
|
||||||
RDIR = "{run}/"
|
|
||||||
if run["name"] == "all":
|
|
||||||
config["run"]["name"] = list(scenarios.keys())
|
|
||||||
elif run["name"]:
|
|
||||||
RDIR = run["name"] + "/"
|
|
||||||
else:
|
|
||||||
RDIR = ""
|
|
||||||
|
|
||||||
logs = path_provider("logs/", RDIR, run["shared_resources"])
|
logs = path_provider("logs/", RDIR, run["shared_resources"])
|
||||||
benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"])
|
benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"])
|
||||||
@ -56,9 +39,9 @@ localrules:
|
|||||||
wildcard_constraints:
|
wildcard_constraints:
|
||||||
simpl="[a-zA-Z0-9]*",
|
simpl="[a-zA-Z0-9]*",
|
||||||
clusters="[0-9]+(m|c)?|all",
|
clusters="[0-9]+(m|c)?|all",
|
||||||
ll="(v|c)([0-9\.]+|opt)",
|
ll=r"(v|c)([0-9\.]+|opt)",
|
||||||
opts="[-+a-zA-Z0-9\.]*",
|
opts=r"[-+a-zA-Z0-9\.]*",
|
||||||
sector_opts="[-+a-zA-Z0-9\.\s]*",
|
sector_opts=r"[-+a-zA-Z0-9\.\s]*",
|
||||||
|
|
||||||
|
|
||||||
include: "rules/common.smk"
|
include: "rules/common.smk"
|
||||||
@ -127,7 +110,7 @@ rule dag:
|
|||||||
conda:
|
conda:
|
||||||
"envs/environment.yaml"
|
"envs/environment.yaml"
|
||||||
shell:
|
shell:
|
||||||
"""
|
r"""
|
||||||
snakemake --rulegraph all | sed -n "/digraph/,\$p" > {output.dot}
|
snakemake --rulegraph all | sed -n "/digraph/,\$p" > {output.dot}
|
||||||
dot -Tpdf -o {output.pdf} {output.dot}
|
dot -Tpdf -o {output.pdf} {output.dot}
|
||||||
dot -Tpng -o {output.png} {output.dot}
|
dot -Tpng -o {output.png} {output.dot}
|
||||||
|
@ -26,3 +26,16 @@ NordBalt,Klaipeda (LT),Nybro (SE),450,,700,built,,https://en.wikipedia.org/wiki/
|
|||||||
Estlink 1,Harku (EE),Espoo (FI),105,,350,built,,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889
|
Estlink 1,Harku (EE),Espoo (FI),105,,350,built,,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889
|
||||||
Greenlink,Waterford (IE),Pembroke (UK),,180,500,under construction,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/286,-6.987,52.260,-4.986,51.686
|
Greenlink,Waterford (IE),Pembroke (UK),,180,500,under construction,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/286,-6.987,52.260,-4.986,51.686
|
||||||
Celtic Interconnector,Aghada (IE),La Martyre (FR),,572,700,under consideration,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/107,-8.16642,51.91413,-4.184,48.459
|
Celtic Interconnector,Aghada (IE),La Martyre (FR),,572,700,under consideration,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/107,-8.16642,51.91413,-4.184,48.459
|
||||||
|
GiLA,Bordeaux (FR),Nantes (FR),,312,640,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-1.209,46.901,-0.576,44.960
|
||||||
|
HG North Tyrrhenian Corridor,Milan (IT),Viterbo (IT),,500,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,9.409,45.553,12.015,42.244
|
||||||
|
HG Adriatic Corridor,Ferrara (IT),Foggia (IT),,582,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,11.661,44.855,15.550,41.513
|
||||||
|
SAPEI 2,Fioumesanto (IT),Montalto (IT),,390,1000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,8.283,40.790,11.602,42.331
|
||||||
|
HG Ionian-Tyrrhenian Corridor,Rossano (IT),Latina (IT),,496,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,16.629,39.568,12.779,41.430
|
||||||
|
HG Ionian-Tyrrhenian Corridor 2,Rossano (IT),Catania (IT),,330,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,16.629,39.568,15.049,37.408
|
||||||
|
Germany-UK Hybrid Interconnector,Fetteresso (UK),Emden (DE),800,,2000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-2.383,56.991,7.207,53.376
|
||||||
|
NU-Link Interconnector,Hornsea (UK),Moerdijk (NL),,460,1200,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-0.261,53.655,4.586,51.661
|
||||||
|
APOLLO-LINK,La Farga (ES),La Spezia (IT),,725,2091,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,2.883,42.062,9.884,44.107
|
||||||
|
Baltic WindConnector (BWC),Lubmin (DE),Lihula (EE),,960,2000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,13.686,54.139,23.818,58.675
|
||||||
|
High-Voltage Direct Current Interconnector Project Romania-Hungary,Constanta (RO),Albertirsa (HU),,930,2500,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,28.588,44.201,19.584,47.224
|
||||||
|
Rhine-Main-Link,Ovelgönne (DE),Marxheim (DE),,433,4000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,8.379,53.315,8.435,50.078
|
||||||
|
Green Aegean Interconnector,Arachthos (GR),Ottenhofen (DE),,600,3000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,20.967,39.185,11.868,48.207
|
||||||
|
|
@ -10,6 +10,8 @@ Release Notes
|
|||||||
Upcoming Release
|
Upcoming Release
|
||||||
================
|
================
|
||||||
|
|
||||||
|
* Allow the use of more solvers in clustering (Xpress, COPT, Gurobi, CPLEX, SCIP, MOSEK).
|
||||||
|
|
||||||
* Enhanced support for choosing different weather years
|
* Enhanced support for choosing different weather years
|
||||||
(https://github.com/PyPSA/pypsa-eur/pull/204):
|
(https://github.com/PyPSA/pypsa-eur/pull/204):
|
||||||
|
|
||||||
@ -59,6 +61,9 @@ Upcoming Release
|
|||||||
* Removed rule ``copy_config``. Instead, a config file is created for each
|
* Removed rule ``copy_config``. Instead, a config file is created for each
|
||||||
network output of the ``solve_*`` rules, with the same content as ``n.meta``.
|
network output of the ``solve_*`` rules, with the same content as ``n.meta``.
|
||||||
|
|
||||||
|
* Added new HVDC transmission projects from `TYNDP 2024 draft projects
|
||||||
|
<https://tyndp.entsoe.eu/news/176-pan-european-electricity-transmission-projects-and-33-storage-projects-will-be-assessed-in-tyndp-2024>`_.
|
||||||
|
|
||||||
* Upgrade to Snakemake v8.5+. This version is the new minimum version required.
|
* Upgrade to Snakemake v8.5+. This version is the new minimum version required.
|
||||||
To upgrade an existing environment, run ``conda install -c bioconda
|
To upgrade an existing environment, run ``conda install -c bioconda
|
||||||
snakemake-minimal">=8.5"`` and ``pip install snakemake-storage-plugin-http``
|
snakemake-minimal">=8.5"`` and ``pip install snakemake-storage-plugin-http``
|
||||||
|
@ -115,7 +115,7 @@ if config["enable"]["retrieve"] and config["enable"].get(
|
|||||||
mem_mb=5000,
|
mem_mb=5000,
|
||||||
retries: 2
|
retries: 2
|
||||||
run:
|
run:
|
||||||
move(input[0], output[0])
|
copyfile(input[0], output[0])
|
||||||
validate_checksum(output[0], input[0])
|
validate_checksum(output[0], input[0])
|
||||||
|
|
||||||
|
|
||||||
|
@ -31,7 +31,7 @@ rule solve_network:
|
|||||||
mem_mb=memory,
|
mem_mb=memory,
|
||||||
runtime=config_provider("solving", "runtime", default="6h"),
|
runtime=config_provider("solving", "runtime", default="6h"),
|
||||||
shadow:
|
shadow:
|
||||||
"minimal"
|
"shallow"
|
||||||
conda:
|
conda:
|
||||||
"../envs/environment.yaml"
|
"../envs/environment.yaml"
|
||||||
script:
|
script:
|
||||||
@ -62,7 +62,7 @@ rule solve_operations_network:
|
|||||||
mem_mb=(lambda w: 10000 + 372 * int(w.clusters)),
|
mem_mb=(lambda w: 10000 + 372 * int(w.clusters)),
|
||||||
runtime=config_provider("solving", "runtime", default="6h"),
|
runtime=config_provider("solving", "runtime", default="6h"),
|
||||||
shadow:
|
shadow:
|
||||||
"minimal"
|
"shallow"
|
||||||
conda:
|
conda:
|
||||||
"../envs/environment.yaml"
|
"../envs/environment.yaml"
|
||||||
script:
|
script:
|
||||||
|
@ -110,7 +110,8 @@ rule solve_sector_network_perfect:
|
|||||||
output:
|
output:
|
||||||
network=RESULTS
|
network=RESULTS
|
||||||
+ "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
|
+ "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
|
||||||
config="configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.yaml",
|
config=RESULTS
|
||||||
|
+ "configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.yaml",
|
||||||
threads: solver_threads
|
threads: solver_threads
|
||||||
resources:
|
resources:
|
||||||
mem_mb=config_provider("solving", "mem"),
|
mem_mb=config_provider("solving", "mem"),
|
||||||
|
@ -11,7 +11,9 @@ import os
|
|||||||
import re
|
import re
|
||||||
import urllib
|
import urllib
|
||||||
from functools import partial
|
from functools import partial
|
||||||
|
from os.path import exists
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from shutil import copyfile
|
||||||
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import pytz
|
import pytz
|
||||||
@ -25,6 +27,40 @@ logger = logging.getLogger(__name__)
|
|||||||
REGION_COLS = ["geometry", "name", "x", "y", "country"]
|
REGION_COLS = ["geometry", "name", "x", "y", "country"]
|
||||||
|
|
||||||
|
|
||||||
|
def copy_default_files(workflow):
|
||||||
|
default_files = {
|
||||||
|
"config/config.default.yaml": "config/config.yaml",
|
||||||
|
"config/scenarios.template.yaml": "config/scenarios.yaml",
|
||||||
|
}
|
||||||
|
for template, target in default_files.items():
|
||||||
|
target = os.path.join(workflow.current_basedir, target)
|
||||||
|
template = os.path.join(workflow.current_basedir, template)
|
||||||
|
if not exists(target) and exists(template):
|
||||||
|
copyfile(template, target)
|
||||||
|
|
||||||
|
|
||||||
|
def get_scenarios(run):
|
||||||
|
scenario_config = run.get("scenarios", {})
|
||||||
|
if run["name"] and scenario_config.get("enable"):
|
||||||
|
fn = Path(scenario_config["file"])
|
||||||
|
scenarios = yaml.safe_load(fn.read_text())
|
||||||
|
if run["name"] == "all":
|
||||||
|
run["name"] = list(scenarios.keys())
|
||||||
|
return scenarios
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def get_rdir(run):
|
||||||
|
scenario_config = run.get("scenarios", {})
|
||||||
|
if run["name"] and scenario_config.get("enable"):
|
||||||
|
RDIR = "{run}/"
|
||||||
|
elif run["name"]:
|
||||||
|
RDIR = run["name"] + "/"
|
||||||
|
else:
|
||||||
|
RDIR = ""
|
||||||
|
return RDIR
|
||||||
|
|
||||||
|
|
||||||
def get_run_path(fn, dir, rdir, shared_resources):
|
def get_run_path(fn, dir, rdir, shared_resources):
|
||||||
"""
|
"""
|
||||||
Dynamically provide paths based on shared resources and filename.
|
Dynamically provide paths based on shared resources and filename.
|
||||||
@ -66,20 +102,17 @@ def get_run_path(fn, dir, rdir, shared_resources):
|
|||||||
"add_electricity"
|
"add_electricity"
|
||||||
)
|
)
|
||||||
is_shared = no_relevant_wildcards and no_elec_rule
|
is_shared = no_relevant_wildcards and no_elec_rule
|
||||||
|
rdir = "" if is_shared else rdir
|
||||||
elif isinstance(shared_resources, str):
|
elif isinstance(shared_resources, str):
|
||||||
rdir = shared_resources + "/"
|
rdir = shared_resources + "/"
|
||||||
is_shared = False
|
|
||||||
elif isinstance(shared_resources, bool):
|
elif isinstance(shared_resources, bool):
|
||||||
is_shared = shared_resources
|
rdir = ""
|
||||||
else:
|
else:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"shared_resources must be a boolean, str, or 'base' for special handling."
|
"shared_resources must be a boolean, str, or 'base' for special handling."
|
||||||
)
|
)
|
||||||
|
|
||||||
if is_shared:
|
return f"{dir}{rdir}{fn}"
|
||||||
return f"{dir}{fn}"
|
|
||||||
else:
|
|
||||||
return f"{dir}{rdir}{fn}"
|
|
||||||
|
|
||||||
|
|
||||||
def path_provider(dir, rdir, shared_resources):
|
def path_provider(dir, rdir, shared_resources):
|
||||||
|
@ -145,7 +145,7 @@ def disable_grid_expansion_if_limit_hit(n):
|
|||||||
for name, glc in glcs.iterrows():
|
for name, glc in glcs.iterrows():
|
||||||
total_expansion = (
|
total_expansion = (
|
||||||
(
|
(
|
||||||
n.lines.query("p_nom_extendable")
|
n.lines.query("s_nom_extendable")
|
||||||
.eval(f"s_nom_min * {cols[limit_type]}")
|
.eval(f"s_nom_min * {cols[limit_type]}")
|
||||||
.sum()
|
.sum()
|
||||||
)
|
)
|
||||||
|
@ -202,9 +202,10 @@ if __name__ == "__main__":
|
|||||||
|
|
||||||
contained_years = pd.date_range(freq="YE", **snakemake.params.snapshots).year
|
contained_years = pd.date_range(freq="YE", **snakemake.params.snapshots).year
|
||||||
norm_year = config_hydro.get("eia_norm_year")
|
norm_year = config_hydro.get("eia_norm_year")
|
||||||
|
missing_years = contained_years.difference(eia_stats.index)
|
||||||
if norm_year:
|
if norm_year:
|
||||||
eia_stats.loc[contained_years] = eia_stats.loc[norm_year]
|
eia_stats.loc[contained_years] = eia_stats.loc[norm_year]
|
||||||
elif missing_years := eia_stats.index.difference(contained_years):
|
elif missing_years.any():
|
||||||
eia_stats.loc[missing_years] = eia_stats.median()
|
eia_stats.loc[missing_years] = eia_stats.median()
|
||||||
|
|
||||||
inflow = cutout.runoff(
|
inflow = cutout.runoff(
|
||||||
|
@ -44,7 +44,7 @@ if __name__ == "__main__":
|
|||||||
|
|
||||||
cf = snakemake.params.sequestration_potential
|
cf = snakemake.params.sequestration_potential
|
||||||
|
|
||||||
gdf = gpd.read_file(snakemake.input.sequestration_potential[0])
|
gdf = gpd.read_file(snakemake.input.sequestration_potential)
|
||||||
|
|
||||||
regions = gpd.read_file(snakemake.input.regions_offshore)
|
regions = gpd.read_file(snakemake.input.regions_offshore)
|
||||||
if cf["include_onshore"]:
|
if cf["include_onshore"]:
|
||||||
|
@ -267,7 +267,7 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="scip"):
|
|||||||
m.objective = (clusters * clusters - 2 * clusters * L * n_clusters).sum()
|
m.objective = (clusters * clusters - 2 * clusters * L * n_clusters).sum()
|
||||||
if solver_name == "gurobi":
|
if solver_name == "gurobi":
|
||||||
logging.getLogger("gurobipy").propagate = False
|
logging.getLogger("gurobipy").propagate = False
|
||||||
elif solver_name not in ["scip", "cplex"]:
|
elif solver_name not in ["scip", "cplex", "xpress", "copt", "mosek"]:
|
||||||
logger.info(
|
logger.info(
|
||||||
f"The configured solver `{solver_name}` does not support quadratic objectives. Falling back to `scip`."
|
f"The configured solver `{solver_name}` does not support quadratic objectives. Falling back to `scip`."
|
||||||
)
|
)
|
||||||
|
@ -26,7 +26,7 @@ if __name__ == "__main__":
|
|||||||
|
|
||||||
version = snakemake.params.version
|
version = snakemake.params.version
|
||||||
if "/" in version:
|
if "/" in version:
|
||||||
baseurl = f"https://raw.githubusercontent.com/{version}/outputs"
|
baseurl = f"https://raw.githubusercontent.com/{version}/outputs/"
|
||||||
else:
|
else:
|
||||||
baseurl = f"https://raw.githubusercontent.com/PyPSA/technology-data/{version}/outputs/"
|
baseurl = f"https://raw.githubusercontent.com/PyPSA/technology-data/{version}/outputs/"
|
||||||
filepath = Path(snakemake.output[0])
|
filepath = Path(snakemake.output[0])
|
||||||
|
Loading…
Reference in New Issue
Block a user