From ecccc1429fdc1c6c4dda81c886bb60170ed3e113 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 18 Jul 2024 15:39:27 +0200 Subject: [PATCH 01/55] add rule to retrieve JRC IDEES 2021 --- rules/retrieve.smk | 9 ++++++ scripts/retrieve_jrc_idees.py | 55 +++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 scripts/retrieve_jrc_idees.py diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 10ad9684..5b0871ca 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -54,6 +54,15 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", script: "../scripts/retrieve_eurostat_data.py" + rule retrieve_jrc_idees: + output: + directory("data/bundle/jrc-idees-2021"), + log: + "logs/retrieve_jrc_idees.log", + retries: 2 + script: + "../scripts/retrieve_jrc_idees.py" + rule retrieve_eurostat_household_data: output: "data/eurostat/eurostat-household_energy_balances-february_2024.csv", diff --git a/scripts/retrieve_jrc_idees.py b/scripts/retrieve_jrc_idees.py new file mode 100644 index 00000000..09484311 --- /dev/null +++ b/scripts/retrieve_jrc_idees.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2024- The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +""" +Retrieve and extract JRC IDEES 2021 data. +""" + +import os +import requests +from pathlib import Path +from bs4 import BeautifulSoup +from _helpers import configure_logging, progress_retrieve, set_scenario_config +import zipfile +import logging + + +logger = logging.getLogger(__name__) + +# Define the base URL +url_jrc = "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/JRC-IDEES/JRC-IDEES-2021_v1/" + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("retrieve_jrc_idees") + rootpath = ".." + else: + rootpath = "." + + configure_logging(snakemake) + set_scenario_config(snakemake) + + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + + # create a local directory to save the zip files + local_dir = snakemake.output[0] + if not os.path.exists(local_dir): + os.makedirs(local_dir) + + # get the list of zip files from the JRC URL + response = requests.get(url_jrc) + soup = BeautifulSoup(response.text, 'html.parser') + zip_files = [link.get('href') for link in soup.find_all('a') + if link.get('href').endswith('.zip')] + + logger.info(f"Downloading {len(zip_files)} .zip files for JRC IDEES from '{url_jrc}'.") + + # download and unpack each zip file + for zip_file in zip_files: + logger.info(f"Downloading and unpacking {zip_file}") + zip_url = url_jrc + zip_file + to_fn = local_dir+"/" + zip_file[:-4] + progress_retrieve(zip_url, to_fn, disable=disable_progress) + From 957176a20d66161d54c62bdea1a842e3637a6be5 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 18 Jul 2024 15:39:52 +0200 Subject: [PATCH 02/55] adjustment to build_energy_totals --- rules/build_sector.smk | 2 +- scripts/build_energy_totals.py | 125 +++++++++++++++++++-------------- 2 files changed, 73 insertions(+), 54 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 6614b163..866f6962 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -290,7 +290,7 @@ rule build_energy_totals: co2="data/bundle/eea/UNFCCC_v23.csv", swiss="data/switzerland-new_format-all_years.csv", swiss_transport="data/gr-e-11.03.02.01.01-cc.csv", - idees="data/bundle/jrc-idees-2015", + idees="data/bundle/jrc-idees-2021", district_heat_share="data/district_heat_share.csv", eurostat="data/eurostat/Balances-April2023", eurostat_households="data/eurostat/eurostat-household_energy_balances-february_2024.csv", diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index a476ec65..1dbc93e6 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -110,7 +110,8 @@ def reverse(dictionary: dict) -> dict: idees_rename = {"GR": "EL", "GB": "UK"} eu28 = cc.EU28as("ISO2").ISO2.tolist() - +# TODO GB kicked out JRC-IDEES 2021 +eu27 = cc.EU27as("ISO2").ISO2.tolist() eu28_eea = eu28.copy() eu28_eea.remove("GB") eu28_eea.append("UK") @@ -329,9 +330,9 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: """ ct_idees = idees_rename.get(ct, ct) - fn_residential = f"{base_dir}/JRC-IDEES-2015_Residential_{ct_idees}.xlsx" - fn_tertiary = f"{base_dir}/JRC-IDEES-2015_Tertiary_{ct_idees}.xlsx" - fn_transport = f"{base_dir}/JRC-IDEES-2015_Transport_{ct_idees}.xlsx" + fn_residential = f"{base_dir}/JRC-IDEES-2021_Residential_{ct_idees}.xlsx" + fn_tertiary = f"{base_dir}/JRC-IDEES-2021_Tertiary_{ct_idees}.xlsx" + fn_transport = f"{base_dir}/JRC-IDEES-2021_Transport_{ct_idees}.xlsx" ct_totals = {} @@ -357,14 +358,16 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: row = "Energy consumption by fuel - Eurostat structure (ktoe)" ct_totals["total residential"] = df.loc[row] - assert df.index[47] == "Electricity" - ct_totals["electricity residential"] = df.iloc[47] + assert df.index[40] == "Electricity" + ct_totals["electricity residential"] = df.iloc[40] - assert df.index[46] == "Derived heat" - ct_totals["derived heat residential"] = df.iloc[46] + # TODO derived heat changed to distributed heat and numbers changed as well! + # this needs to be checked + assert df.index[39] == "Distributed heat" + ct_totals["derived heat residential"] = df.iloc[39] - assert df.index[50] == "Thermal uses" - ct_totals["thermal uses residential"] = df.iloc[50] + assert df.index[43] == "Thermal uses" + ct_totals["thermal uses residential"] = df.iloc[43] # services @@ -390,14 +393,15 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: row = "Energy consumption by fuel - Eurostat structure (ktoe)" ct_totals["total services"] = df.loc[row] - assert df.index[50] == "Electricity" - ct_totals["electricity services"] = df.iloc[50] + assert df.index[43] == "Electricity" + ct_totals["electricity services"] = df.iloc[43] - assert df.index[49] == "Derived heat" - ct_totals["derived heat services"] = df.iloc[49] + # TODO check derived heat changed to distributed heat + assert df.index[42] == "Distributed heat" + ct_totals["derived heat services"] = df.iloc[42] - assert df.index[53] == "Thermal uses" - ct_totals["thermal uses services"] = df.iloc[53] + assert df.index[46] == "Thermal uses" + ct_totals["thermal uses services"] = df.iloc[46] # agriculture, forestry and fishing @@ -410,7 +414,7 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: "Lighting", "Ventilation", "Specific electricity uses", - "Pumping devices (electric)", + "Pumping devices (electricity)", ] ct_totals["total agriculture electricity"] = df.loc[rows].sum() @@ -419,8 +423,8 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: rows = [ "Motor drives", - "Farming machine drives (diesel oil incl. biofuels)", - "Pumping devices (diesel oil incl. biofuels)", + "Farming machine drives (diesel oil and liquid biofuels)", + "Pumping devices (diesel oil and liquid biofuels)", ] ct_totals["total agriculture machinery"] = df.loc[rows].sum() @@ -435,7 +439,7 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: ct_totals["electricity road"] = df.loc["Electricity"] - ct_totals["total two-wheel"] = df.loc["Powered 2-wheelers (Gasoline)"] + ct_totals["total two-wheel"] = df.loc["Powered two-wheelers (Gasoline)"] assert df.index[19] == "Passenger cars" ct_totals["total passenger cars"] = df.iloc[19] @@ -449,58 +453,64 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: assert df.index[39] == "Battery electric vehicles" ct_totals["electricity other road passenger"] = df.iloc[39] - assert df.index[41] == "Light duty vehicles" + assert df.index[41] == "Light commercial vehicles" ct_totals["total light duty road freight"] = df.iloc[41] assert df.index[49] == "Battery electric vehicles" ct_totals["electricity light duty road freight"] = df.iloc[49] - row = "Heavy duty vehicles (Diesel oil incl. biofuels)" + row = "Heavy goods vehicles (Diesel oil incl. biofuels)" ct_totals["total heavy duty road freight"] = df.loc[row] assert df.index[61] == "Passenger cars" ct_totals["passenger car efficiency"] = df.iloc[61] + df = pd.read_excel(fn_transport, "TrRail_ene", index_col=0) - ct_totals["total rail"] = df.loc["by fuel (EUROSTAT DATA)"] + ct_totals["total rail"] = df.loc["by fuel"] ct_totals["electricity rail"] = df.loc["Electricity"] - assert df.index[15] == "Passenger transport" - ct_totals["total rail passenger"] = df.iloc[15] + assert df.index[9] == "Passenger transport" + ct_totals["total rail passenger"] = df.iloc[9] - assert df.index[16] == "Metro and tram, urban light rail" - assert df.index[19] == "Electric" - assert df.index[20] == "High speed passenger trains" - ct_totals["electricity rail passenger"] = df.iloc[[16, 19, 20]].sum() + assert df.index[10] == "Metro and tram, urban light rail" + assert df.index[13] == "Electric" + assert df.index[14] == "High speed passenger trains" + ct_totals["electricity rail passenger"] = df.iloc[[10, 13, 14]].sum() - assert df.index[21] == "Freight transport" - ct_totals["total rail freight"] = df.iloc[21] + assert df.index[15] == "Freight transport" + ct_totals["total rail freight"] = df.iloc[15] - assert df.index[23] == "Electric" - ct_totals["electricity rail freight"] = df.iloc[23] + assert df.index[17] == "Electric" + ct_totals["electricity rail freight"] = df.iloc[17] df = pd.read_excel(fn_transport, "TrAvia_ene", index_col=0) - assert df.index[6] == "Passenger transport" - ct_totals["total aviation passenger"] = df.iloc[6] + assert df.index[4] == "Passenger transport" + ct_totals["total aviation passenger"] = df.iloc[4] - assert df.index[10] == "Freight transport" - ct_totals["total aviation freight"] = df.iloc[10] + assert df.index[8] == "Freight transport" + ct_totals["total aviation freight"] = df.iloc[8] - assert df.index[7] == "Domestic" - ct_totals["total domestic aviation passenger"] = df.iloc[7] + assert df.index[2] == "Domestic" + ct_totals["total domestic aviation passenger"] = df.iloc[2] - assert df.index[8] == "International - Intra-EU" - assert df.index[9] == "International - Extra-EU" - ct_totals["total international aviation passenger"] = df.iloc[[8, 9]].sum() + # TODO added Ukraine to intra EU flights + assert df.index[6] == "International - Intra-EEAwUK" + assert df.index[7] == "International - Extra-EEAwUK" + ct_totals["total international aviation passenger"] = df.iloc[[6, 7]].sum() + + # TODO freight changed from "Domestic and International - Intra-EU" -> split + # domestic and international (intra-EU and outside EU) + assert df.index[9] == "Domestic" + ct_totals["total domestic aviation freight"] = df.iloc[9] - assert df.index[11] == "Domestic and International - Intra-EU" - ct_totals["total domestic aviation freight"] = df.iloc[11] - assert df.index[12] == "International - Extra-EU" - ct_totals["total international aviation freight"] = df.iloc[12] + assert df.index[10] == "International - Intra-EEAwUK" + assert df.index[11] == "International - Extra-EEAwUK" + ct_totals["total international aviation freight"] = df.iloc[[10, 11]].sum() ct_totals["total domestic aviation"] = ( ct_totals["total domestic aviation freight"] @@ -515,7 +525,7 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: df = pd.read_excel(fn_transport, "TrNavi_ene", index_col=0) # coastal and inland - ct_totals["total domestic navigation"] = df.loc["by fuel (EUROSTAT DATA)"] + ct_totals["total domestic navigation"] = df.loc["Energy consumption (ktoe)"] df = pd.read_excel(fn_transport, "TrRoad_act", index_col=0) @@ -567,10 +577,15 @@ def build_idees(countries: List[str]) -> pd.DataFrame: names=["country", "year"], ) + # clean up dataframe + years = np.arange(2000, 2022) + totals = totals[totals.index.get_level_values(1).isin(years)] + # efficiency kgoe/100km -> ktoe/100km so that after conversion TWh/100km totals.loc[:, "passenger car efficiency"] /= 1e6 # convert ktoe to TWh exclude = totals.columns.str.fullmatch("passenger cars") + totals = totals.copy() totals.loc[:, ~exclude] *= 11.63 / 1e3 return totals @@ -664,7 +679,8 @@ def build_energy_totals( for use in uses: fuel_use = df[f"electricity {sector} {use}"] - fuel = df[f"electricity {sector}"] + fuel = (df[f"electricity {sector}"] + .replace(0, np.nan).infer_objects(copy=False)) avg = fuel_use.div(fuel).mean() logger.debug( f"{sector}: average fraction of electricity for {use} is {avg:.3f}" @@ -680,6 +696,7 @@ def build_energy_totals( df[f"total {sector} {use}"] - df[f"electricity {sector} {use}"] ) nonelectric = df[f"total {sector}"] - df[f"electricity {sector}"] + nonelectric = nonelectric.copy().replace(0, np.nan) avg = nonelectric_use.div(nonelectric).mean() logger.debug( f"{sector}: average fraction of non-electric for {use} is {avg:.3f}" @@ -716,6 +733,7 @@ def build_energy_totals( nonelectric = ( no_norway[f"total {sector}"] - no_norway[f"electricity {sector}"] ) + nonelectric = nonelectric.copy().replace(0, np.nan) fraction = nonelectric_use.div(nonelectric).mean() df.loc["NO", f"total {sector} {use}"] = ( total_heating * fraction @@ -793,7 +811,8 @@ def build_energy_totals( mean_BA = df.loc["BA"].loc[2014:2021, "total residential"].mean() mean_RS = df.loc["RS"].loc[2014:2021, "total residential"].mean() ratio = mean_BA / mean_RS - df.loc["BA"] = df.loc["BA"].replace(0.0, np.nan).values + df.loc["BA"] = (df.loc["BA"].replace(0.0, np.nan) + .infer_objects(copy=False).values) df.loc["BA"] = df.loc["BA"].combine_first(ratio * df.loc["RS"]).values return df @@ -1375,7 +1394,7 @@ def update_residential_from_eurostat(energy: pd.DataFrame) -> pd.DataFrame: "Updated energy balances for residential using disaggregate final energy consumption data in Households from Eurostat" ) - +#%% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -1391,7 +1410,7 @@ if __name__ == "__main__": population = nuts3["pop"].groupby(nuts3.country).sum() countries = snakemake.params.countries - idees_countries = pd.Index(countries).intersection(eu28) + idees_countries = pd.Index(countries).intersection(eu27) input_eurostat = snakemake.input.eurostat eurostat = build_eurostat( @@ -1405,8 +1424,8 @@ if __name__ == "__main__": energy = build_energy_totals(countries, eurostat, swiss, idees) - # Data from IDEES only exists from 2000-2015. - logger.info("Extrapolate IDEES data based on eurostat for years 2015-2021.") + # Data from IDEES only exists from 2000-2021. + logger.info("Extrapolate IDEES data based on eurostat for years 2021-x.") energy = rescale_idees_from_eurostat(idees_countries, energy, eurostat) update_residential_from_eurostat(energy) From 8cc23945b5ec9cb3fa4107e14b2679ae9d5b65d5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 13:43:38 +0000 Subject: [PATCH 03/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_energy_totals.py | 19 +++++++------- scripts/retrieve_jrc_idees.py | 47 +++++++++++++++++++--------------- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 1dbc93e6..a6eb70d1 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -465,7 +465,6 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: assert df.index[61] == "Passenger cars" ct_totals["passenger car efficiency"] = df.iloc[61] - df = pd.read_excel(fn_transport, "TrRail_ene", index_col=0) ct_totals["total rail"] = df.loc["by fuel"] @@ -501,13 +500,12 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: assert df.index[6] == "International - Intra-EEAwUK" assert df.index[7] == "International - Extra-EEAwUK" ct_totals["total international aviation passenger"] = df.iloc[[6, 7]].sum() - + # TODO freight changed from "Domestic and International - Intra-EU" -> split # domestic and international (intra-EU and outside EU) assert df.index[9] == "Domestic" ct_totals["total domestic aviation freight"] = df.iloc[9] - assert df.index[10] == "International - Intra-EEAwUK" assert df.index[11] == "International - Extra-EEAwUK" ct_totals["total international aviation freight"] = df.iloc[[10, 11]].sum() @@ -580,7 +578,7 @@ def build_idees(countries: List[str]) -> pd.DataFrame: # clean up dataframe years = np.arange(2000, 2022) totals = totals[totals.index.get_level_values(1).isin(years)] - + # efficiency kgoe/100km -> ktoe/100km so that after conversion TWh/100km totals.loc[:, "passenger car efficiency"] /= 1e6 # convert ktoe to TWh @@ -679,8 +677,9 @@ def build_energy_totals( for use in uses: fuel_use = df[f"electricity {sector} {use}"] - fuel = (df[f"electricity {sector}"] - .replace(0, np.nan).infer_objects(copy=False)) + fuel = ( + df[f"electricity {sector}"].replace(0, np.nan).infer_objects(copy=False) + ) avg = fuel_use.div(fuel).mean() logger.debug( f"{sector}: average fraction of electricity for {use} is {avg:.3f}" @@ -811,8 +810,9 @@ def build_energy_totals( mean_BA = df.loc["BA"].loc[2014:2021, "total residential"].mean() mean_RS = df.loc["RS"].loc[2014:2021, "total residential"].mean() ratio = mean_BA / mean_RS - df.loc["BA"] = (df.loc["BA"].replace(0.0, np.nan) - .infer_objects(copy=False).values) + df.loc["BA"] = ( + df.loc["BA"].replace(0.0, np.nan).infer_objects(copy=False).values + ) df.loc["BA"] = df.loc["BA"].combine_first(ratio * df.loc["RS"]).values return df @@ -1394,7 +1394,8 @@ def update_residential_from_eurostat(energy: pd.DataFrame) -> pd.DataFrame: "Updated energy balances for residential using disaggregate final energy consumption data in Households from Eurostat" ) -#%% + +# %% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake diff --git a/scripts/retrieve_jrc_idees.py b/scripts/retrieve_jrc_idees.py index 09484311..e163a163 100644 --- a/scripts/retrieve_jrc_idees.py +++ b/scripts/retrieve_jrc_idees.py @@ -6,50 +6,57 @@ Retrieve and extract JRC IDEES 2021 data. """ -import os -import requests -from pathlib import Path -from bs4 import BeautifulSoup -from _helpers import configure_logging, progress_retrieve, set_scenario_config -import zipfile import logging +import os +import zipfile +from pathlib import Path +import requests +from _helpers import configure_logging, progress_retrieve, set_scenario_config +from bs4 import BeautifulSoup logger = logging.getLogger(__name__) # Define the base URL -url_jrc = "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/JRC-IDEES/JRC-IDEES-2021_v1/" - +url_jrc = ( + "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/JRC-IDEES/JRC-IDEES-2021_v1/" +) + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake + snakemake = mock_snakemake("retrieve_jrc_idees") rootpath = ".." else: rootpath = "." - + configure_logging(snakemake) set_scenario_config(snakemake) - + disable_progress = snakemake.config["run"].get("disable_progressbar", False) - + # create a local directory to save the zip files local_dir = snakemake.output[0] if not os.path.exists(local_dir): os.makedirs(local_dir) - + # get the list of zip files from the JRC URL response = requests.get(url_jrc) - soup = BeautifulSoup(response.text, 'html.parser') - zip_files = [link.get('href') for link in soup.find_all('a') - if link.get('href').endswith('.zip')] - - logger.info(f"Downloading {len(zip_files)} .zip files for JRC IDEES from '{url_jrc}'.") - + soup = BeautifulSoup(response.text, "html.parser") + zip_files = [ + link.get("href") + for link in soup.find_all("a") + if link.get("href").endswith(".zip") + ] + + logger.info( + f"Downloading {len(zip_files)} .zip files for JRC IDEES from '{url_jrc}'." + ) + # download and unpack each zip file for zip_file in zip_files: logger.info(f"Downloading and unpacking {zip_file}") zip_url = url_jrc + zip_file - to_fn = local_dir+"/" + zip_file[:-4] + to_fn = local_dir + "/" + zip_file[:-4] progress_retrieve(zip_url, to_fn, disable=disable_progress) - From 5f009590f5e9ecbbf41c030b77b94e608946072d Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 18 Jul 2024 15:48:45 +0200 Subject: [PATCH 04/55] remove todos --- scripts/build_energy_totals.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index a6eb70d1..b3f6ddb7 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -110,7 +110,6 @@ def reverse(dictionary: dict) -> dict: idees_rename = {"GR": "EL", "GB": "UK"} eu28 = cc.EU28as("ISO2").ISO2.tolist() -# TODO GB kicked out JRC-IDEES 2021 eu27 = cc.EU27as("ISO2").ISO2.tolist() eu28_eea = eu28.copy() eu28_eea.remove("GB") From 4c46c57fecf002c0027b977b2fd6f15d0df5dca4 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 19 Jul 2024 10:20:19 +0200 Subject: [PATCH 05/55] avoid double download of JRC idees --- scripts/build_energy_totals.py | 10 ++++++--- scripts/retrieve_jrc_idees.py | 41 +++++++++++++--------------------- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index b3f6ddb7..a93562b2 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -329,9 +329,9 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: """ ct_idees = idees_rename.get(ct, ct) - fn_residential = f"{base_dir}/JRC-IDEES-2021_Residential_{ct_idees}.xlsx" - fn_tertiary = f"{base_dir}/JRC-IDEES-2021_Tertiary_{ct_idees}.xlsx" - fn_transport = f"{base_dir}/JRC-IDEES-2021_Transport_{ct_idees}.xlsx" + fn_residential = f"{base_dir}/{ct_idees}/JRC-IDEES-2021_Residential_{ct_idees}.xlsx" + fn_tertiary = f"{base_dir}/{ct_idees}/JRC-IDEES-2021_Tertiary_{ct_idees}.xlsx" + fn_transport = f"{base_dir}/{ct_idees}/JRC-IDEES-2021_Transport_{ct_idees}.xlsx" ct_totals = {} @@ -1103,6 +1103,10 @@ def build_transport_data( transport_data = pd.concat([transport_data, swiss_cars]).sort_index() transport_data.rename(columns={"passenger cars": "number cars"}, inplace=True) + + # clean up dataframe + years = np.arange(2000, 2022) + transport_data = transport_data[transport_data.index.get_level_values(1).isin(years)] missing = transport_data.index[transport_data["number cars"].isna()] if not missing.empty: diff --git a/scripts/retrieve_jrc_idees.py b/scripts/retrieve_jrc_idees.py index e163a163..6c61ee19 100644 --- a/scripts/retrieve_jrc_idees.py +++ b/scripts/retrieve_jrc_idees.py @@ -10,22 +10,19 @@ import logging import os import zipfile from pathlib import Path - -import requests from _helpers import configure_logging, progress_retrieve, set_scenario_config -from bs4 import BeautifulSoup + logger = logging.getLogger(__name__) # Define the base URL url_jrc = ( - "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/JRC-IDEES/JRC-IDEES-2021_v1/" + "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/JRC-IDEES/JRC-IDEES-2021_v1/JRC-IDEES-2021.zip" ) if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("retrieve_jrc_idees") rootpath = ".." else: @@ -33,30 +30,22 @@ if __name__ == "__main__": configure_logging(snakemake) set_scenario_config(snakemake) - disable_progress = snakemake.config["run"].get("disable_progressbar", False) - # create a local directory to save the zip files - local_dir = snakemake.output[0] - if not os.path.exists(local_dir): - os.makedirs(local_dir) - - # get the list of zip files from the JRC URL - response = requests.get(url_jrc) - soup = BeautifulSoup(response.text, "html.parser") - zip_files = [ - link.get("href") - for link in soup.find_all("a") - if link.get("href").endswith(".zip") - ] + to_fn = snakemake.output[0] + to_fn_zp = to_fn + ".zip" + # download .zip file logger.info( - f"Downloading {len(zip_files)} .zip files for JRC IDEES from '{url_jrc}'." + f"Downloading JRC IDEES from {url_jrc}." ) + progress_retrieve(url_jrc, to_fn_zp, disable=disable_progress) + + # extract + logger.info("Extracting JRC IDEES data.") + with zipfile.ZipFile(to_fn_zp, "r") as zip_ref: + zip_ref.extractall(to_fn) - # download and unpack each zip file - for zip_file in zip_files: - logger.info(f"Downloading and unpacking {zip_file}") - zip_url = url_jrc + zip_file - to_fn = local_dir + "/" + zip_file[:-4] - progress_retrieve(zip_url, to_fn, disable=disable_progress) + logger.info(f"JRC IDEES data available in '{to_fn}'.") + + \ No newline at end of file From 086d46cdb4aa0a8943da99057d57f6068615ae4b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 08:21:07 +0000 Subject: [PATCH 06/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_energy_totals.py | 6 ++++-- scripts/retrieve_jrc_idees.py | 15 +++++---------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index a93562b2..72c2b5bd 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -1103,10 +1103,12 @@ def build_transport_data( transport_data = pd.concat([transport_data, swiss_cars]).sort_index() transport_data.rename(columns={"passenger cars": "number cars"}, inplace=True) - + # clean up dataframe years = np.arange(2000, 2022) - transport_data = transport_data[transport_data.index.get_level_values(1).isin(years)] + transport_data = transport_data[ + transport_data.index.get_level_values(1).isin(years) + ] missing = transport_data.index[transport_data["number cars"].isna()] if not missing.empty: diff --git a/scripts/retrieve_jrc_idees.py b/scripts/retrieve_jrc_idees.py index 6c61ee19..0e23b82c 100644 --- a/scripts/retrieve_jrc_idees.py +++ b/scripts/retrieve_jrc_idees.py @@ -10,19 +10,18 @@ import logging import os import zipfile from pathlib import Path -from _helpers import configure_logging, progress_retrieve, set_scenario_config +from _helpers import configure_logging, progress_retrieve, set_scenario_config logger = logging.getLogger(__name__) # Define the base URL -url_jrc = ( - "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/JRC-IDEES/JRC-IDEES-2021_v1/JRC-IDEES-2021.zip" -) +url_jrc = "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/JRC-IDEES/JRC-IDEES-2021_v1/JRC-IDEES-2021.zip" if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake + snakemake = mock_snakemake("retrieve_jrc_idees") rootpath = ".." else: @@ -36,16 +35,12 @@ if __name__ == "__main__": to_fn_zp = to_fn + ".zip" # download .zip file - logger.info( - f"Downloading JRC IDEES from {url_jrc}." - ) + logger.info(f"Downloading JRC IDEES from {url_jrc}.") progress_retrieve(url_jrc, to_fn_zp, disable=disable_progress) - + # extract logger.info("Extracting JRC IDEES data.") with zipfile.ZipFile(to_fn_zp, "r") as zip_ref: zip_ref.extractall(to_fn) logger.info(f"JRC IDEES data available in '{to_fn}'.") - - \ No newline at end of file From 104fa7311935482620bc12e680515ae4adb6c12c Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 19 Jul 2024 14:36:46 +0200 Subject: [PATCH 07/55] remove rescale to Eurostat --- scripts/build_energy_totals.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 72c2b5bd..a53189e0 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -1147,8 +1147,8 @@ def rescale_idees_from_eurostat( idees_countries: List[str], energy: pd.DataFrame, eurostat: pd.DataFrame ) -> pd.DataFrame: """ - Takes JRC IDEES data from 2015 and rescales it by the ratio of the Eurostat - data and the 2015 Eurostat data. + Takes JRC IDEES data from 2021 and rescales it by the ratio of the Eurostat + data and the 2021 Eurostat data. Missing data: ['passenger car efficiency', 'passenger cars'] Parameters @@ -1178,7 +1178,7 @@ def rescale_idees_from_eurostat( main_cols = ["Total all products", "Electricity"] # read in the eurostat data for 2015 - eurostat_2015 = eurostat.xs(2015, level="year")[main_cols] + eurostat_2015 = eurostat.xs(2021, level="year")[main_cols] # calculate the ratio of the two data sets ratio = eurostat[main_cols] / eurostat_2015 ratio = ratio.droplevel([2, 5]) @@ -1430,10 +1430,6 @@ if __name__ == "__main__": energy = build_energy_totals(countries, eurostat, swiss, idees) - # Data from IDEES only exists from 2000-2021. - logger.info("Extrapolate IDEES data based on eurostat for years 2021-x.") - energy = rescale_idees_from_eurostat(idees_countries, energy, eurostat) - update_residential_from_eurostat(energy) energy.to_csv(snakemake.output.energy_name) From dea48c965a0dd36f05ca61c2b632ed71dfc179d2 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 19 Jul 2024 14:41:36 +0200 Subject: [PATCH 08/55] correct indexing of DH share --- data/district_heat_share.csv | 2 +- scripts/build_energy_totals.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/data/district_heat_share.csv b/data/district_heat_share.csv index 5afd65c8..07d4f51d 100644 --- a/data/district_heat_share.csv +++ b/data/district_heat_share.csv @@ -22,7 +22,7 @@ RS,25,5821 SI,8.86,1739 ES,0.251589260787732,1273 SE,50.4, -UK,2, +GB,2, BY,70, EE,52,5406 KO,3,207 diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index a53189e0..349a2c82 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -860,8 +860,10 @@ def build_district_heat_share(countries: List[str], idees: pd.DataFrame) -> pd.S .squeeze() ) # make conservative assumption and take minimum from both data sets + new_index = pd.MultiIndex.from_product([dh_share.index, + district_heat_share.index.get_level_values(1).unique()]) district_heat_share = pd.concat( - [district_heat_share, dh_share.reindex_like(district_heat_share)], axis=1 + [district_heat_share, dh_share.reindex(new_index, level=0)], axis=1 ).min(axis=1) district_heat_share.name = "district heat share" From 028d0d6e7aeb9d4ef8c50313299e8b0c0cab095c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 13:15:35 +0000 Subject: [PATCH 09/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_energy_totals.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 349a2c82..70ed4bec 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -860,8 +860,9 @@ def build_district_heat_share(countries: List[str], idees: pd.DataFrame) -> pd.S .squeeze() ) # make conservative assumption and take minimum from both data sets - new_index = pd.MultiIndex.from_product([dh_share.index, - district_heat_share.index.get_level_values(1).unique()]) + new_index = pd.MultiIndex.from_product( + [dh_share.index, district_heat_share.index.get_level_values(1).unique()] + ) district_heat_share = pd.concat( [district_heat_share, dh_share.reindex(new_index, level=0)], axis=1 ).min(axis=1) From 39e853e2b363427a52b3dc056c335a1365ebd1a7 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 19 Jul 2024 15:34:44 +0200 Subject: [PATCH 10/55] make sure not to divide by zero --- scripts/build_energy_totals.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 70ed4bec..4e353c11 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -847,7 +847,7 @@ def build_district_heat_share(countries: List[str], idees: pd.DataFrame) -> pd.S ) total_heat = idees[["thermal uses residential", "thermal uses services"]].sum( axis=1 - ) + ).replace(0, np.nan) district_heat_share = district_heat / total_heat @@ -866,6 +866,8 @@ def build_district_heat_share(countries: List[str], idees: pd.DataFrame) -> pd.S district_heat_share = pd.concat( [district_heat_share, dh_share.reindex(new_index, level=0)], axis=1 ).min(axis=1) + + district_heat_share = district_heat_share.reindex(countries, level=0) district_heat_share.name = "district heat share" From 0e1e50900635af5626e1412fa1c314045696ee4a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 13:43:14 +0000 Subject: [PATCH 11/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_energy_totals.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 4e353c11..9491cc57 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -845,9 +845,11 @@ def build_district_heat_share(countries: List[str], idees: pd.DataFrame) -> pd.S district_heat = idees[["derived heat residential", "derived heat services"]].sum( axis=1 ) - total_heat = idees[["thermal uses residential", "thermal uses services"]].sum( - axis=1 - ).replace(0, np.nan) + total_heat = ( + idees[["thermal uses residential", "thermal uses services"]] + .sum(axis=1) + .replace(0, np.nan) + ) district_heat_share = district_heat / total_heat @@ -866,7 +868,7 @@ def build_district_heat_share(countries: List[str], idees: pd.DataFrame) -> pd.S district_heat_share = pd.concat( [district_heat_share, dh_share.reindex(new_index, level=0)], axis=1 ).min(axis=1) - + district_heat_share = district_heat_share.reindex(countries, level=0) district_heat_share.name = "district heat share" From 53bbc28908faf26ea93ff58fa4c7e95078e4abed Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 22 Jul 2024 21:15:40 +0200 Subject: [PATCH 12/55] add eurostat data for GB agriculture --- scripts/build_energy_totals.py | 42 ++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 9491cc57..d4afe7c8 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -55,6 +55,21 @@ logger = logging.getLogger(__name__) idx = pd.IndexSlice + +# from JRC-2021 methodology p.58 +agriculture_idees_eurostat_mapping = { + "Solids": ["C0000X0350-0370", "P1000", "S2000"], + "LPG": ["O4630"], + "Diesel oil and liquid biofuels": ["O4671XR5220B", "R5210P", "R5210B", "R5220P", "R5220B", "R5230P", "R5230B", "R5290"], + "Fuel oil and other liquids": ["O4680", "O4100_TOT_4200-4500XBIO", "O4652XR5210B", "O4651", "O4653", "O4661XR5230B", "O4669", "O4640", "O4691", "O4692", "O4695", "O4694", "O4693", "O4699"], + "Natural gas and biogas": ["G3000", "C0350-0370", "R5300"], + "Biomass and waste": ["R5110-5150_W6000RI", "R5160", "W6210", "W6100_6220"], + "Solar and geothermal": ["RA200", "RA410"], + "Ambient heat": ["RA600"], + "Distributed heat": ["H8000"], + "Electricity": ["E7000"] +} + def cartesian(s1: pd.Series, s2: pd.Series) -> pd.DataFrame: """ Compute the Cartesian product of two pandas Series. @@ -648,6 +663,33 @@ def build_energy_totals( df = pd.concat([df.drop("CH", errors="ignore"), swiss]).sort_index() # get values for missing countries based on Eurostat EnergyBalances + + # agriculture + + to_fill = df.index[ + df["total agriculture"].isna() + & df.index.get_level_values("country").isin(eurostat_countries) + ] + c = to_fill.get_level_values("country") + y = to_fill.get_level_values("year") + + # take total final energy consumption from Eurostat + eurostat_sector = 'Agriculture & forestry' + slicer = idx[c, y, :, :, eurostat_sector] + + fill_values = eurostat.loc[slicer]["Total all products"].groupby(level=[0,1]).sum() + # fill missing years for some countries by mean over the other years + means = fill_values.groupby(level='country').transform('mean') + fill_values = fill_values.where(fill_values != 0, means) + + # split into end uses by average EU data from IDEES + uses = ["electricity", "heat", "machinery"] + + for use in uses: + avg = (idees["total agriculture electricity"] + /idees["total agriculture"]).mean() + df.loc[to_fill, f"total agriculture {use}"] = df.loc[to_fill, "total agriculture"] * avg + # divide cooking/space/water according to averages in EU28 uses = ["space", "cooking", "water"] From b957f88645bf6aa31261e58c697d4da525c39334 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 22 Jul 2024 21:40:47 +0200 Subject: [PATCH 13/55] fix bugs and clean up --- scripts/build_energy_totals.py | 54 +++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index d4afe7c8..55da77e3 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -55,21 +55,6 @@ logger = logging.getLogger(__name__) idx = pd.IndexSlice - -# from JRC-2021 methodology p.58 -agriculture_idees_eurostat_mapping = { - "Solids": ["C0000X0350-0370", "P1000", "S2000"], - "LPG": ["O4630"], - "Diesel oil and liquid biofuels": ["O4671XR5220B", "R5210P", "R5210B", "R5220P", "R5220B", "R5230P", "R5230B", "R5290"], - "Fuel oil and other liquids": ["O4680", "O4100_TOT_4200-4500XBIO", "O4652XR5210B", "O4651", "O4653", "O4661XR5230B", "O4669", "O4640", "O4691", "O4692", "O4695", "O4694", "O4693", "O4699"], - "Natural gas and biogas": ["G3000", "C0350-0370", "R5300"], - "Biomass and waste": ["R5110-5150_W6000RI", "R5160", "W6210", "W6100_6220"], - "Solar and geothermal": ["RA200", "RA410"], - "Ambient heat": ["RA600"], - "Distributed heat": ["H8000"], - "Electricity": ["E7000"] -} - def cartesian(s1: pd.Series, s2: pd.Series) -> pd.DataFrame: """ Compute the Cartesian product of two pandas Series. @@ -603,6 +588,31 @@ def build_idees(countries: List[str]) -> pd.DataFrame: return totals +def fill_missing_years(fill_values: pd.Series) -> pd.Series: + """ + Fill missing years for some countries by mean over the other years. + + Parameters + ---------- + fill_values : pd.Series + A pandas Series with a MultiIndex (levels: country and year) representing + energy values, where some values may be zero and need to be filled. + + Returns + ------- + pd.Series + A pandas Series with zero values replaced by the mean value of the corresponding + country. + + Notes + ----- + - The function groups the data by the 'country' level and computes the mean for each group. + - Zero values in the original Series are replaced by the mean value of their respective country group. + """ + means = fill_values.groupby(level='country').transform('mean') + return fill_values.where(fill_values != 0, means) + + def build_energy_totals( countries: List[str], eurostat: pd.DataFrame, @@ -656,6 +666,8 @@ def build_energy_totals( slicer = idx[in_eurostat, :, :, "Bunkers", :] fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=[0, 1]).sum() + # fill missing years for some countries by mean over the other years + fill_values = fill_missing_years(fill_values) df.loc[in_eurostat, "total international navigation"] = fill_values # add swiss energy data @@ -679,8 +691,8 @@ def build_energy_totals( fill_values = eurostat.loc[slicer]["Total all products"].groupby(level=[0,1]).sum() # fill missing years for some countries by mean over the other years - means = fill_values.groupby(level='country').transform('mean') - fill_values = fill_values.where(fill_values != 0, means) + fill_values = fill_missing_years(fill_values) + df.loc[to_fill, "total agriculture"] = fill_values # split into end uses by average EU data from IDEES uses = ["electricity", "heat", "machinery"] @@ -711,6 +723,8 @@ def build_energy_totals( fill_values = ( eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=[0, 1]).sum() ) + # fill missing years for some countries by mean over the other years + fill_values = fill_missing_years(fill_values) df.loc[to_fill, f"{fuel} {sector}"] = fill_values for sector in ["residential", "services"]: @@ -786,16 +800,22 @@ def build_energy_totals( slicer = idx[c, y, :, :, "Domestic aviation"] fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=[0, 1]).sum() + # fill missing years for some countries by mean over the other years + fill_values = fill_missing_years(fill_values) df.loc[to_fill, "total domestic aviation"] = fill_values slicer = idx[c, y, :, :, "International aviation"] fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=[0, 1]).sum() + # fill missing years for some countries by mean over the other years + fill_values = fill_missing_years(fill_values) df.loc[to_fill, "total international aviation"] = fill_values # missing domestic navigation slicer = idx[c, y, :, :, "Domestic Navigation"] fill_values = eurostat.loc[slicer, "Total all products"].groupby(level=[0, 1]).sum() + # fill missing years for some countries by mean over the other years + fill_values = fill_missing_years(fill_values) df.loc[to_fill, "total domestic navigation"] = fill_values # split road traffic for non-IDEES From fb26aaf38dc85e37570162060212ef22425b6c6c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 19:42:52 +0000 Subject: [PATCH 14/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_energy_totals.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 55da77e3..6404cd93 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -609,7 +609,7 @@ def fill_missing_years(fill_values: pd.Series) -> pd.Series: - The function groups the data by the 'country' level and computes the mean for each group. - Zero values in the original Series are replaced by the mean value of their respective country group. """ - means = fill_values.groupby(level='country').transform('mean') + means = fill_values.groupby(level="country").transform("mean") return fill_values.where(fill_values != 0, means) @@ -675,33 +675,36 @@ def build_energy_totals( df = pd.concat([df.drop("CH", errors="ignore"), swiss]).sort_index() # get values for missing countries based on Eurostat EnergyBalances - + # agriculture - + to_fill = df.index[ df["total agriculture"].isna() & df.index.get_level_values("country").isin(eurostat_countries) ] c = to_fill.get_level_values("country") y = to_fill.get_level_values("year") - + # take total final energy consumption from Eurostat - eurostat_sector = 'Agriculture & forestry' + eurostat_sector = "Agriculture & forestry" slicer = idx[c, y, :, :, eurostat_sector] - - fill_values = eurostat.loc[slicer]["Total all products"].groupby(level=[0,1]).sum() + + fill_values = eurostat.loc[slicer]["Total all products"].groupby(level=[0, 1]).sum() # fill missing years for some countries by mean over the other years fill_values = fill_missing_years(fill_values) df.loc[to_fill, "total agriculture"] = fill_values - + # split into end uses by average EU data from IDEES uses = ["electricity", "heat", "machinery"] - + for use in uses: - avg = (idees["total agriculture electricity"] - /idees["total agriculture"]).mean() - df.loc[to_fill, f"total agriculture {use}"] = df.loc[to_fill, "total agriculture"] * avg - + avg = ( + idees["total agriculture electricity"] / idees["total agriculture"] + ).mean() + df.loc[to_fill, f"total agriculture {use}"] = ( + df.loc[to_fill, "total agriculture"] * avg + ) + # divide cooking/space/water according to averages in EU28 uses = ["space", "cooking", "water"] From 3c62a3b4cb21e8a86eb0478a8fd30baf41d09266 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 22 Jul 2024 21:52:54 +0200 Subject: [PATCH 15/55] add release notes --- doc/release_notes.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index eb29ce4b..fcfd8083 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,9 @@ Release Notes Upcoming Release ================ +* Upadte JRC-IDEES-2015 to `JRC-IDEES-2021 +`__. + * Changed default assumptions about waste heat usage from PtX and fuel cells in district heating. The default value for the link efficiency scaling factor was changed from 100% to 25%. It can be set to other values in the configuration ``sector: use_TECHNOLOGY_waste_heat``. From 7451349fed6f2814d0b26627b6982552f8738da4 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Tue, 23 Jul 2024 15:10:05 +0200 Subject: [PATCH 16/55] take eurostat year for transport demand --- scripts/build_transport_demand.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index 4a29667a..074dc260 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) def build_nodal_transport_data(fn, pop_layout, year): # get numbers of car and fuel efficiency per country transport_data = pd.read_csv(fn, index_col=[0, 1]) - transport_data = transport_data.xs(min(2015, year), level="year") + transport_data = transport_data.xs(year, level="year") # break number of cars down to nodal level based on population density nodal_transport_data = transport_data.loc[pop_layout.ct].fillna(0.0) From c99cd72727b790cc37cf65046c3211edf7c2e5fc Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 29 Jul 2024 13:36:06 +0200 Subject: [PATCH 17/55] include Tonis Feedback --- scripts/build_energy_totals.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 6404cd93..44bb470f 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -360,10 +360,8 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: assert df.index[40] == "Electricity" ct_totals["electricity residential"] = df.iloc[40] - # TODO derived heat changed to distributed heat and numbers changed as well! - # this needs to be checked assert df.index[39] == "Distributed heat" - ct_totals["derived heat residential"] = df.iloc[39] + ct_totals["distributed heat residential"] = df.iloc[39] assert df.index[43] == "Thermal uses" ct_totals["thermal uses residential"] = df.iloc[43] @@ -395,9 +393,8 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: assert df.index[43] == "Electricity" ct_totals["electricity services"] = df.iloc[43] - # TODO check derived heat changed to distributed heat assert df.index[42] == "Distributed heat" - ct_totals["derived heat services"] = df.iloc[42] + ct_totals["distributed heat services"] = df.iloc[42] assert df.index[46] == "Thermal uses" ct_totals["thermal uses services"] = df.iloc[46] @@ -900,14 +897,14 @@ def build_district_heat_share(countries: List[str], idees: pd.DataFrame) -> pd.S Notes ----- - - The function calculates the district heating share as the sum of residential and services derived heat, divided by the sum of residential and services thermal uses. + - The function calculates the district heating share as the sum of residential and services distributed heat, divided by the sum of residential and services thermal uses. - The district heating share is then reindexed to match the provided list of countries. - Missing district heating shares are filled from `data/district_heat_share.csv`. - The function makes a conservative assumption and takes the minimum district heating share from both the IDEES data and `data/district_heat_share.csv`. """ # district heating share - district_heat = idees[["derived heat residential", "derived heat services"]].sum( + district_heat = idees[["distributed heat residential", "distributed heat services"]].sum( axis=1 ) total_heat = ( @@ -1265,7 +1262,7 @@ def rescale_idees_from_eurostat( "total residential water", "total residential cooking", "total residential", - "derived heat residential", + "distributed heat residential", "thermal uses residential", ], "elec": [ @@ -1281,7 +1278,7 @@ def rescale_idees_from_eurostat( "total services water", "total services cooking", "total services", - "derived heat services", + "distributed heat services", "thermal uses services", ], "elec": [ From 201889424c48c7a0c9a6d05d9e23f739c4377651 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 29 Jul 2024 15:39:01 +0200 Subject: [PATCH 18/55] update industry to new JRC-idees --- config/config.default.yaml | 2 +- rules/build_sector.smk | 4 +- ...ustrial_energy_demand_per_country_today.py | 4 +- ...build_industrial_production_per_country.py | 72 ++++++++++--------- 4 files changed, 44 insertions(+), 38 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index d1d13065..373f4a6c 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -716,7 +716,7 @@ industry: MWh_CH4_per_tMeOH: 10.25 MWh_MeOH_per_tMeOH: 5.528 hotmaps_locate_missing: false - reference_year: 2015 + reference_year: 2021 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#costs diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 866f6962..cd259d39 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -509,7 +509,7 @@ rule build_industrial_production_per_country: countries=config_provider("countries"), input: ammonia_production=resources("ammonia_production.csv"), - jrc="data/bundle/jrc-idees-2015", + jrc="data/bundle/jrc-idees-2021", eurostat="data/eurostat/Balances-April2023", output: industrial_production_per_country=resources( @@ -656,7 +656,7 @@ rule build_industrial_energy_demand_per_country_today: countries=config_provider("countries"), industry=config_provider("industry"), input: - jrc="data/bundle/jrc-idees-2015", + jrc="data/bundle/jrc-idees-2021", industrial_production_per_country=resources( "industrial_production_per_country.csv" ), diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index b77ba8d6..114485e7 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -8,7 +8,7 @@ Build industrial energy demand per country. Inputs ------- -- ``data/bundle/jrc-idees-2015`` +- ``data/bundle/jrc-idees-2021`` - ``industrial_production_per_country.csv`` Outputs @@ -120,7 +120,7 @@ jrc_names = {"GR": "EL", "GB": "UK"} def industrial_energy_demand_per_country(country, year, jrc_dir): jrc_country = jrc_names.get(country, country) - fn = f"{jrc_dir}/JRC-IDEES-2015_EnergyBalance_{jrc_country}.xlsx" + fn = f"{jrc_dir}/{jrc_country}/JRC-IDEES-2021_EnergyBalance_{jrc_country}.xlsx" sheets = list(sector_sheets.values()) df_dict = pd.read_excel(fn, sheet_name=sheets, index_col=0) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index ec86a78d..1bb75271 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -16,7 +16,7 @@ Relevant Settings Inputs ------- - ``resources/ammonia_production.csv`` -- ``data/bundle-sector/jrc-idees-2015`` +- ``data/bundle-sector/jrc-idees-2021`` - ``data/eurostat`` Outputs @@ -50,11 +50,11 @@ The following subcategories [kton/a] are considered: - Aluminium - primary production - Aluminium - secondary production - Other non-ferrous metals -- Transport Equipment -- Machinery Equipment +- Transport equipment +- Machinery equipment - Textiles and leather - Wood and wood products -- Other Industrial Sectors +- Other industrial sectors - Ammonia - HVC - Chlorine @@ -79,25 +79,25 @@ ktoe_to_twh = 0.01163 sub_sheet_name_dict = { "Iron and steel": "ISI", - "Chemicals Industry": "CHI", + "Chemical industry": "CHI", "Non-metallic mineral products": "NMM", "Pulp, paper and printing": "PPA", "Food, beverages and tobacco": "FBT", "Non Ferrous Metals": "NFM", - "Transport Equipment": "TRE", - "Machinery Equipment": "MAE", + "Transport equipment": "TRE", + "Machinery equipment": "MAE", "Textiles and leather": "TEL", "Wood and wood products": "WWP", - "Other Industrial Sectors": "OIS", + "Other industrial sectors": "OIS", } -eu28 = cc.EU28as("ISO2").ISO2.values +eu27 = cc.EU27as("ISO2").ISO2.values jrc_names = {"GR": "EL", "GB": "UK"} sect2sub = { "Iron and steel": ["Electric arc", "Integrated steelworks"], - "Chemicals Industry": [ + "Chemical industry": [ "Basic chemicals", "Other chemicals", "Pharmaceutical products etc.", @@ -119,11 +119,11 @@ sect2sub = { "Aluminium - secondary production", "Other non-ferrous metals", ], - "Transport Equipment": ["Transport Equipment"], - "Machinery Equipment": ["Machinery Equipment"], + "Transport equipment": ["Transport equipment"], + "Machinery equipment": ["Machinery equipment"], "Textiles and leather": ["Textiles and leather"], "Wood and wood products": ["Wood and wood products"], - "Other Industrial Sectors": ["Other Industrial Sectors"], + "Other industrial sectors": ["Other industrial sectors"], } sub2sect = {v: k for k, vv in sect2sub.items() for v in vv} @@ -145,27 +145,29 @@ fields = { "Aluminium - primary production": "Aluminium - primary production", "Aluminium - secondary production": "Aluminium - secondary production", "Other non-ferrous metals": "Other non-ferrous metals (kt lead eq.)", - "Transport Equipment": "Physical output (index)", - "Machinery Equipment": "Physical output (index)", + "Transport equipment": "Physical output (index)", + "Machinery equipment": "Physical output (index)", "Textiles and leather": "Physical output (index)", "Wood and wood products": "Physical output (index)", - "Other Industrial Sectors": "Physical output (index)", + "Other industrial sectors": "Physical output (index)", } eb_sectors = { "Iron & steel": "Iron and steel", - "Chemical & petrochemical": "Chemicals Industry", + "Chemical & petrochemical": "Chemical industry", "Non-ferrous metals": "Non-metallic mineral products", "Paper, pulp & printing": "Pulp, paper and printing", "Food, beverages & tobacco": "Food, beverages and tobacco", "Non-metallic minerals": "Non Ferrous Metals", - "Transport equipment": "Transport Equipment", - "Machinery": "Machinery Equipment", + "Transport equipment": "Transport equipment", + "Machinery": "Machinery equipment", "Textile & leather": "Textiles and leather", "Wood & wood products": "Wood and wood products", - "Not elsewhere specified (industry)": "Other Industrial Sectors", + "Not elsewhere specified (industry)": "Other industrial sectors", } + + # TODO: this should go in a csv in `data` # Annual energy consumption in Switzerland by sector in 2015 (in TJ) # From: Energieverbrauch in der Industrie und im Dienstleistungssektor, Der Bundesrat @@ -173,16 +175,16 @@ eb_sectors = { e_switzerland = pd.Series( { "Iron and steel": 7889.0, - "Chemicals Industry": 26871.0, + "Chemical industry": 26871.0, "Non-metallic mineral products": 15513.0 + 3820.0, "Pulp, paper and printing": 12004.0, "Food, beverages and tobacco": 17728.0, "Non Ferrous Metals": 3037.0, - "Transport Equipment": 14993.0, - "Machinery Equipment": 4724.0, + "Transport equipment": 14993.0, + "Machinery equipment": 4724.0, "Textiles and leather": 1742.0, "Wood and wood products": 0.0, - "Other Industrial Sectors": 10825.0, + "Other industrial sectors": 10825.0, "current electricity": 53760.0, } ) @@ -199,8 +201,12 @@ def get_energy_ratio(country, eurostat_dir, jrc_dir, year): if country == "CH": e_country = e_switzerland * tj_to_ktoe else: + ct_eurostat = country.replace("GB","UK") + if ct_eurostat == "UK": + year=2019 + logger.info("Assume Eurostat data for GB from 2019.") # estimate physical output, energy consumption in the sector and country - fn = f"{eurostat_dir}/{country}-Energy-balance-sheets-April-2023-edition.xlsb" + fn = f"{eurostat_dir}/{ct_eurostat}-Energy-balance-sheets-April-2023-edition.xlsb" df = pd.read_excel( fn, sheet_name=str(min(2021, year)), @@ -210,19 +216,19 @@ def get_energy_ratio(country, eurostat_dir, jrc_dir, year): ) e_country = df.loc[eb_sectors.keys(), "Total"].rename(eb_sectors) - fn = f"{jrc_dir}/JRC-IDEES-2015_Industry_EU28.xlsx" + fn = f"{jrc_dir}/EU27/JRC-IDEES-2021_Industry_EU27.xlsx" with mute_print(): df = pd.read_excel(fn, sheet_name="Ind_Summary", index_col=0, header=0).squeeze( "columns" ) - assert df.index[48] == "by sector" + assert df.index[49] == "by sector" year_i = df.columns.get_loc(year) - e_eu28 = df.iloc[49:76, year_i] - e_eu28.index = e_eu28.index.str.lstrip() + e_eu27 = df.iloc[50:77, year_i] + e_eu27.index = e_eu27.index.str.lstrip() - e_ratio = e_country / e_eu28 + e_ratio = e_country / e_eu27 return pd.Series({k: e_ratio[v] for k, v in sub2sect.items()}) @@ -230,7 +236,7 @@ def get_energy_ratio(country, eurostat_dir, jrc_dir, year): def industry_production_per_country(country, year, eurostat_dir, jrc_dir): def get_sector_data(sector, country): jrc_country = jrc_names.get(country, country) - fn = f"{jrc_dir}/JRC-IDEES-2015_Industry_{jrc_country}.xlsx" + fn = f"{jrc_dir}/{jrc_country}/JRC-IDEES-2021_Industry_{jrc_country}.xlsx" sheet = sub_sheet_name_dict[sector] with mute_print(): df = pd.read_excel(fn, sheet_name=sheet, index_col=0, header=0).squeeze( @@ -245,10 +251,10 @@ def industry_production_per_country(country, year, eurostat_dir, jrc_dir): return df - ct = "EU28" if country not in eu28 else country + ct = "EU27" if country not in eu27 else country demand = pd.concat([get_sector_data(s, ct) for s in sect2sub]) - if country not in eu28: + if country not in eu27: demand *= get_energy_ratio(country, eurostat_dir, jrc_dir, year) demand.name = country From fb2ab4c50bf715cbfb3e0962738c97e7782fa1aa Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 29 Jul 2024 15:44:30 +0200 Subject: [PATCH 19/55] update ammonia data --- rules/build_sector.smk | 2 +- scripts/build_ammonia_production.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index cd259d39..227f64da 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -437,7 +437,7 @@ rule build_salt_cavern_potentials: rule build_ammonia_production: input: - usgs="data/bundle/myb1-2017-nitro.xls", + usgs="data/bundle/myb1-2021-nitro-ert.xlsx", output: ammonia_production=resources("ammonia_production.csv"), threads: 1 diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index 37692f2f..83dfde20 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -18,7 +18,8 @@ Outputs Description ------- -This functions takes data from the `Minerals Yearbook `_ (June 2024) published by the US Geological Survey (USGS) and the National Minerals Information Center and extracts the annual ammonia production per country in ktonN/a. The data is converted to ktonNH3/a. +This functions takes data from the `Minerals Yearbook `_ + (July 2024) published by the US Geological Survey (USGS) and the National Minerals Information Center and extracts the annual ammonia production per country in ktonN/a. The data is converted to ktonNH3/a. """ import country_converter as coco @@ -48,9 +49,9 @@ if __name__ == "__main__": ammonia.index = cc.convert(ammonia.index, to="iso2") - years = [str(i) for i in range(2013, 2018)] + years = [str(i) for i in range(2017, 2022)] - ammonia = ammonia[years] + ammonia = ammonia.rename(columns=lambda x: str(x))[years] # convert from ktonN to ktonNH3 ammonia *= 17 / 14 From 314f6c6f0feeccaac9ed6ea1fba4bff28b2e1c00 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 29 Jul 2024 16:00:13 +0200 Subject: [PATCH 20/55] retrieve ammonia demand data --- rules/retrieve.smk | 9 +++++++ scripts/retrieve_ammonia_demand.py | 42 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 scripts/retrieve_ammonia_demand.py diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 24e387c5..7bf7c680 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -65,6 +65,15 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", script: "../scripts/retrieve_jrc_idees.py" + rule retrieve_ammonia_demand: + output: + "data/bundle/myb1-2021-nitro-ert.xlsx", + log: + "logs/retrieve_ammonia_demand.log", + retries: 2 + script: + "../scripts/retrieve_ammonia_demand.py" + rule retrieve_eurostat_household_data: output: "data/eurostat/eurostat-household_energy_balances-february_2024.csv", diff --git a/scripts/retrieve_ammonia_demand.py b/scripts/retrieve_ammonia_demand.py new file mode 100644 index 00000000..cdfa582b --- /dev/null +++ b/scripts/retrieve_ammonia_demand.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2024- The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +""" +Retrieve ammonia demand from https://www.usgs.gov/centers/national-minerals-information-center/nitrogen-statistics-and-information. +""" + +import logging +import os +import zipfile +from pathlib import Path + +from _helpers import configure_logging, progress_retrieve, set_scenario_config + +logger = logging.getLogger(__name__) + +# Define the base URL +url = "https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/media/files/myb1-2022-nitro-ert.xlsx" + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("retrieve_ammonia_demand") + rootpath = ".." + else: + rootpath = "." + + configure_logging(snakemake) + set_scenario_config(snakemake) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + + to_fn = snakemake.output[0] + + + # download .zip file + logger.info(f"Downloading Ammonia demand from {url}.") + progress_retrieve(url, to_fn, disable=disable_progress) + + + logger.info(f"Ammonia demand data available in '{to_fn}'.") From aa820fa0464480ec5e11c1782d2ce1791ad9f370 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 29 Jul 2024 16:00:37 +0200 Subject: [PATCH 21/55] change industry reference year to 2019 --- config/config.default.yaml | 2 +- scripts/build_industrial_production_per_country.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 373f4a6c..521013b0 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -716,7 +716,7 @@ industry: MWh_CH4_per_tMeOH: 10.25 MWh_MeOH_per_tMeOH: 5.528 hotmaps_locate_missing: false - reference_year: 2021 + reference_year: 2019 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#costs diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 1bb75271..332627dd 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -293,6 +293,7 @@ def separate_basic_chemicals(demand, year): """ Separate basic chemicals into ammonia, chlorine, methanol and HVC. """ + # ammonia data from 2017-2021 ammonia = pd.read_csv(snakemake.input.ammonia_production, index_col=0) there = ammonia.index.intersection(demand.index) @@ -301,8 +302,11 @@ def separate_basic_chemicals(demand, year): logger.info(f"Following countries have no ammonia demand: {missing.tolist()}") demand["Ammonia"] = 0.0 - - demand.loc[there, "Ammonia"] = ammonia.loc[there, str(year)] + + year_to_use = min(max(year, 2017), 2021) + if year_to_use != year: + logger.info(f"Using data from {year_to_use} for ammonia production.") + demand.loc[there, "Ammonia"] = ammonia.loc[there, str(year_to_use)] demand["Basic chemicals"] -= demand["Ammonia"] From 2f3fde26d0cc832818bcd85a51952591e8efb302 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 29 Jul 2024 16:09:38 +0200 Subject: [PATCH 22/55] update ammonia demand data to 2022 --- rules/build_sector.smk | 2 +- rules/retrieve.smk | 2 +- scripts/build_ammonia_production.py | 4 ++-- scripts/build_industrial_production_per_country.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 227f64da..0ced9c05 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -437,7 +437,7 @@ rule build_salt_cavern_potentials: rule build_ammonia_production: input: - usgs="data/bundle/myb1-2021-nitro-ert.xlsx", + usgs="data/bundle/myb1-2022-nitro-ert.xlsx", output: ammonia_production=resources("ammonia_production.csv"), threads: 1 diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 7bf7c680..120c2e1b 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -67,7 +67,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", rule retrieve_ammonia_demand: output: - "data/bundle/myb1-2021-nitro-ert.xlsx", + "data/bundle/myb1-2022-nitro-ert.xlsx", log: "logs/retrieve_ammonia_demand.log", retries: 2 diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index 83dfde20..e0e2de5e 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -43,13 +43,13 @@ if __name__ == "__main__": skiprows=5, header=0, index_col=0, - skipfooter=19, + skipfooter=7, na_values=["--"], ) ammonia.index = cc.convert(ammonia.index, to="iso2") - years = [str(i) for i in range(2017, 2022)] + years = [str(i) for i in range(2018, 2023)] ammonia = ammonia.rename(columns=lambda x: str(x))[years] diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 332627dd..90bf5482 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -303,7 +303,7 @@ def separate_basic_chemicals(demand, year): demand["Ammonia"] = 0.0 - year_to_use = min(max(year, 2017), 2021) + year_to_use = min(max(year, 2018), 2022) if year_to_use != year: logger.info(f"Using data from {year_to_use} for ammonia production.") demand.loc[there, "Ammonia"] = ammonia.loc[there, str(year_to_use)] From b0e00249aa941b884336b35347c62c8df878ae1e Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 29 Jul 2024 17:37:06 +0200 Subject: [PATCH 23/55] adjust mapping energy demand per ct --- ...ustrial_energy_demand_per_country_today.py | 95 ++++++++++--------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 114485e7..983b255e 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -36,7 +36,7 @@ For each country and each subcategory of - Integrated steelworks - Machinery Equipment - Methanol -- Other Industrial Sectors +- Other industrial sectors - Other chemicals - Other non-ferrous metals - Paper production @@ -74,46 +74,49 @@ ktoe_to_twh = 0.011630 # name in JRC-IDEES Energy Balances sector_sheets = { - "Integrated steelworks": "cisb", - "Electric arc": "cise", - "Alumina production": "cnfa", - "Aluminium - primary production": "cnfp", - "Aluminium - secondary production": "cnfs", - "Other non-ferrous metals": "cnfo", - "Basic chemicals": "cbch", - "Other chemicals": "coch", - "Pharmaceutical products etc.": "cpha", - "Basic chemicals feedstock": "cpch", - "Cement": "ccem", - "Ceramics & other NMM": "ccer", - "Glass production": "cgla", - "Pulp production": "cpul", - "Paper production": "cpap", - "Printing and media reproduction": "cprp", - "Food, beverages and tobacco": "cfbt", - "Transport Equipment": "ctre", - "Machinery Equipment": "cmae", - "Textiles and leather": "ctel", - "Wood and wood products": "cwwp", - "Mining and quarrying": "cmiq", - "Construction": "ccon", - "Non-specified": "cnsi", + "Integrated steelworks": "FC_IND_IS_BF_E", + "Electric arc": "FC_IND_IS_EA_E", + "Alumina production": "FC_IND_NFM_AM_E", + "Aluminium - primary production": "FC_IND_NFM_PA_E", + "Aluminium - secondary production": "FC_IND_NFM_SA_E", + "Other non-ferrous metals": "FC_IND_NFM_OM_E", + "Basic chemicals": "FC_IND_CPC_BC_E", + "Other chemicals": "FC_IND_CPC_OC_E", + "Pharmaceutical products etc.": "FC_IND_CPC_PH_E", + "Basic chemicals feedstock": "FC_IND_CPC_BC_E", + "Cement": "FC_IND_NMM_CM_E", + "Ceramics & other NMM": "FC_IND_NMM_CR_E", + "Glass production": "FC_IND_NMM_GL_E", + "Pulp production": "FC_IND_PPP_PU_E", + "Paper production": "FC_IND_PPP_PA_E", + "Printing and media reproduction": "FC_IND_PPP_PR_E", + "Food, beverages and tobacco": "FC_IND_FBT_E", + "Transport equipment": "FC_IND_TE_E", + "Machinery equipment": "FC_IND_MAC_E", + "Textiles and leather": "FC_IND_TL_E", + "Wood and wood products": "FC_IND_WP_E", + "Mining and quarrying": "FC_IND_MQ_E", + "Construction": "FC_IND_CON_E", + "Non-specified": "FC_IND_NSP_E", } fuels = { - "All Products": "all", - "Solid Fuels": "solid", - "Total petroleum products (without biofuels)": "liquid", - "Gases": "gas", + "Total": "all", + "Solid fossil fuels": "solid", + "Peat and peat products": "solid", + "Oil shale and oil sands": "solid", + "Oil and petroleum products": "liquid", + "Manufactured gases": "gas", + "Natural gas": "gas", "Nuclear heat": "heat", - "Derived heat": "heat", - "Biomass and Renewable wastes": "biomass", - "Wastes (non-renewable)": "waste", + "Heat": "heat", + "Renewables and biofuels": "biomass", + "Non-renewable waste": "waste", "Electricity": "electricity", } -eu28 = cc.EU28as("ISO2").ISO2.tolist() +eu27 = cc.EU27as("ISO2").ISO2.tolist() jrc_names = {"GR": "EL", "GB": "UK"} @@ -139,7 +142,7 @@ def industrial_energy_demand_per_country(country, year, jrc_dir): ) sel = ["Mining and quarrying", "Construction", "Non-specified"] - df["Other Industrial Sectors"] = df[sel].sum(axis=1) + df["Other industrial sectors"] = df[sel].sum(axis=1) df["Basic chemicals"] += df["Basic chemicals feedstock"] df.drop(columns=sel + ["Basic chemicals feedstock"], index="all", inplace=True) @@ -189,20 +192,20 @@ def separate_basic_chemicals(demand, production): return demand -def add_non_eu28_industrial_energy_demand(countries, demand, production): - non_eu28 = countries.difference(eu28) - if non_eu28.empty: +def add_non_eu27_industrial_energy_demand(countries, demand, production): + non_eu27 = countries.difference(eu27) + if non_eu27.empty: return demand - eu28_production = production.loc[countries.intersection(eu28)].sum() - eu28_energy = demand.groupby(level=1).sum() - eu28_averages = eu28_energy / eu28_production + eu27_production = production.loc[countries.intersection(eu27)].sum() + eu27_energy = demand.groupby(level=1).sum() + eu27_averages = eu27_energy / eu27_production - demand_non_eu28 = pd.concat( - {k: v * eu28_averages for k, v in production.loc[non_eu28].iterrows()} + demand_non_eu27 = pd.concat( + {k: v * eu27_averages for k, v in production.loc[non_eu27].iterrows()} ) - return pd.concat([demand, demand_non_eu28]) + return pd.concat([demand, demand_non_eu27]) def industrial_energy_demand(countries, year): @@ -232,10 +235,10 @@ if __name__ == "__main__": set_scenario_config(snakemake) params = snakemake.params.industry - year = params.get("reference_year", 2015) + year = params.get("reference_year", 2019) countries = pd.Index(snakemake.params.countries) - demand = industrial_energy_demand(countries.intersection(eu28), year) + demand = industrial_energy_demand(countries.intersection(eu27), year) # output in MtMaterial/a production = ( @@ -245,7 +248,7 @@ if __name__ == "__main__": demand = separate_basic_chemicals(demand, production) - demand = add_non_eu28_industrial_energy_demand(countries, demand, production) + demand = add_non_eu27_industrial_energy_demand(countries, demand, production) # for format compatibility demand = demand.stack(future_stack=True).unstack(level=[0, 2]) From b92f0d0c3086e03ff4ae88733e3ca46bc0af206c Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 30 Jul 2024 15:26:28 +0200 Subject: [PATCH 24/55] update industry sector ratios to new JRC data --- rules/build_sector.smk | 2 +- scripts/build_industry_sector_ratios.py | 442 +++++++++++++----------- 2 files changed, 235 insertions(+), 209 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 0ced9c05..0722bffa 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -459,7 +459,7 @@ rule build_industry_sector_ratios: ammonia=config_provider("sector", "ammonia", default=False), input: ammonia_production=resources("ammonia_production.csv"), - idees="data/bundle/jrc-idees-2015", + idees="data/bundle/jrc-idees-2021", output: industry_sector_ratios=resources("industry_sector_ratios.csv"), threads: 1 diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index b7a62d91..0f8bec24 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -17,7 +17,7 @@ Relevant Settings Inputs ------- - ``resources/ammonia_production.csv`` -- ``data/bundle-sector/jrc-idees-2015`` +- ``data/bundle-sector/jrc-idees-2021`` Outputs ------- @@ -50,40 +50,17 @@ The unit of the specific energy consumption is MWh/t material and tCO2/t materia import pandas as pd from _helpers import mute_print, set_scenario_config +import country_converter as coco + + +cc = coco.CountryConverter() # GWh/ktoe OR MWh/toe toe_to_MWh = 11.630 -eu28 = [ - "FR", - "DE", - "GB", - "IT", - "ES", - "PL", - "SE", - "NL", - "BE", - "FI", - "DK", - "PT", - "RO", - "AT", - "BG", - "EE", - "GR", - "LV", - "CZ", - "HU", - "IE", - "SK", - "LT", - "HR", - "LU", - "SI", - "CY", - "MT", -] +eu27 = cc.EU27as("ISO2").ISO2.tolist() + + sheet_names = { "Iron and steel": "ISI", @@ -92,11 +69,11 @@ sheet_names = { "Pulp, paper and printing": "PPA", "Food, beverages and tobacco": "FBT", "Non Ferrous Metals": "NFM", - "Transport Equipment": "TRE", - "Machinery Equipment": "MAE", + "Transport equipment": "TRE", + "Machinery equipment": "MAE", "Textiles and leather": "TEL", "Wood and wood products": "WWP", - "Other Industrial Sectors": "OIS", + "Other industrial sectors": "OIS", } @@ -116,7 +93,7 @@ index = [ ] -def load_idees_data(sector, country="EU28"): +def load_idees_data(sector, country="EU27"): suffixes = {"out": "", "fec": "_fec", "ued": "_ued", "emi": "_emi"} sheets = {k: sheet_names[sector] + v for k, v in suffixes.items()} @@ -125,7 +102,7 @@ def load_idees_data(sector, country="EU28"): with mute_print(): idees = pd.read_excel( - f"{snakemake.input.idees}/JRC-IDEES-2015_Industry_{country}.xlsx", + f"{snakemake.input.idees}/{country}/JRC-IDEES-2021_Industry_{country}.xlsx", sheet_name=list(sheets.values()), index_col=0, header=0, @@ -134,15 +111,25 @@ def load_idees_data(sector, country="EU28"): for k, v in sheets.items(): idees[k] = idees.pop(v).squeeze() + idees[k] = idees[k][year] return idees def iron_and_steel(): - # There are two different approaches to produce iron and steel: - # i.e., integrated steelworks and electric arc. - # Electric arc approach has higher efficiency and relies more on electricity. - # We assume that integrated steelworks will be replaced by electric arc entirely. + """ + This function calculates the energy consumption and emissions for different + approaches to producing iron and steel. The two primary approaches are + integrated steelworks and electric arc furnaces (EAF). The function assumes + that integrated steelworks will be replaced entirely by electric arc + furnaces due to their higher efficiency and greater reliance on electricity. + + Returns: + pd.DataFrame: A DataFrame containing the energy consumption (in MWh/t material) + and process emissions (in tCO2/t material) for different steel + production approaches, including electric arc, DRI + electric arc, + and integrated steelworks. + """ sector = "Iron and steel" idees = load_idees_data(sector) @@ -155,51 +142,51 @@ def iron_and_steel(): df[sector] = 0.0 - s_fec = idees["fec"][51:57] + s_fec = idees["fec"][52:68] assert s_fec.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] - df.at["elec", sector] += s_fec[sel].sum() + df.at["elec", sector] += s_fec.loc[sel].sum() - df.at["heat", sector] += s_fec["Low enthalpy heat"] + df.at["heat", sector] += s_fec.loc["Low-enthalpy heat"] subsector = "Steel: Smelters" - s_fec = idees["fec"][61:67] - s_ued = idees["ued"][61:67] + s_fec = idees["fec"][63:68] + s_ued = idees["ued"][63:68] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector # efficiency changes due to transforming all the smelters into methane - key = "Natural gas (incl. biogas)" - eff_met = s_ued[key] / s_fec[key] + key = "Natural gas and biogas" + eff_met = s_ued.loc[key] / s_fec.loc[key] df.at["methane", sector] += s_ued[subsector] / eff_met subsector = "Steel: Electric arc" - s_fec = idees["fec"][67:68] + s_fec = idees["fec"][69:70] assert s_fec.index[0] == subsector df.at["elec", sector] += s_fec[subsector] - subsector = "Steel: Furnaces, Refining and Rolling" - s_fec = idees["fec"][68:75] - s_ued = idees["ued"][68:75] + subsector = "Steel: Furnaces, refining and rolling" + s_fec = idees["fec"][70:77] + s_ued = idees["ued"][70:77] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector - key = "Steel: Furnaces, Refining and Rolling - Electric" + key = "Steel: Furnaces, refining and rolling - Electric" eff = s_ued[key] / s_fec[key] # assume fully electrified, other processes scaled by used energy df.at["elec", sector] += s_ued[subsector] / eff - subsector = "Steel: Products finishing" - s_fec = idees["fec"][75:92] - s_ued = idees["ued"][75:92] + subsector = "Steel: Product finishing" + s_fec = idees["fec"][77:95] + s_ued = idees["ued"][77:95] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector - key = "Steel: Products finishing - Electric" + key = "Steel: Product finishing - Electric" eff = s_ued[key] / s_fec[key] # assume fully electrified @@ -207,7 +194,7 @@ def iron_and_steel(): # Process emissions (per physical output) - s_emi = idees["emi"][51:93] + s_emi = idees["emi"][52:95] assert s_emi.index[0] == sector s_out = idees["out"][7:8] @@ -242,63 +229,63 @@ def iron_and_steel(): df[sector] = 0.0 - s_fec = idees["fec"][3:9] + s_fec = idees["fec"][3:50] assert s_fec.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] - subsector = "Steel: Sinter/Pellet making" + subsector = 'Steel: Sinter/Pellet-making' - s_fec = idees["fec"][13:19] - s_ued = idees["ued"][13:19] + s_fec = idees["fec"][14:20] + s_ued = idees["ued"][14:20] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector df.loc["elec", sector] += s_fec["Electricity"] - sel = ["Natural gas (incl. biogas)", "Residual fuel oil"] + sel = ["Natural gas and biogas", "Fuel oil"] df.loc["methane", sector] += s_fec[sel].sum() df.loc["coal", sector] += s_fec["Solids"] - subsector = "Steel: Blast /Basic oxygen furnace" + subsector = 'Steel: Blast /Basic oxygen furnace' - s_fec = idees["fec"][19:25] - s_ued = idees["ued"][19:25] + s_fec = idees["fec"][20:26] + s_ued = idees["ued"][20:26] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector - sel = ["Natural gas (incl. biogas)", "Residual fuel oil"] + sel = ["Natural gas and biogas", "Fuel oil"] df.loc["methane", sector] += s_fec[sel].sum() df.loc["coal", sector] += s_fec["Solids"] df.loc["coke", sector] = s_fec["Coke"] - subsector = "Steel: Furnaces, Refining and Rolling" + subsector = "Steel: Furnaces, refining and rolling" - s_fec = idees["fec"][25:32] - s_ued = idees["ued"][25:32] + s_fec = idees["fec"][26:33] + s_ued = idees["ued"][26:33] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector - key = "Steel: Furnaces, Refining and Rolling - Electric" + key = 'Steel: Furnaces, refining and rolling - Electric' eff = s_ued[key] / s_fec[key] # assume fully electrified, other processes scaled by used energy df.loc["elec", sector] += s_ued[subsector] / eff - subsector = "Steel: Products finishing" + subsector = "Steel: Product finishing" - s_fec = idees["fec"][32:49] - s_ued = idees["ued"][32:49] + s_fec = idees["fec"][33:50] + s_ued = idees["ued"][33:50] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector - key = "Steel: Products finishing - Electric" + key = 'Steel: Product finishing - Electric' eff = s_ued[key] / s_fec[key] # assume fully electrified @@ -306,7 +293,7 @@ def iron_and_steel(): # Process emissions (per physical output) - s_emi = idees["emi"][3:50] + s_emi = idees["emi"][3:51] assert s_emi.index[0] == sector s_out = idees["out"][6:7] @@ -323,6 +310,17 @@ def iron_and_steel(): def chemicals_industry(): + """ + This function calculates the energy consumption and emissions for the chemicals industry, + focusing on various subsectors such as basic chemicals, steam processing, furnaces, + and process cooling. The function also accounts for specific processes in ammonia, + chlorine, methanol production, and other chemicals. + + Returns: + pd.DataFrame: A DataFrame containing the energy consumption (in MWh/t material) + and process emissions (in tCO2/t material) for various subsectors + within the chemicals industry. + """ sector = "Chemicals Industry" idees = load_idees_data(sector) @@ -340,15 +338,15 @@ def chemicals_industry(): sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] subsector = "Chemicals: Feedstock (energy used as raw material)" - # There are Solids, Refinery gas, LPG, Diesel oil, Residual fuel oil, + # There are Solids, Refinery gas, LPG, Diesel oil, Fuel oil, # Other liquids, Naphtha, Natural gas for feedstock. # Naphta represents 47%, methane 17%. LPG (18%) solids, refinery gas, - # diesel oil, residual fuel oils and other liquids are assimilated to Naphtha + # diesel oil, Fuel oils and other liquids are assimilated to Naphtha - s_fec = idees["fec"][13:22] + s_fec = idees["fec"][14:23] assert s_fec.index[0] == subsector df.loc["naphtha", sector] += s_fec["Naphtha"] @@ -362,7 +360,7 @@ def chemicals_industry(): "Refinery gas", "LPG", "Diesel oil", - "Residual fuel oil", + "Fuel oil", "Other liquids", ] df.loc["naphtha", sector] += s_fec[sel].sum() @@ -372,21 +370,21 @@ def chemicals_industry(): # converted to methane, since we need >1000 C temperatures here. # The current efficiency of methane is assumed in the conversion. - s_fec = idees["fec"][22:33] - s_ued = idees["ued"][22:33] + s_fec = idees["fec"][23:34] + s_ued = idees["ued"][23:34] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector # efficiency of natural gas - eff_ch4 = s_ued["Natural gas (incl. biogas)"] / s_fec["Natural gas (incl. biogas)"] + eff_ch4 = s_ued["Natural gas and biogas"] / s_fec["Natural gas and biogas"] # replace all fec by methane df.loc["methane", sector] += s_ued[subsector] / eff_ch4 subsector = "Chemicals: Furnaces" - s_fec = idees["fec"][33:41] - s_ued = idees["ued"][33:41] + s_fec = idees["fec"][34:42] + s_ued = idees["ued"][34:42] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector @@ -399,8 +397,8 @@ def chemicals_industry(): subsector = "Chemicals: Process cooling" - s_fec = idees["fec"][41:55] - s_ued = idees["ued"][41:55] + s_fec = idees["fec"][42:56] + s_ued = idees["ued"][42:56] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector @@ -412,17 +410,17 @@ def chemicals_industry(): subsector = "Chemicals: Generic electric process" - s_fec = idees["fec"][55:56] + s_fec = idees["fec"][56:57] assert s_fec.index[0] == subsector df.loc["elec", sector] += s_fec[subsector] # Process emissions - # Correct everything by subtracting 2015's ammonia demand and + # Correct everything by subtracting 2019's ammonia demand and # putting in ammonia demand for H2 and electricity separately - s_emi = idees["emi"][3:57] + s_emi = idees["emi"][3:58] assert s_emi.index[0] == sector # convert from MtHVC/a to ktHVC/a @@ -447,7 +445,7 @@ def chemicals_industry(): # subtract ammonia energy demand (in ktNH3/a) ammonia = pd.read_csv(snakemake.input.ammonia_production, index_col=0) - ammonia_total = ammonia.loc[ammonia.index.intersection(eu28), str(year)].sum() + ammonia_total = ammonia.loc[ammonia.index.intersection(eu27), str(year)].sum() df.loc["methane", sector] -= ammonia_total * params["MWh_CH4_per_tNH3_SMR"] df.loc["elec", sector] -= ammonia_total * params["MWh_elec_per_tNH3_SMR"] @@ -507,22 +505,22 @@ def chemicals_industry(): df[sector] = 0.0 - s_fec = idees["fec"][58:64] + s_fec = idees["fec"][59:65] assert s_fec.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] - subsector = "Chemicals: High enthalpy heat processing" + subsector = 'Chemicals: High-enthalpy heat processing' - s_fec = idees["fec"][68:81] - s_ued = idees["ued"][68:81] + s_fec = idees["fec"][70:83] + s_ued = idees["ued"][70:83] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector - key = "High enthalpy heat processing - Electric (microwave)" + key = 'High-enthalpy heat processing - Electric (microwave)' eff_elec = s_ued[key] / s_fec[key] # assume fully electrified @@ -530,8 +528,8 @@ def chemicals_industry(): subsector = "Chemicals: Furnaces" - s_fec = idees["fec"][81:89] - s_ued = idees["ued"][81:89] + s_fec = idees["fec"][83:92] + s_ued = idees["ued"][83:92] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector @@ -543,8 +541,8 @@ def chemicals_industry(): subsector = "Chemicals: Process cooling" - s_fec = idees["fec"][89:103] - s_ued = idees["ued"][89:103] + s_fec = idees["fec"][91:105] + s_ued = idees["ued"][91:105] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector @@ -556,14 +554,14 @@ def chemicals_industry(): subsector = "Chemicals: Generic electric process" - s_fec = idees["fec"][103:104] + s_fec = idees["fec"][105:106] assert s_fec.index[0] == subsector df.loc["elec", sector] += s_fec[subsector] # Process emissions - s_emi = idees["emi"][58:105] + s_emi = idees["emi"][59:107] s_out = idees["out"][9:10] assert s_emi.index[0] == sector assert sector in str(s_out.index) @@ -581,22 +579,22 @@ def chemicals_industry(): df[sector] = 0.0 - s_fec = idees["fec"][106:112] + s_fec = idees["fec"][108:114] assert s_fec.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] - subsector = "Chemicals: High enthalpy heat processing" + subsector = 'Chemicals: High-enthalpy heat processing' - s_fec = idees["fec"][116:129] - s_ued = idees["ued"][116:129] + s_fec = idees["fec"][119:132] + s_ued = idees["ued"][119:132] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector - key = "High enthalpy heat processing - Electric (microwave)" + key = 'High-enthalpy heat processing - Electric (microwave)' eff_elec = s_ued[key] / s_fec[key] # assume fully electrified @@ -604,8 +602,8 @@ def chemicals_industry(): subsector = "Chemicals: Furnaces" - s_fec = idees["fec"][129:137] - s_ued = idees["ued"][129:137] + s_fec = idees["fec"][132:140] + s_ued = idees["ued"][132:140] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector @@ -617,8 +615,8 @@ def chemicals_industry(): subsector = "Chemicals: Process cooling" - s_fec = idees["fec"][137:151] - s_ued = idees["ued"][137:151] + s_fec = idees["fec"][140:154] + s_ued = idees["ued"][140:154] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector @@ -630,7 +628,7 @@ def chemicals_industry(): subsector = "Chemicals: Generic electric process" - s_fec = idees["fec"][151:152] + s_fec = idees["fec"][154:155] s_out = idees["out"][10:11] assert s_fec.index[0] == subsector assert sector in str(s_out.index) @@ -648,8 +646,16 @@ def chemicals_industry(): def nonmetalic_mineral_products(): - # This includes cement, ceramic and glass production. - # This includes process emissions related to the fabrication of clinker. + """ + This function calculates the energy consumption and emissions for the non-metallic mineral + products industry, focusing on three main sectors: cement, ceramics, and glass production. + It takes into account the specific processes and their associated energy types and emissions. + + Returns: + pd.DataFrame: A DataFrame containing the energy consumption (in MWh/t material) + and process emissions (in tCO2/t material) for the cement, ceramics, + and glass production sectors. + """ sector = "Non-metallic mineral products" idees = load_idees_data(sector) @@ -681,27 +687,27 @@ def nonmetalic_mineral_products(): sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # pre-processing: keep existing elec and biomass, rest to methane df.loc["elec", sector] += s_fec["Cement: Grinding, milling of raw material"] - df.loc["biomass", sector] += s_fec["Biomass"] + df.loc["biomass", sector] += s_fec['Biomass and waste'] df.loc["methane", sector] += ( - s_fec["Cement: Pre-heating and pre-calcination"] - s_fec["Biomass"] + s_fec["Cement: Pre-heating and pre-calcination"] - s_fec['Biomass and waste'] ) subsector = "Cement: Clinker production (kilns)" - s_fec = idees["fec"][34:43] - s_ued = idees["ued"][34:43] + s_fec = idees["fec"][23:32] + s_ued = idees["ued"][23:32] assert s_fec.index[0] == subsector assert s_ued.index[0] == subsector - df.loc["biomass", sector] += s_fec["Biomass"] + df.loc["biomass", sector] += s_fec['Biomass and waste'] df.loc["methane", sector] += ( - s_fec["Cement: Clinker production (kilns)"] - s_fec["Biomass"] + s_fec["Cement: Clinker production (kilns)"] - s_fec['Biomass and waste'] ) - df.loc["elec", sector] += s_fec["Cement: Grinding, packaging"] + df.loc["elec", sector] += s_fec['Cement: Grinding, packaging and precasting'] # Process emissions @@ -709,7 +715,7 @@ def nonmetalic_mineral_products(): # Calcium carbonate -> lime + CO2 # CaCO3 -> CaO + CO2 - s_emi = idees["emi"][3:44] + s_emi = idees["emi"][3:45] assert s_emi.index[0] == sector s_out = idees["out"][7:8] @@ -737,19 +743,21 @@ def nonmetalic_mineral_products(): df[sector] = 0.0 - s_fec = idees["fec"][45:94] - s_ued = idees["ued"][45:94] + s_fec = idees["fec"][46:95] + s_ued = idees["ued"][46:95] assert s_fec.index[0] == sector assert s_ued.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # Efficiency changes due to electrification key = "Ceramics: Microwave drying and sintering" - eff_elec = s_ued[key] / s_fec[key] + # the values are zero in new JRC-data -> assume here value from JRC-2015 + # eff_elec = s_ued[key] / s_fec[key] + eff_elec = 11.6/26 sel = [ "Ceramics: Mixing of raw material", @@ -767,7 +775,7 @@ def nonmetalic_mineral_products(): df.loc["elec", sector] += s_ued["Ceramics: Product finishing"] / eff_elec - s_emi = idees["emi"][45:94] + s_emi = idees["emi"][46:96] assert s_emi.index[0] == sector s_out = idees["out"][8:9] @@ -795,15 +803,15 @@ def nonmetalic_mineral_products(): df[sector] = 0.0 - s_fec = idees["fec"][95:123] - s_ued = idees["ued"][95:123] + s_fec = idees["fec"][97:126] + s_ued = idees["ued"][97:126] assert s_fec.index[0] == sector assert s_ued.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # Efficiency changes due to electrification key = "Glass: Electric melting tank" @@ -817,7 +825,7 @@ def nonmetalic_mineral_products(): sel = ["Glass: Forming", "Glass: Annealing", "Glass: Finishing processes"] df.loc["elec", sector] += s_ued[sel].sum() / eff_elec - s_emi = idees["emi"][95:124] + s_emi = idees["emi"][97:127] assert s_emi.index[0] == sector s_out = idees["out"][9:10] @@ -834,8 +842,16 @@ def nonmetalic_mineral_products(): def pulp_paper_printing(): - # Pulp, paper and printing can be completely electrified. - # There are no process emissions associated to this sector. + """ + Models the energy consumption for the pulp, paper, and printing sector, + assuming complete electrification of all processes. This sector does not have + any process emissions associated with it. + + Returns: + pd.DataFrame: A DataFrame containing the energy consumption (in MWh/t material) + for the pulp, paper, and printing sector. + + """ sector = "Pulp, paper and printing" idees = load_idees_data(sector) @@ -857,15 +873,15 @@ def pulp_paper_printing(): df[sector] = 0.0 - s_fec = idees["fec"][3:28] - s_ued = idees["ued"][3:28] + s_fec = idees["fec"][3:29] + s_ued = idees["ued"][3:29] assert s_fec.index[0] == sector assert s_ued.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # Industry-specific sel = [ @@ -876,7 +892,7 @@ def pulp_paper_printing(): df.loc["elec", sector] += s_fec[sel].sum() # Efficiency changes due to biomass - eff_bio = s_ued["Biomass"] / s_fec["Biomass"] + eff_bio = s_ued['Biomass and waste'] / s_fec['Biomass and waste'] df.loc["biomass", sector] += s_ued["Pulp: Pulping thermal"] / eff_bio s_out = idees["out"][8:9] @@ -906,15 +922,15 @@ def pulp_paper_printing(): df[sector] = 0.0 - s_fec = idees["fec"][29:78] - s_ued = idees["ued"][29:78] + s_fec = idees["fec"][30:80] + s_ued = idees["ued"][30:80] assert s_fec.index[0] == sector assert s_ued.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # Industry-specific df.loc["elec", sector] += s_fec["Paper: Stock preparation"] @@ -925,22 +941,22 @@ def pulp_paper_printing(): # add electricity from process that is already electrified df.loc["elec", sector] += s_fec["Paper: Product finishing - Electricity"] - s_fec = idees["fec"][53:64] - s_ued = idees["ued"][53:64] + s_fec = idees["fec"][55:66] + s_ued = idees["ued"][55:66] assert s_fec.index[0] == "Paper: Paper machine - Steam use" assert s_ued.index[0] == "Paper: Paper machine - Steam use" # Efficiency changes due to biomass - eff_bio = s_ued["Biomass"] / s_fec["Biomass"] + eff_bio = s_ued['Biomass and waste'] / s_fec['Biomass and waste'] df.loc["biomass", sector] += s_ued["Paper: Paper machine - Steam use"] / eff_bio - s_fec = idees["fec"][66:77] - s_ued = idees["ued"][66:77] + s_fec = idees["fec"][68:79] + s_ued = idees["ued"][68:79] assert s_fec.index[0] == "Paper: Product finishing - Steam use" assert s_ued.index[0] == "Paper: Product finishing - Steam use" # Efficiency changes due to biomass - eff_bio = s_ued["Biomass"] / s_fec["Biomass"] + eff_bio = s_ued['Biomass and waste'] / s_fec['Biomass and waste'] df.loc["biomass", sector] += s_ued["Paper: Product finishing - Steam use"] / eff_bio s_out = idees["out"][9:10] @@ -959,8 +975,8 @@ def pulp_paper_printing(): df[sector] = 0.0 - s_fec = idees["fec"][79:90] - s_ued = idees["ued"][79:90] + s_fec = idees["fec"][81:93] + s_ued = idees["ued"][81:93] assert s_fec.index[0] == sector assert s_ued.index[0] == sector @@ -968,8 +984,8 @@ def pulp_paper_printing(): df.loc["elec", sector] += s_fec[sel].sum() df.loc["elec", sector] += s_ued[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] - df.loc["heat", sector] += s_ued["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] + df.loc["heat", sector] += s_ued["Low-enthalpy heat"] # Industry-specific df.loc["elec", sector] += s_fec["Printing and publishing"] @@ -986,8 +1002,16 @@ def pulp_paper_printing(): def food_beverages_tobacco(): - # Food, beverages and tobaco can be completely electrified. - # There are no process emissions associated to this sector. + """ + Calculates the energy consumption for the food, beverages, and tobacco sector, + assuming complete electrification of all processes. This sector does not have + any process emissions associated with it. + + + Returns: + pd.DataFrame: A DataFrame containing the energy consumption (in MWh/t material) + for the food, beverages, and tobacco sector. + """ sector = "Food, beverages and tobacco" idees = load_idees_data(sector) @@ -996,15 +1020,15 @@ def food_beverages_tobacco(): df[sector] = 0.0 - s_fec = idees["fec"][3:78] - s_ued = idees["ued"][3:78] + s_fec = idees["fec"][3:79] + s_ued = idees["ued"][3:79] assert s_fec.index[0] == sector assert s_ued.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # Efficiency changes due to electrification @@ -1052,7 +1076,7 @@ def non_ferrous_metals(): # Alumina - # High enthalpy heat is converted to methane. + # High-enthalpy heat is converted to methane. # Process heat at T>500C is required here. # Refining is electrified. # There are no process emissions associated to Alumina manufacturing. @@ -1069,24 +1093,24 @@ def non_ferrous_metals(): sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # High-enthalpy heat is transformed into methane - s_fec = idees["fec"][13:24] - s_ued = idees["ued"][13:24] - assert s_fec.index[0] == "Alumina production: High enthalpy heat" - assert s_ued.index[0] == "Alumina production: High enthalpy heat" + s_fec = idees["fec"][14:25] + s_ued = idees["ued"][14:25] + assert s_fec.index[0] == 'Alumina production: High-enthalpy heat' + assert s_ued.index[0] == 'Alumina production: High-enthalpy heat' - eff_met = s_ued["Natural gas (incl. biogas)"] / s_fec["Natural gas (incl. biogas)"] + eff_met = s_ued["Natural gas and biogas"] / s_fec["Natural gas and biogas"] df.loc["methane", sector] += ( - s_fec["Alumina production: High enthalpy heat"] / eff_met + s_fec['Alumina production: High-enthalpy heat'] / eff_met ) # Efficiency changes due to electrification - s_fec = idees["fec"][24:30] - s_ued = idees["ued"][24:30] + s_fec = idees["fec"][25:31] + s_ued = idees["ued"][25:31] assert s_fec.index[0] == "Alumina production: Refining" assert s_ued.index[0] == "Alumina production: Refining" @@ -1111,15 +1135,15 @@ def non_ferrous_metals(): df[sector] = 0.0 - s_fec = idees["fec"][31:66] - s_ued = idees["ued"][31:66] + s_fec = idees["fec"][32:68] + s_ued = idees["ued"][32:68] assert s_fec.index[0] == sector assert s_ued.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # Add aluminium electrolysis (smelting df.loc["elec", sector] += s_fec["Aluminium electrolysis (smelting)"] @@ -1135,7 +1159,7 @@ def non_ferrous_metals(): eff_elec = s_ued[key] / s_fec[key] df.loc["elec", sector] += s_ued["Aluminium finishing"] / eff_elec - s_emi = idees["emi"][31:67] + s_emi = idees["emi"][32:69] assert s_emi.index[0] == sector s_out = idees["out"][11:12] @@ -1160,15 +1184,15 @@ def non_ferrous_metals(): df[sector] = 0.0 - s_fec = idees["fec"][68:109] - s_ued = idees["ued"][68:109] + s_fec = idees["fec"][70:112] + s_ued = idees["ued"][70:112] assert s_fec.index[0] == sector assert s_ued.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # Efficiency changes due to electrification key = "Secondary aluminium - Electric" @@ -1181,7 +1205,7 @@ def non_ferrous_metals(): key = "Aluminium processing (metallurgy e.g. cast house, reheating)" df.loc["elec", sector] += s_ued[key] / eff_elec - key = "Aluminium finishing - Electric" + key = 'Aluminium finishing - Electric' eff_elec = s_ued[key] / s_fec[key] df.loc["elec", sector] += s_ued["Aluminium finishing"] / eff_elec @@ -1200,15 +1224,15 @@ def non_ferrous_metals(): df[sector] = 0.0 - s_fec = idees["fec"][110:152] - s_ued = idees["ued"][110:152] + s_fec = idees["fec"][113:156] + s_ued = idees["ued"][113:156] assert s_fec.index[0] == sector assert s_ued.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # Efficiency changes due to electrification key = "Metal production - Electric" @@ -1224,7 +1248,7 @@ def non_ferrous_metals(): eff_elec = s_ued[key] / s_fec[key] df.loc["elec", sector] += s_ued["Metal finishing"] / eff_elec - s_emi = idees["emi"][110:153] + s_emi = idees["emi"][113:157] assert s_emi.index[0] == sector s_out = idees["out"][13:14] @@ -1247,22 +1271,22 @@ def non_ferrous_metals(): def transport_equipment(): - sector = "Transport Equipment" + sector = "Transport equipment" idees = load_idees_data(sector) df = pd.DataFrame(index=index) df[sector] = 0.0 - s_fec = idees["fec"][3:45] - s_ued = idees["ued"][3:45] + s_fec = idees["fec"][3:46] + s_ued = idees["ued"][3:46] assert s_fec.index[0] == sector assert s_ued.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # Efficiency changes due to electrification key = "Trans. Eq.: Electric Foundries" @@ -1281,7 +1305,7 @@ def transport_equipment(): df.loc["elec", sector] += s_fec["Trans. Eq.: Product finishing"] # Steam processing is supplied with biomass - eff_biomass = s_ued["Biomass"] / s_fec["Biomass"] + eff_biomass = s_ued['Biomass and waste'] / s_fec['Biomass and waste'] df.loc["biomass", sector] += s_ued["Trans. Eq.: Steam processing"] / eff_biomass s_out = idees["out"][3:4] @@ -1297,7 +1321,7 @@ def transport_equipment(): def machinery_equipment(): - sector = "Machinery Equipment" + sector = "Machinery equipment" idees = load_idees_data(sector) @@ -1305,15 +1329,15 @@ def machinery_equipment(): df[sector] = 0.0 - s_fec = idees["fec"][3:45] - s_ued = idees["ued"][3:45] + s_fec = idees["fec"][3:46] + s_ued = idees["ued"][3:46] assert s_fec.index[0] == sector assert s_ued.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # Efficiency changes due to electrification key = "Mach. Eq.: Electric Foundries" @@ -1333,7 +1357,7 @@ def machinery_equipment(): df.loc["elec", sector] += s_fec["Mach. Eq.: Product finishing"] # Steam processing is supplied with biomass - eff_biomass = s_ued["Biomass"] / s_fec["Biomass"] + eff_biomass = s_ued['Biomass and waste'] / s_fec['Biomass and waste'] df.loc["biomass", sector] += s_ued["Mach. Eq.: Steam processing"] / eff_biomass s_out = idees["out"][3:4] @@ -1357,26 +1381,28 @@ def textiles_and_leather(): df[sector] = 0.0 - s_fec = idees["fec"][3:57] - s_ued = idees["ued"][3:57] + s_fec = idees["fec"][3:58] + s_ued = idees["ued"][3:58] assert s_fec.index[0] == sector assert s_ued.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # Efficiency changes due to electrification key = "Textiles: Electric drying" - eff_elec = s_ued[key] / s_fec[key] + # in new JRC data zero assume old data + # eff_elec = s_ued[key] / s_fec[key] + eff_elec = 73.7 / 146.6 df.loc["elec", sector] += s_ued["Textiles: Drying"] / eff_elec df.loc["elec", sector] += s_fec["Textiles: Electric general machinery"] df.loc["elec", sector] += s_fec["Textiles: Finishing Electric"] # Steam processing is supplied with biomass - eff_biomass = s_ued[15:26]["Biomass"] / s_fec[15:26]["Biomass"] + eff_biomass = s_ued[15:26]['Biomass and waste'] / s_fec[15:26]['Biomass and waste'] df.loc["biomass", sector] += ( s_ued["Textiles: Pretreatment with steam"] / eff_biomass ) @@ -1405,15 +1431,15 @@ def wood_and_wood_products(): df[sector] = 0.0 - s_fec = idees["fec"][3:46] - s_ued = idees["ued"][3:46] + s_fec = idees["fec"][3:47] + s_ued = idees["ued"][3:47] assert s_fec.index[0] == sector assert s_ued.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # Efficiency changes due to electrification key = "Wood: Electric drying" @@ -1424,7 +1450,7 @@ def wood_and_wood_products(): df.loc["elec", sector] += s_fec["Wood: Finishing Electric"] # Steam processing is supplied with biomass - eff_biomass = s_ued[15:25]["Biomass"] / s_fec[15:25]["Biomass"] + eff_biomass = s_ued[15:25]['Biomass and waste'] / s_fec[15:25]['Biomass and waste'] df.loc["biomass", sector] += ( s_ued["Wood: Specific processes with steam"] / eff_biomass ) @@ -1442,23 +1468,23 @@ def wood_and_wood_products(): def other_industrial_sectors(): - sector = "Other Industrial Sectors" - + + sector = "Other industrial sectors" idees = load_idees_data(sector) df = pd.DataFrame(index=index) df[sector] = 0.0 - s_fec = idees["fec"][3:67] - s_ued = idees["ued"][3:67] + s_fec = idees["fec"][3:68] + s_ued = idees["ued"][3:68] assert s_fec.index[0] == sector assert s_ued.index[0] == sector sel = ["Lighting", "Air compressors", "Motor drives", "Fans and pumps"] df.loc["elec", sector] += s_fec[sel].sum() - df.loc["heat", sector] += s_fec["Low enthalpy heat"] + df.loc["heat", sector] += s_fec["Low-enthalpy heat"] # Efficiency changes due to electrification key = "Other Industrial sectors: Electric processing" @@ -1484,7 +1510,7 @@ def other_industrial_sectors(): df.loc["elec", sector] += s_fec[key] # Steam processing is supplied with biomass - eff_biomass = s_ued[15:25]["Biomass"] / s_fec[15:25]["Biomass"] + eff_biomass = s_ued[15:25]['Biomass and waste'] / s_fec[15:25]['Biomass and waste'] df.loc["biomass", sector] += ( s_ued["Other Industrial sectors: Steam processing"] / eff_biomass ) @@ -1508,10 +1534,10 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industry_sector_ratios") set_scenario_config(snakemake) - # TODO make params option - year = 2015 params = snakemake.params.industry + + year = params["reference_year"] df = pd.concat( [ From c52737db7290b60be4f752d64039d931f4284150 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 30 Jul 2024 15:28:08 +0200 Subject: [PATCH 25/55] small fixes in docstrings --- scripts/build_industrial_energy_demand_per_country_today.py | 4 ++-- scripts/build_industry_sector_ratios_intermediate.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 983b255e..8f63795e 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -34,7 +34,7 @@ For each country and each subcategory of - Glass production - HVC - Integrated steelworks -- Machinery Equipment +- Machinery equipment - Methanol - Other industrial sectors - Other chemicals @@ -44,7 +44,7 @@ For each country and each subcategory of - Printing and media reproduction - Pulp production - Textiles and leather -- Transport Equipment +- Transport equipment - Wood and wood products the output file contains the energy demand in TWh/a for the following carriers diff --git a/scripts/build_industry_sector_ratios_intermediate.py b/scripts/build_industry_sector_ratios_intermediate.py index 5fe042ab..1f6dfcd6 100644 --- a/scripts/build_industry_sector_ratios_intermediate.py +++ b/scripts/build_industry_sector_ratios_intermediate.py @@ -56,8 +56,8 @@ For each bus, the following industry subcategories - Aluminium - primary production - Aluminium - secondary production - Other non-ferrous metals -- Transport Equipment -- Machinery Equipment +- Transport equipment +- Machinery equipment - Textiles and leather - Wood and wood products - Other Industrial Sectors From d48500fc3b49eb93f3440071b33069c8de334ada Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 31 Jul 2024 16:06:16 +0200 Subject: [PATCH 26/55] use ffill and bfill --- scripts/build_energy_totals.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 44bb470f..f4905927 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -587,7 +587,8 @@ def build_idees(countries: List[str]) -> pd.DataFrame: def fill_missing_years(fill_values: pd.Series) -> pd.Series: """ - Fill missing years for some countries by mean over the other years. + Fill missing years for some countries by first using forward fill (ffill) + and then backward fill (bfill). Parameters ---------- @@ -598,16 +599,23 @@ def fill_missing_years(fill_values: pd.Series) -> pd.Series: Returns ------- pd.Series - A pandas Series with zero values replaced by the mean value of the corresponding - country. + A pandas Series with zero values replaced by the forward-filled and + backward-filled values of the corresponding country. Notes ----- - - The function groups the data by the 'country' level and computes the mean for each group. - - Zero values in the original Series are replaced by the mean value of their respective country group. + - The function groups the data by the 'country' level and performs forward fill + and backward fill to fill zero values. + - Zero values in the original Series are replaced by the ffilled and bfilled + value of their respective country group. """ - means = fill_values.groupby(level="country").transform("mean") - return fill_values.where(fill_values != 0, means) + # Replace zero values with NaN for correct filling + fill_values = fill_values.replace(0, pd.NA) + + # Forward fill and then backward fill within each country group + fill_values = fill_values.groupby(level="country").ffill().bfill() + + return fill_values def build_energy_totals( @@ -724,6 +732,7 @@ def build_energy_totals( eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=[0, 1]).sum() ) # fill missing years for some countries by mean over the other years + breakpoint() fill_values = fill_missing_years(fill_values) df.loc[to_fill, f"{fuel} {sector}"] = fill_values From e37824336edc318e0b4803dd28106c3e13e49804 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 31 Jul 2024 14:10:09 +0000 Subject: [PATCH 27/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_energy_totals.py | 14 +-- ...build_industrial_production_per_country.py | 7 +- scripts/build_industry_sector_ratios.py | 96 +++++++++---------- scripts/retrieve_ammonia_demand.py | 2 - 4 files changed, 57 insertions(+), 62 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index f4905927..18c13666 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -587,7 +587,7 @@ def build_idees(countries: List[str]) -> pd.DataFrame: def fill_missing_years(fill_values: pd.Series) -> pd.Series: """ - Fill missing years for some countries by first using forward fill (ffill) + Fill missing years for some countries by first using forward fill (ffill) and then backward fill (bfill). Parameters @@ -599,14 +599,14 @@ def fill_missing_years(fill_values: pd.Series) -> pd.Series: Returns ------- pd.Series - A pandas Series with zero values replaced by the forward-filled and + A pandas Series with zero values replaced by the forward-filled and backward-filled values of the corresponding country. Notes ----- - - The function groups the data by the 'country' level and performs forward fill + - The function groups the data by the 'country' level and performs forward fill and backward fill to fill zero values. - - Zero values in the original Series are replaced by the ffilled and bfilled + - Zero values in the original Series are replaced by the ffilled and bfilled value of their respective country group. """ # Replace zero values with NaN for correct filling @@ -913,9 +913,9 @@ def build_district_heat_share(countries: List[str], idees: pd.DataFrame) -> pd.S """ # district heating share - district_heat = idees[["distributed heat residential", "distributed heat services"]].sum( - axis=1 - ) + district_heat = idees[ + ["distributed heat residential", "distributed heat services"] + ].sum(axis=1) total_heat = ( idees[["thermal uses residential", "thermal uses services"]] .sum(axis=1) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 90bf5482..9c543303 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -167,7 +167,6 @@ eb_sectors = { } - # TODO: this should go in a csv in `data` # Annual energy consumption in Switzerland by sector in 2015 (in TJ) # From: Energieverbrauch in der Industrie und im Dienstleistungssektor, Der Bundesrat @@ -201,9 +200,9 @@ def get_energy_ratio(country, eurostat_dir, jrc_dir, year): if country == "CH": e_country = e_switzerland * tj_to_ktoe else: - ct_eurostat = country.replace("GB","UK") + ct_eurostat = country.replace("GB", "UK") if ct_eurostat == "UK": - year=2019 + year = 2019 logger.info("Assume Eurostat data for GB from 2019.") # estimate physical output, energy consumption in the sector and country fn = f"{eurostat_dir}/{ct_eurostat}-Energy-balance-sheets-April-2023-edition.xlsb" @@ -302,7 +301,7 @@ def separate_basic_chemicals(demand, year): logger.info(f"Following countries have no ammonia demand: {missing.tolist()}") demand["Ammonia"] = 0.0 - + year_to_use = min(max(year, 2018), 2022) if year_to_use != year: logger.info(f"Using data from {year_to_use} for ammonia production.") diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index 0f8bec24..760dca76 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -48,10 +48,9 @@ If the `config["industry"]["ammonia"] assume here value from JRC-2015 # eff_elec = s_ued[key] / s_fec[key] - eff_elec = 11.6/26 + eff_elec = 11.6 / 26 sel = [ "Ceramics: Mixing of raw material", @@ -844,13 +845,12 @@ def nonmetalic_mineral_products(): def pulp_paper_printing(): """ Models the energy consumption for the pulp, paper, and printing sector, - assuming complete electrification of all processes. This sector does not have - any process emissions associated with it. - + assuming complete electrification of all processes. This sector does not + have any process emissions associated with it. + Returns: pd.DataFrame: A DataFrame containing the energy consumption (in MWh/t material) for the pulp, paper, and printing sector. - """ sector = "Pulp, paper and printing" @@ -892,7 +892,7 @@ def pulp_paper_printing(): df.loc["elec", sector] += s_fec[sel].sum() # Efficiency changes due to biomass - eff_bio = s_ued['Biomass and waste'] / s_fec['Biomass and waste'] + eff_bio = s_ued["Biomass and waste"] / s_fec["Biomass and waste"] df.loc["biomass", sector] += s_ued["Pulp: Pulping thermal"] / eff_bio s_out = idees["out"][8:9] @@ -947,7 +947,7 @@ def pulp_paper_printing(): assert s_ued.index[0] == "Paper: Paper machine - Steam use" # Efficiency changes due to biomass - eff_bio = s_ued['Biomass and waste'] / s_fec['Biomass and waste'] + eff_bio = s_ued["Biomass and waste"] / s_fec["Biomass and waste"] df.loc["biomass", sector] += s_ued["Paper: Paper machine - Steam use"] / eff_bio s_fec = idees["fec"][68:79] @@ -956,7 +956,7 @@ def pulp_paper_printing(): assert s_ued.index[0] == "Paper: Product finishing - Steam use" # Efficiency changes due to biomass - eff_bio = s_ued['Biomass and waste'] / s_fec['Biomass and waste'] + eff_bio = s_ued["Biomass and waste"] / s_fec["Biomass and waste"] df.loc["biomass", sector] += s_ued["Paper: Product finishing - Steam use"] / eff_bio s_out = idees["out"][9:10] @@ -1003,13 +1003,12 @@ def pulp_paper_printing(): def food_beverages_tobacco(): """ - Calculates the energy consumption for the food, beverages, and tobacco sector, - assuming complete electrification of all processes. This sector does not have - any process emissions associated with it. + Calculates the energy consumption for the food, beverages, and tobacco + sector, assuming complete electrification of all processes. This sector + does not have any process emissions associated with it. - Returns: - pd.DataFrame: A DataFrame containing the energy consumption (in MWh/t material) + pd.DataFrame: A DataFrame containing the energy consumption (in MWh/t material) for the food, beverages, and tobacco sector. """ @@ -1099,12 +1098,12 @@ def non_ferrous_metals(): s_fec = idees["fec"][14:25] s_ued = idees["ued"][14:25] - assert s_fec.index[0] == 'Alumina production: High-enthalpy heat' - assert s_ued.index[0] == 'Alumina production: High-enthalpy heat' + assert s_fec.index[0] == "Alumina production: High-enthalpy heat" + assert s_ued.index[0] == "Alumina production: High-enthalpy heat" eff_met = s_ued["Natural gas and biogas"] / s_fec["Natural gas and biogas"] df.loc["methane", sector] += ( - s_fec['Alumina production: High-enthalpy heat'] / eff_met + s_fec["Alumina production: High-enthalpy heat"] / eff_met ) # Efficiency changes due to electrification @@ -1205,7 +1204,7 @@ def non_ferrous_metals(): key = "Aluminium processing (metallurgy e.g. cast house, reheating)" df.loc["elec", sector] += s_ued[key] / eff_elec - key = 'Aluminium finishing - Electric' + key = "Aluminium finishing - Electric" eff_elec = s_ued[key] / s_fec[key] df.loc["elec", sector] += s_ued["Aluminium finishing"] / eff_elec @@ -1305,7 +1304,7 @@ def transport_equipment(): df.loc["elec", sector] += s_fec["Trans. Eq.: Product finishing"] # Steam processing is supplied with biomass - eff_biomass = s_ued['Biomass and waste'] / s_fec['Biomass and waste'] + eff_biomass = s_ued["Biomass and waste"] / s_fec["Biomass and waste"] df.loc["biomass", sector] += s_ued["Trans. Eq.: Steam processing"] / eff_biomass s_out = idees["out"][3:4] @@ -1357,7 +1356,7 @@ def machinery_equipment(): df.loc["elec", sector] += s_fec["Mach. Eq.: Product finishing"] # Steam processing is supplied with biomass - eff_biomass = s_ued['Biomass and waste'] / s_fec['Biomass and waste'] + eff_biomass = s_ued["Biomass and waste"] / s_fec["Biomass and waste"] df.loc["biomass", sector] += s_ued["Mach. Eq.: Steam processing"] / eff_biomass s_out = idees["out"][3:4] @@ -1402,7 +1401,7 @@ def textiles_and_leather(): df.loc["elec", sector] += s_fec["Textiles: Finishing Electric"] # Steam processing is supplied with biomass - eff_biomass = s_ued[15:26]['Biomass and waste'] / s_fec[15:26]['Biomass and waste'] + eff_biomass = s_ued[15:26]["Biomass and waste"] / s_fec[15:26]["Biomass and waste"] df.loc["biomass", sector] += ( s_ued["Textiles: Pretreatment with steam"] / eff_biomass ) @@ -1450,7 +1449,7 @@ def wood_and_wood_products(): df.loc["elec", sector] += s_fec["Wood: Finishing Electric"] # Steam processing is supplied with biomass - eff_biomass = s_ued[15:25]['Biomass and waste'] / s_fec[15:25]['Biomass and waste'] + eff_biomass = s_ued[15:25]["Biomass and waste"] / s_fec[15:25]["Biomass and waste"] df.loc["biomass", sector] += ( s_ued["Wood: Specific processes with steam"] / eff_biomass ) @@ -1468,7 +1467,7 @@ def wood_and_wood_products(): def other_industrial_sectors(): - + sector = "Other industrial sectors" idees = load_idees_data(sector) @@ -1510,7 +1509,7 @@ def other_industrial_sectors(): df.loc["elec", sector] += s_fec[key] # Steam processing is supplied with biomass - eff_biomass = s_ued[15:25]['Biomass and waste'] / s_fec[15:25]['Biomass and waste'] + eff_biomass = s_ued[15:25]["Biomass and waste"] / s_fec[15:25]["Biomass and waste"] df.loc["biomass", sector] += ( s_ued["Other Industrial sectors: Steam processing"] / eff_biomass ) @@ -1534,9 +1533,8 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industry_sector_ratios") set_scenario_config(snakemake) - params = snakemake.params.industry - + year = params["reference_year"] df = pd.concat( diff --git a/scripts/retrieve_ammonia_demand.py b/scripts/retrieve_ammonia_demand.py index cdfa582b..56e326bb 100644 --- a/scripts/retrieve_ammonia_demand.py +++ b/scripts/retrieve_ammonia_demand.py @@ -33,10 +33,8 @@ if __name__ == "__main__": to_fn = snakemake.output[0] - # download .zip file logger.info(f"Downloading Ammonia demand from {url}.") progress_retrieve(url, to_fn, disable=disable_progress) - logger.info(f"Ammonia demand data available in '{to_fn}'.") From 4499fa4e73eed5a148171831cd03a3ace8e048d2 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 31 Jul 2024 16:14:12 +0200 Subject: [PATCH 28/55] fix typo --- doc/release_notes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 9c1ba489..16f48235 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,11 +10,11 @@ Release Notes Upcoming Release ================ -* Upadte JRC-IDEES-2015 to `JRC-IDEES-2021 +* Update JRC-IDEES-2015 to `JRC-IDEES-2021 `__. -* Updata Ammonia production from USGS to 2022 `data +* Update Ammonia production from USGS to 2022 `data `__. From c9557a5988df701c522e30ced7525028f1ca5f9c Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 31 Jul 2024 18:50:41 +0200 Subject: [PATCH 29/55] remove breakpoint --- scripts/build_energy_totals.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 18c13666..2afd7bb9 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -732,7 +732,6 @@ def build_energy_totals( eurostat.loc[slicer, eurostat_fuels[fuel]].groupby(level=[0, 1]).sum() ) # fill missing years for some countries by mean over the other years - breakpoint() fill_values = fill_missing_years(fill_values) df.loc[to_fill, f"{fuel} {sector}"] = fill_values From 5cea20456fa03fe1b355f50c3a4a50de42690867 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 31 Jul 2024 18:59:57 +0200 Subject: [PATCH 30/55] correct hyperlink in release notes --- doc/release_notes.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 16f48235..f26ec91c 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,13 +10,9 @@ Release Notes Upcoming Release ================ -* Update JRC-IDEES-2015 to `JRC-IDEES-2021 -`__. - - -* Update Ammonia production from USGS to 2022 `data -`__. +* Update JRC-IDEES-2015 to `JRC-IDEES-2021 `__. +* Update Ammonia production from USGS to 2022 `data `__. * Renamed the carrier of batteries in BEVs from `battery storage` to `EV battery` and the corresponding bus carrier from `Li ion` to `EV battery`. This is to avoid confusion with stationary battery storage. From 0ec367ed7a771782495002437c2d1bae8da66179 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Tue, 6 Aug 2024 17:37:36 +0200 Subject: [PATCH 31/55] save idees directly in data not in bundle Co-authored-by: Fabian Neumann --- rules/retrieve.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 720fa914..8d0615db 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -58,7 +58,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", rule retrieve_jrc_idees: output: - directory("data/bundle/jrc-idees-2021"), + directory("data/jrc-idees-2021"), log: "logs/retrieve_jrc_idees.log", retries: 2 From 2c7e30bb1774b804efe88cb04c7f4ecad6f47469 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 6 Aug 2024 17:46:23 +0200 Subject: [PATCH 32/55] remove ammonia demand script --- rules/build_sector.smk | 2 +- rules/retrieve.smk | 9 ------- scripts/retrieve_ammonia_demand.py | 40 ------------------------------ 3 files changed, 1 insertion(+), 50 deletions(-) delete mode 100644 scripts/retrieve_ammonia_demand.py diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 62655b7b..8e9cb4de 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -429,7 +429,7 @@ rule build_salt_cavern_potentials: rule build_ammonia_production: input: - usgs="data/bundle/myb1-2022-nitro-ert.xlsx", + usgs=storage("https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/media/files/myb1-2022-nitro-ert.xlsx"), output: ammonia_production=resources("ammonia_production.csv"), threads: 1 diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 8d0615db..b5950b92 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -23,7 +23,6 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", "corine/g250_clc06_V18_5.tif", "eea/UNFCCC_v23.csv", "nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", - "myb1-2017-nitro.xls", "emobility/KFZ__count", "emobility/Pkw__count", "h2_salt_caverns_GWh_per_sqkm.geojson", @@ -65,14 +64,6 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", script: "../scripts/retrieve_jrc_idees.py" - rule retrieve_ammonia_demand: - output: - "data/bundle/myb1-2022-nitro-ert.xlsx", - log: - "logs/retrieve_ammonia_demand.log", - retries: 2 - script: - "../scripts/retrieve_ammonia_demand.py" rule retrieve_eurostat_household_data: output: diff --git a/scripts/retrieve_ammonia_demand.py b/scripts/retrieve_ammonia_demand.py deleted file mode 100644 index 56e326bb..00000000 --- a/scripts/retrieve_ammonia_demand.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: : 2024- The PyPSA-Eur Authors -# -# SPDX-License-Identifier: MIT -""" -Retrieve ammonia demand from https://www.usgs.gov/centers/national-minerals-information-center/nitrogen-statistics-and-information. -""" - -import logging -import os -import zipfile -from pathlib import Path - -from _helpers import configure_logging, progress_retrieve, set_scenario_config - -logger = logging.getLogger(__name__) - -# Define the base URL -url = "https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/media/files/myb1-2022-nitro-ert.xlsx" - -if __name__ == "__main__": - if "snakemake" not in globals(): - from _helpers import mock_snakemake - - snakemake = mock_snakemake("retrieve_ammonia_demand") - rootpath = ".." - else: - rootpath = "." - - configure_logging(snakemake) - set_scenario_config(snakemake) - disable_progress = snakemake.config["run"].get("disable_progressbar", False) - - to_fn = snakemake.output[0] - - # download .zip file - logger.info(f"Downloading Ammonia demand from {url}.") - progress_retrieve(url, to_fn, disable=disable_progress) - - logger.info(f"Ammonia demand data available in '{to_fn}'.") From 5725e8bf22818fc771d2c4ec054a91aa6894c544 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 6 Aug 2024 17:47:35 +0200 Subject: [PATCH 33/55] remove comment about uk flights --- scripts/build_energy_totals.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 2afd7bb9..a0dc07fd 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -492,7 +492,6 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: assert df.index[2] == "Domestic" ct_totals["total domestic aviation passenger"] = df.iloc[2] - # TODO added Ukraine to intra EU flights assert df.index[6] == "International - Intra-EEAwUK" assert df.index[7] == "International - Extra-EEAwUK" ct_totals["total international aviation passenger"] = df.iloc[[6, 7]].sum() From a9d991f547caba1cf76d96a6d2dd05a839e104d0 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 6 Aug 2024 17:51:10 +0200 Subject: [PATCH 34/55] fix release notes --- doc/release_notes.rst | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index f26ec91c..20103814 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -9,10 +9,15 @@ Release Notes Upcoming Release ================ +* Update JRC-IDEES-2015 to `JRC-IDEES-2021 `__. The reference year is changed from 2015 to 2019. -* Update JRC-IDEES-2015 to `JRC-IDEES-2021 `__. +* split solid biomass potentials into solid biomass and municipal solid waste. Add option to use municipal solid waste. This option is only activated in combination with the flag ``waste_to_energy`` -* Update Ammonia production from USGS to 2022 `data `__. +* Add option to import solid biomass + +* Add option to produce electrobiofuels (flag ``electrobiofuels``) from solid biomass and hydrogen, as a combination of BtL and Fischer-Tropsch to make more use of the biogenic carbon + +* Add flag ``sector: fossil_fuels`` in config to remove the option of importing fossil fuels * Renamed the carrier of batteries in BEVs from `battery storage` to `EV battery` and the corresponding bus carrier from `Li ion` to `EV battery`. This is to avoid confusion with stationary battery storage. @@ -45,8 +50,25 @@ Upcoming Release * Bugfix: Impose minimum value of zero for district heating progress between current and future market share in :mod:`build_district_heat_share`. +* The ``{scope}`` wildcard was removed, since its outputs were not used. + * Enable parallelism in :mod:`determine_availability_matrix_MD_UA.py` and remove plots. This requires the use of temporary files. +* Updated pre-built `weather data cutouts + `__. These are now merged cutouts with + solar irradiation from the new SARAH-3 dataset while taking all other + variables from ERA5. Cutouts are now available for multiple years (2010, 2013, + 2019, and 2023). + +* Added option ``solving: curtailment_mode``` which fixes the dispatch profiles + of generators with time-varying p_max_pu by setting ``p_min_pu = p_max_pu`` + and adds an auxiliary curtailment generator with negative sign (to absorb + excess power) at every AC bus. This can speed up the solving process as the + curtailment decision is aggregated into a single generator per region. + +* In :mod:`base_network`, replace own voronoi polygon calculation function with + Geopandas `gdf.voronoi_polygons` method. + PyPSA-Eur 0.11.0 (25th May 2024) ===================================== From 3c528b7b0ac29ba85159fe9a80e51ce2c5f7310a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 15:51:56 +0000 Subject: [PATCH 35/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_sector.smk | 4 +++- rules/retrieve.smk | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 8e9cb4de..b7ce9265 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -429,7 +429,9 @@ rule build_salt_cavern_potentials: rule build_ammonia_production: input: - usgs=storage("https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/media/files/myb1-2022-nitro-ert.xlsx"), + usgs=storage( + "https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/media/files/myb1-2022-nitro-ert.xlsx" + ), output: ammonia_production=resources("ammonia_production.csv"), threads: 1 diff --git a/rules/retrieve.smk b/rules/retrieve.smk index b5950b92..1dde3da3 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -64,7 +64,6 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", script: "../scripts/retrieve_jrc_idees.py" - rule retrieve_eurostat_household_data: output: "data/eurostat/eurostat-household_energy_balances-february_2024.csv", From 682d53d1b5618915945a782f42364ff7117f33ce Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 7 Aug 2024 10:07:39 +0200 Subject: [PATCH 36/55] remove /bundle --- rules/build_sector.smk | 8 ++++---- scripts/build_energy_totals.py | 4 +--- .../build_industrial_energy_demand_per_country_today.py | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index b7ce9265..c89e3cd6 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -282,7 +282,7 @@ rule build_energy_totals: co2="data/bundle/eea/UNFCCC_v23.csv", swiss="data/switzerland-new_format-all_years.csv", swiss_transport="data/gr-e-11.03.02.01.01-cc.csv", - idees="data/bundle/jrc-idees-2021", + idees="data/jrc-idees-2021", district_heat_share="data/district_heat_share.csv", eurostat="data/eurostat/Balances-April2023", eurostat_households="data/eurostat/eurostat-household_energy_balances-february_2024.csv", @@ -453,7 +453,7 @@ rule build_industry_sector_ratios: ammonia=config_provider("sector", "ammonia", default=False), input: ammonia_production=resources("ammonia_production.csv"), - idees="data/bundle/jrc-idees-2021", + idees="data/jrc-idees-2021", output: industry_sector_ratios=resources("industry_sector_ratios.csv"), threads: 1 @@ -503,7 +503,7 @@ rule build_industrial_production_per_country: countries=config_provider("countries"), input: ammonia_production=resources("ammonia_production.csv"), - jrc="data/bundle/jrc-idees-2021", + jrc="data/jrc-idees-2021", eurostat="data/eurostat/Balances-April2023", output: industrial_production_per_country=resources( @@ -650,7 +650,7 @@ rule build_industrial_energy_demand_per_country_today: countries=config_provider("countries"), industry=config_provider("industry"), input: - jrc="data/bundle/jrc-idees-2021", + jrc="data/jrc-idees-2021", industrial_production_per_country=resources( "industrial_production_per_country.csv" ), diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index a0dc07fd..229eb1b1 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -25,7 +25,7 @@ Inputs - `data/bundle/eea_UNFCCC_v23.csv`: CO2 emissions data from EEA. - `data/switzerland-new_format-all_years.csv`: Swiss energy data. - `data/gr-e-11.03.02.01.01-cc.csv`: Swiss transport data -- `data/bundle/jrc-idees`: JRC IDEES data. +- `data/jrc-idees`: JRC IDEES data. - `data/district_heat_share.csv`: District heating shares. - `data/eurostat/Balances-April2023`: Eurostat energy balances. - `data/eurostat/eurostat-household_energy_balances-february_2024.csv`: Eurostat household energy balances. @@ -496,8 +496,6 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: assert df.index[7] == "International - Extra-EEAwUK" ct_totals["total international aviation passenger"] = df.iloc[[6, 7]].sum() - # TODO freight changed from "Domestic and International - Intra-EU" -> split - # domestic and international (intra-EU and outside EU) assert df.index[9] == "Domestic" ct_totals["total domestic aviation freight"] = df.iloc[9] diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 8f63795e..fa82a660 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -8,7 +8,7 @@ Build industrial energy demand per country. Inputs ------- -- ``data/bundle/jrc-idees-2021`` +- ``data/jrc-idees-2021`` - ``industrial_production_per_country.csv`` Outputs From 8794f530504cc2a2c1734402e818c88e16508687 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 7 Aug 2024 10:11:07 +0200 Subject: [PATCH 37/55] rename eurostat_2015 --- scripts/build_energy_totals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 229eb1b1..9fd8a1f1 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -1252,9 +1252,9 @@ def rescale_idees_from_eurostat( main_cols = ["Total all products", "Electricity"] # read in the eurostat data for 2015 - eurostat_2015 = eurostat.xs(2021, level="year")[main_cols] + eurostat_2021 = eurostat.xs(2021, level="year")[main_cols] # calculate the ratio of the two data sets - ratio = eurostat[main_cols] / eurostat_2015 + ratio = eurostat[main_cols] / eurostat_2021 ratio = ratio.droplevel([2, 5]) cols_rename = {"Total all products": "total", "Electricity": "ele"} index_rename = {v: k for k, v in idees_rename.items()} From 6b3fc183866277bc856fd55b452c476bfdaeff36 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 7 Aug 2024 10:33:47 +0200 Subject: [PATCH 38/55] include review fneum build_industrial_production_per_country --- scripts/build_industrial_production_per_country.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 9c543303..4a9c43ed 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -202,13 +202,12 @@ def get_energy_ratio(country, eurostat_dir, jrc_dir, year): else: ct_eurostat = country.replace("GB", "UK") if ct_eurostat == "UK": - year = 2019 logger.info("Assume Eurostat data for GB from 2019.") # estimate physical output, energy consumption in the sector and country fn = f"{eurostat_dir}/{ct_eurostat}-Energy-balance-sheets-April-2023-edition.xlsb" df = pd.read_excel( fn, - sheet_name=str(min(2021, year)), + sheet_name=str(min(2019, year)), index_col=2, header=0, skiprows=4, @@ -292,7 +291,7 @@ def separate_basic_chemicals(demand, year): """ Separate basic chemicals into ammonia, chlorine, methanol and HVC. """ - # ammonia data from 2017-2021 + # ammonia data from 2018-2022 ammonia = pd.read_csv(snakemake.input.ammonia_production, index_col=0) there = ammonia.index.intersection(demand.index) @@ -304,7 +303,7 @@ def separate_basic_chemicals(demand, year): year_to_use = min(max(year, 2018), 2022) if year_to_use != year: - logger.info(f"Using data from {year_to_use} for ammonia production.") + logger.info(f"Year {year} outside data range. Using data from {year_to_use} for ammonia production.") demand.loc[there, "Ammonia"] = ammonia.loc[there, str(year_to_use)] demand["Basic chemicals"] -= demand["Ammonia"] From 7848e08bc77be21b58db1b569e973e55874a0b4a Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 7 Aug 2024 10:40:23 +0200 Subject: [PATCH 39/55] create previous idees split for aviation --- scripts/build_energy_totals.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 9fd8a1f1..0a407438 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -497,11 +497,12 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: ct_totals["total international aviation passenger"] = df.iloc[[6, 7]].sum() assert df.index[9] == "Domestic" - ct_totals["total domestic aviation freight"] = df.iloc[9] - assert df.index[10] == "International - Intra-EEAwUK" + ct_totals["total domestic aviation freight"] = df.iloc[[9,10]].sum() + + assert df.index[11] == "International - Extra-EEAwUK" - ct_totals["total international aviation freight"] = df.iloc[[10, 11]].sum() + ct_totals["total international aviation freight"] = df.iloc[11].sum() ct_totals["total domestic aviation"] = ( ct_totals["total domestic aviation freight"] From 7e75aab29537226dc348b27c31e26dac23ab23ed Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 7 Aug 2024 10:47:22 +0200 Subject: [PATCH 40/55] directly read in nan values --- scripts/build_energy_totals.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 0a407438..e508bd54 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -170,6 +170,7 @@ def eurostat_per_country(input_eurostat: str, country: str) -> pd.DataFrame: sheet_name=None, skiprows=4, index_col=list(range(4)), + na_values=":" ) sheet.pop("Cover") return pd.concat(sheet) @@ -502,7 +503,7 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: assert df.index[11] == "International - Extra-EEAwUK" - ct_totals["total international aviation freight"] = df.iloc[11].sum() + ct_totals["total international aviation freight"] = df.iloc[11] ct_totals["total domestic aviation"] = ( ct_totals["total domestic aviation freight"] @@ -607,8 +608,6 @@ def fill_missing_years(fill_values: pd.Series) -> pd.Series: - Zero values in the original Series are replaced by the ffilled and bfilled value of their respective country group. """ - # Replace zero values with NaN for correct filling - fill_values = fill_values.replace(0, pd.NA) # Forward fill and then backward fill within each country group fill_values = fill_values.groupby(level="country").ffill().bfill() From 5bf36c218c3bd25f04d5fb01e699e503b5523b7f Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 7 Aug 2024 11:49:33 +0200 Subject: [PATCH 41/55] add ch data as csv --- ...ch_industrial_production_per_subsector.csv | 16 +++++++ rules/build_sector.smk | 1 + ...build_industrial_production_per_country.py | 43 ++++++++----------- 3 files changed, 36 insertions(+), 24 deletions(-) create mode 100644 data/ch_industrial_production_per_subsector.csv diff --git a/data/ch_industrial_production_per_subsector.csv b/data/ch_industrial_production_per_subsector.csv new file mode 100644 index 00000000..acb7ed81 --- /dev/null +++ b/data/ch_industrial_production_per_subsector.csv @@ -0,0 +1,16 @@ +sector,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023 +Nahrung,19035,17728,16486,16782,16261,17062,17132,17335,15625,15713 +Textil / Leder,1845,1742,1450,1497,1433,1488,1331,1496,1371,1112 +Papier / Druck,12407,12257,11100,11086,9530,9436,8482,9186,8850,7657 +Chemie / Pharma,27253,27050,27501,26953,26506,26055,24889,25595,25028,23240 +Zement / Beton,15513,13307,13576,13429,13263,13221,12612,12843,12919,11400 +Andere NE-Mineralien,4029,3820,3884,4254,3546,3577,3475,3602,3603,3178 +Metall / Eisen,7841,7889,7638,8049,8053,7188,7051,6872,6782,5531 +NE-Metall,3386,3037,2833,2813,2743,2931,2719,2992,2896,2653 +Metall / Geräte,14652,14993,14272,14689,14300,15037,13946,15042,13862,12578 +Maschinen,4561,4724,4861,4957,4365,4364,4061,4290,3698,3545 +Andere Industrien,10750,10825,10225,9833,9689,9545,8696,9371,8823,8157 +current electricity,55142,53760,51302,52173,51604,51389,48933,51730,50926,46969 +,,,,,,,,,, +,,,,,,,,,, +"source: https://pubdb.bfe.admin.ch/de/publication/download/11817, accessed August 2024",,,,,,,,,, diff --git a/rules/build_sector.smk b/rules/build_sector.smk index c89e3cd6..4227422c 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -502,6 +502,7 @@ rule build_industrial_production_per_country: industry=config_provider("industry"), countries=config_provider("countries"), input: + ch_industrial_production="data/ch_industrial_production_per_subsector.csv", ammonia_production=resources("ammonia_production.csv"), jrc="data/jrc-idees-2021", eurostat="data/eurostat/Balances-April2023", diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 4a9c43ed..5b691be6 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -31,7 +31,7 @@ The industrial production is taken from the `JRC-IDEES ` dataset. The industrial production is calculated for the year specified in the config["industry"]["reference_year"]. -The ammonia production is provided by the rule `build_ammonia_production `. Since Switzerland is not part of the EU28 nor reported by eurostat, the energy consumption in the industrial sectors is taken from the `BFE dataset. +The ammonia production is provided by the rule `build_ammonia_production `. Since Switzerland is not part of the EU28 nor reported by eurostat, the energy consumption in the industrial sectors is taken from the `BFE dataset. After the industrial production is calculated, the basic chemicals are separated into ammonia, chlorine, methanol and HVC. The production of these chemicals is assumed to be proportional to the production of basic chemicals without ammonia. The following subcategories [kton/a] are considered: @@ -167,27 +167,19 @@ eb_sectors = { } -# TODO: this should go in a csv in `data` -# Annual energy consumption in Switzerland by sector in 2015 (in TJ) -# From: Energieverbrauch in der Industrie und im Dienstleistungssektor, Der Bundesrat -# http://www.bfe.admin.ch/themen/00526/00541/00543/index.html?lang=de&dossier_id=00775 -e_switzerland = pd.Series( - { - "Iron and steel": 7889.0, - "Chemical industry": 26871.0, - "Non-metallic mineral products": 15513.0 + 3820.0, - "Pulp, paper and printing": 12004.0, - "Food, beverages and tobacco": 17728.0, - "Non Ferrous Metals": 3037.0, - "Transport equipment": 14993.0, - "Machinery equipment": 4724.0, - "Textiles and leather": 1742.0, - "Wood and wood products": 0.0, - "Other industrial sectors": 10825.0, - "current electricity": 53760.0, +ch_mapping = { + "Nahrung": "Food, beverages and tobacco", + "Textil / Leder": "Textiles and leather", + "Papier / Druck": "Pulp, paper and printing", + "Chemie / Pharma": "Chemical industry", + "Zement / Beton": "Non-metallic mineral products", + "Andere NE-Mineralien": "Other non-ferrous metals", + "Metall / Eisen": "Iron and steel", + "NE-Metall": "Non Ferrous Metals", + "Metall / Geräte" : "Transport equipment", + "Maschinen": "Machinery equipment", + "Andere Industrien": "Other industrial sectors", } -) - def find_physical_output(df): start = np.where(df.index.str.contains("Physical output", na=""))[0][0] @@ -198,11 +190,14 @@ def find_physical_output(df): def get_energy_ratio(country, eurostat_dir, jrc_dir, year): if country == "CH": - e_country = e_switzerland * tj_to_ktoe + # data ranges between 2014-2023 + e_country = pd.read_csv(snakemake.input.ch_industrial_production, + index_col=0).dropna() + e_country = e_country.rename(index=ch_mapping).groupby(level=0).sum() + e_country = e_country[str(min(2019, year))] + e_country *= tj_to_ktoe else: ct_eurostat = country.replace("GB", "UK") - if ct_eurostat == "UK": - logger.info("Assume Eurostat data for GB from 2019.") # estimate physical output, energy consumption in the sector and country fn = f"{eurostat_dir}/{ct_eurostat}-Energy-balance-sheets-April-2023-edition.xlsb" df = pd.read_excel( From 6a51a1d94ed636bb5167ced7e75fb6945ad2d18f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Aug 2024 09:59:06 +0000 Subject: [PATCH 42/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/conf.py | 2 +- scripts/build_energy_totals.py | 5 ++--- ...build_industrial_production_per_country.py | 20 +++++++++++-------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index a166dd70..efce867e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -342,5 +342,5 @@ texinfo_documents = [ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - 'https://docs.python.org/': ('https://docs.python.org/3', None), + "https://docs.python.org/": ("https://docs.python.org/3", None), } diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index e508bd54..d8c2f521 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -170,7 +170,7 @@ def eurostat_per_country(input_eurostat: str, country: str) -> pd.DataFrame: sheet_name=None, skiprows=4, index_col=list(range(4)), - na_values=":" + na_values=":", ) sheet.pop("Cover") return pd.concat(sheet) @@ -499,9 +499,8 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: assert df.index[9] == "Domestic" assert df.index[10] == "International - Intra-EEAwUK" - ct_totals["total domestic aviation freight"] = df.iloc[[9,10]].sum() + ct_totals["total domestic aviation freight"] = df.iloc[[9, 10]].sum() - assert df.index[11] == "International - Extra-EEAwUK" ct_totals["total international aviation freight"] = df.iloc[11] diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 5b691be6..ccb4feca 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -172,14 +172,15 @@ ch_mapping = { "Textil / Leder": "Textiles and leather", "Papier / Druck": "Pulp, paper and printing", "Chemie / Pharma": "Chemical industry", - "Zement / Beton": "Non-metallic mineral products", + "Zement / Beton": "Non-metallic mineral products", "Andere NE-Mineralien": "Other non-ferrous metals", "Metall / Eisen": "Iron and steel", "NE-Metall": "Non Ferrous Metals", - "Metall / Geräte" : "Transport equipment", + "Metall / Geräte": "Transport equipment", "Maschinen": "Machinery equipment", - "Andere Industrien": "Other industrial sectors", - } + "Andere Industrien": "Other industrial sectors", +} + def find_physical_output(df): start = np.where(df.index.str.contains("Physical output", na=""))[0][0] @@ -191,11 +192,12 @@ def find_physical_output(df): def get_energy_ratio(country, eurostat_dir, jrc_dir, year): if country == "CH": # data ranges between 2014-2023 - e_country = pd.read_csv(snakemake.input.ch_industrial_production, - index_col=0).dropna() + e_country = pd.read_csv( + snakemake.input.ch_industrial_production, index_col=0 + ).dropna() e_country = e_country.rename(index=ch_mapping).groupby(level=0).sum() e_country = e_country[str(min(2019, year))] - e_country *= tj_to_ktoe + e_country *= tj_to_ktoe else: ct_eurostat = country.replace("GB", "UK") # estimate physical output, energy consumption in the sector and country @@ -298,7 +300,9 @@ def separate_basic_chemicals(demand, year): year_to_use = min(max(year, 2018), 2022) if year_to_use != year: - logger.info(f"Year {year} outside data range. Using data from {year_to_use} for ammonia production.") + logger.info( + f"Year {year} outside data range. Using data from {year_to_use} for ammonia production." + ) demand.loc[there, "Ammonia"] = ammonia.loc[there, str(year_to_use)] demand["Basic chemicals"] -= demand["Ammonia"] From 70db93fd32c208e632e66ff588dd5f72d5149867 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 9 Aug 2024 13:14:26 +0200 Subject: [PATCH 43/55] correct sheet name chemical feedstocks --- scripts/build_industrial_energy_demand_per_country_today.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index fa82a660..c120dafe 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -83,7 +83,7 @@ sector_sheets = { "Basic chemicals": "FC_IND_CPC_BC_E", "Other chemicals": "FC_IND_CPC_OC_E", "Pharmaceutical products etc.": "FC_IND_CPC_PH_E", - "Basic chemicals feedstock": "FC_IND_CPC_BC_E", + "Basic chemicals feedstock": "FC_IND_CPC_NE", "Cement": "FC_IND_NMM_CM_E", "Ceramics & other NMM": "FC_IND_NMM_CR_E", "Glass production": "FC_IND_NMM_GL_E", From c5256c1f370711a2f2fd21b29020581e3bb7e194 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 9 Aug 2024 14:14:02 +0200 Subject: [PATCH 44/55] add coke oven transformation output --- rules/build_sector.smk | 2 ++ scripts/build_energy_totals.py | 29 +++++++++++++++++++ ...ustrial_energy_demand_per_country_today.py | 18 +++++++++++- 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 1304e775..faad2bb1 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -301,6 +301,7 @@ rule build_energy_totals: eurostat="data/eurostat/Balances-April2023", eurostat_households="data/eurostat/eurostat-household_energy_balances-february_2024.csv", output: + transformation_output_coke=resources("transformation_output_coke.csv"), energy_name=resources("energy_totals.csv"), co2_name=resources("co2_totals.csv"), transport_name=resources("transport_data.csv"), @@ -666,6 +667,7 @@ rule build_industrial_energy_demand_per_country_today: countries=config_provider("countries"), industry=config_provider("industry"), input: + transformation_output_coke=resources("transformation_output_coke.csv"), jrc="data/jrc-idees-2021", industrial_production_per_country=resources( "industrial_production_per_country.csv" diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index d8c2f521..f7d754b8 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -1473,6 +1473,31 @@ def update_residential_from_eurostat(energy: pd.DataFrame) -> pd.DataFrame: ) +def build_transformation_output_coke(eurostat, fn): + """ + Extracts and builds the transformation output data for coke ovens from the + Eurostat dataset. + + This function specifically filters the Eurostat data to extract + transformation output related to coke ovens. + Since the transformation output for coke ovens + is not included in the final energy consumption of the iron and steel sector, + it needs to be processed and added separately. The filtered data is saved + as a CSV file. + + Parameters: + eurostat (pd.DataFrame): A pandas DataFrame containing Eurostat data with + a multi-level index + fn (str): The file path where the resulting CSV file should be saved. + + Output: + The resulting transformation output data for coke ovens is saved as a CSV + file at the path specified in fn. + """ + slicer = pd.IndexSlice[:,:,:, "Coke ovens", "Other sources", :] + df = eurostat.loc[slicer, :].droplevel(level=[2,3,4,5]) + df.to_csv(fn) + # %% if __name__ == "__main__": if "snakemake" not in globals(): @@ -1498,6 +1523,10 @@ if __name__ == "__main__": nprocesses=snakemake.threads, disable_progressbar=snakemake.config["run"].get("disable_progressbar", False), ) + + build_transformation_output_coke(eurostat, + snakemake.output.transformation_output_coke) + swiss = build_swiss() idees = build_idees(idees_countries) diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index c120dafe..316d075f 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -227,6 +227,18 @@ def industrial_energy_demand(countries, year): return pd.concat(demand_l, keys=countries) +def add_coke_ovens(demand, fn, year): + df = pd.read_csv(fn, index_col=[0,1]).xs(year, level=1) + df = df.rename(columns={'Total all products':'Total'})[fuels.keys()] + df = df.rename(columns=fuels).T.groupby(level=0).sum().T + df["other"] = df["all"] - df.loc[:,df.columns != "all"].sum(axis=1) + df = df.T.reindex_like(demand.xs("Integrated steelworks", axis=1, level=1)).fillna(0) + sel = demand.columns.get_level_values(1)=="Integrated steelworks" + demand.loc[:,sel] += 0.75 * df.values + + return demand + + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -252,7 +264,11 @@ if __name__ == "__main__": # for format compatibility demand = demand.stack(future_stack=True).unstack(level=[0, 2]) - + + # add energy consumption of coke ovens + demand = add_coke_ovens(demand, snakemake.input.transformation_output_coke, + year) + # style and annotation demand.index.name = "TWh/a" demand.sort_index(axis=1, inplace=True) From 9b8139abbea7cdfd06625575b392ab71d382a9d1 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 9 Aug 2024 14:24:06 +0200 Subject: [PATCH 45/55] add docstring --- ...ustrial_energy_demand_per_country_today.py | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 316d075f..947aad20 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -227,14 +227,42 @@ def industrial_energy_demand(countries, year): return pd.concat(demand_l, keys=countries) -def add_coke_ovens(demand, fn, year): +def add_coke_ovens(demand, fn, year, factor=0.75): + """ + Adds the energy consumption of coke ovens to the energy demand for + integrated steelworks. + + This function reads the energy consumption data for coke ovens from a + CSV file, processes it to match the structure of the `demand` DataFrame, + and then adds a specified share of this energy consumption to the energy + demand for integrated steelworks. + + The `factor` parameter controls what proportion of the coke ovens' energy + consumption should be attributed to the iron and steel production. + The default value of 75% is based on https://doi.org/10.1016/j.erss.2022.102565 + + Parameters: + demand (pd.DataFrame): A pandas DataFrame containing energy demand data + with a multi-level column index where one of the + levels corresponds to "Integrated steelworks". + fn (str): The file path to the CSV file containing the coke ovens energy + consumption data. + year (int): The year for which the coke ovens data should be selected. + factor (float, optional): The proportion of coke ovens energy consumption to add to the + integrated steelworks demand. Defaults to 0.75. + + Returns: + pd.DataFrame: The updated `demand` DataFrame with the coke ovens energy + consumption added to the integrated steelworks energy demand. + """ + df = pd.read_csv(fn, index_col=[0,1]).xs(year, level=1) df = df.rename(columns={'Total all products':'Total'})[fuels.keys()] df = df.rename(columns=fuels).T.groupby(level=0).sum().T df["other"] = df["all"] - df.loc[:,df.columns != "all"].sum(axis=1) df = df.T.reindex_like(demand.xs("Integrated steelworks", axis=1, level=1)).fillna(0) sel = demand.columns.get_level_values(1)=="Integrated steelworks" - demand.loc[:,sel] += 0.75 * df.values + demand.loc[:,sel] += factor * df.values return demand From 156eccefb72739acc7435d8decd2c5f10af0850e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Aug 2024 12:24:36 +0000 Subject: [PATCH 46/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_energy_totals.py | 22 +++++++------ ...ustrial_energy_demand_per_country_today.py | 33 ++++++++++--------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index f7d754b8..48004ea2 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -1480,24 +1480,25 @@ def build_transformation_output_coke(eurostat, fn): This function specifically filters the Eurostat data to extract transformation output related to coke ovens. - Since the transformation output for coke ovens + Since the transformation output for coke ovens is not included in the final energy consumption of the iron and steel sector, it needs to be processed and added separately. The filtered data is saved as a CSV file. Parameters: eurostat (pd.DataFrame): A pandas DataFrame containing Eurostat data with - a multi-level index + a multi-level index fn (str): The file path where the resulting CSV file should be saved. - + Output: The resulting transformation output data for coke ovens is saved as a CSV file at the path specified in fn. """ - slicer = pd.IndexSlice[:,:,:, "Coke ovens", "Other sources", :] - df = eurostat.loc[slicer, :].droplevel(level=[2,3,4,5]) + slicer = pd.IndexSlice[:, :, :, "Coke ovens", "Other sources", :] + df = eurostat.loc[slicer, :].droplevel(level=[2, 3, 4, 5]) df.to_csv(fn) - + + # %% if __name__ == "__main__": if "snakemake" not in globals(): @@ -1523,10 +1524,11 @@ if __name__ == "__main__": nprocesses=snakemake.threads, disable_progressbar=snakemake.config["run"].get("disable_progressbar", False), ) - - build_transformation_output_coke(eurostat, - snakemake.output.transformation_output_coke) - + + build_transformation_output_coke( + eurostat, snakemake.output.transformation_output_coke + ) + swiss = build_swiss() idees = build_idees(idees_countries) diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 947aad20..ef559efa 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -231,16 +231,16 @@ def add_coke_ovens(demand, fn, year, factor=0.75): """ Adds the energy consumption of coke ovens to the energy demand for integrated steelworks. - + This function reads the energy consumption data for coke ovens from a CSV file, processes it to match the structure of the `demand` DataFrame, and then adds a specified share of this energy consumption to the energy demand for integrated steelworks. - + The `factor` parameter controls what proportion of the coke ovens' energy consumption should be attributed to the iron and steel production. The default value of 75% is based on https://doi.org/10.1016/j.erss.2022.102565 - + Parameters: demand (pd.DataFrame): A pandas DataFrame containing energy demand data with a multi-level column index where one of the @@ -248,22 +248,24 @@ def add_coke_ovens(demand, fn, year, factor=0.75): fn (str): The file path to the CSV file containing the coke ovens energy consumption data. year (int): The year for which the coke ovens data should be selected. - factor (float, optional): The proportion of coke ovens energy consumption to add to the + factor (float, optional): The proportion of coke ovens energy consumption to add to the integrated steelworks demand. Defaults to 0.75. - + Returns: pd.DataFrame: The updated `demand` DataFrame with the coke ovens energy consumption added to the integrated steelworks energy demand. """ - df = pd.read_csv(fn, index_col=[0,1]).xs(year, level=1) - df = df.rename(columns={'Total all products':'Total'})[fuels.keys()] + df = pd.read_csv(fn, index_col=[0, 1]).xs(year, level=1) + df = df.rename(columns={"Total all products": "Total"})[fuels.keys()] df = df.rename(columns=fuels).T.groupby(level=0).sum().T - df["other"] = df["all"] - df.loc[:,df.columns != "all"].sum(axis=1) - df = df.T.reindex_like(demand.xs("Integrated steelworks", axis=1, level=1)).fillna(0) - sel = demand.columns.get_level_values(1)=="Integrated steelworks" - demand.loc[:,sel] += factor * df.values - + df["other"] = df["all"] - df.loc[:, df.columns != "all"].sum(axis=1) + df = df.T.reindex_like(demand.xs("Integrated steelworks", axis=1, level=1)).fillna( + 0 + ) + sel = demand.columns.get_level_values(1) == "Integrated steelworks" + demand.loc[:, sel] += factor * df.values + return demand @@ -292,11 +294,10 @@ if __name__ == "__main__": # for format compatibility demand = demand.stack(future_stack=True).unstack(level=[0, 2]) - + # add energy consumption of coke ovens - demand = add_coke_ovens(demand, snakemake.input.transformation_output_coke, - year) - + demand = add_coke_ovens(demand, snakemake.input.transformation_output_coke, year) + # style and annotation demand.index.name = "TWh/a" demand.sort_index(axis=1, inplace=True) From 6686e6e196e08b110c377a37cb735f4f138274b3 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Mon, 12 Aug 2024 15:27:24 +0200 Subject: [PATCH 47/55] Update doc/release_notes.rst Co-authored-by: Fabian Neumann --- doc/release_notes.rst | 8 -------- 1 file changed, 8 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index c4823f5f..5d95d4da 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -11,14 +11,6 @@ Upcoming Release ================ * Update JRC-IDEES-2015 to `JRC-IDEES-2021 `__. The reference year is changed from 2015 to 2019. -* split solid biomass potentials into solid biomass and municipal solid waste. Add option to use municipal solid waste. This option is only activated in combination with the flag ``waste_to_energy`` - -* Add option to import solid biomass - -* Add option to produce electrobiofuels (flag ``electrobiofuels``) from solid biomass and hydrogen, as a combination of BtL and Fischer-Tropsch to make more use of the biogenic carbon - -* Add flag ``sector: fossil_fuels`` in config to remove the option of importing fossil fuels - * Added unsustainable biomass potentials for solid, gaseous, and liquid biomass. The potentials can be phased-out and/or substituted by the phase-in of sustainable biomass types using the config parameters ``biomass: share_unsustainable_use_retained`` and ``biomass: share_sustainable_potential_available``. From c79a9faa2c172c6f2f024b7cf351594f451c1dc7 Mon Sep 17 00:00:00 2001 From: Bobby Xiong <36541459+bobbyxng@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:56:10 +0200 Subject: [PATCH 48/55] Updated under_construction status of links that are commissioned by now (#1205) * Updated links 14802, 14801, 12931, 14803, 14848, 14804: under_construction=False * Updated links 14802, 14801, 14803, 14848, 14804: under_construction=False * Updated link 8394 and parameter_corrections: Continuation of North-Sea-Link. --- data/entsoegridkit/links.csv | 12 ++++++------ data/parameter_corrections.yaml | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/data/entsoegridkit/links.csv b/data/entsoegridkit/links.csv index 00a488dd..abcaf0cc 100644 --- a/data/entsoegridkit/links.csv +++ b/data/entsoegridkit/links.csv @@ -6,7 +6,7 @@ link_id,bus0,bus1,length,underground,under_construction,tags,geometry 5587,1377,2382,76847.0139826037,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"32533", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>"200", "symbol"=>"DC-Line", "country"=>"IT", "t9_code"=>"0", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"0", "oneCircuit"=>"0", "CreatedDate"=>"None", "DeletedDate"=>"None", "ModifiedDate"=>"None", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(8.67675371049727 40.6777653795244,9.03900099999999 40.979898,9.22164899999999 41.133159,9.19977299501706 41.2082924934473)' 5640,1422,1638,234733.218840324,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"32590", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>"Rómulo", "symbol"=>"DC-Line", "country"=>"ES", "t9_code"=>"0", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"0", "oneCircuit"=>"0", "CreatedDate"=>"None", "DeletedDate"=>"None", "ModifiedDate"=>"None", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(2.48932993486183 39.561252379133,1.13159199999999 39.610978,0 39.710356,-0.234388957535875 39.7314420592468)' 13589,2262,7428,316517.539537871,f,f,,'LINESTRING(9.17009350125146 41.2967653544603,9.38095099999999 41.331451,9.858856 41.352072,10.70755 41.479776,11.25 41.448903,12.100067 41.432431,12.380219 41.426253,12.418671 41.401536,12.704315 41.347948,12.805939 41.368564,12.9016442293009 41.3921592955445)' -14802,2258,7029,391819.608605717,f,t,,'LINESTRING(14.0986517070226 42.4133438660838,14.412689 42.431566,15.115814 42.363618,16.269379 42.067646,16.875 42.126747,16.962891 42.135913,18.531189 42.271212,18.7271798293119 42.3522936900005)' +14802,2258,7029,391819.608605717,f,f,,'LINESTRING(14.0986517070226 42.4133438660838,14.412689 42.431566,15.115814 42.363618,16.269379 42.067646,16.875 42.126747,16.962891 42.135913,18.531189 42.271212,18.7271798293119 42.3522936900005)' 14668,2333,3671,146536.932669904,f,t,,'LINESTRING(6.04271995139229 45.4637174756646,6.16607700000001 45.327048,6.351471 45.183973,6.54922499999999 45.148148,6.62338299999999 45.101638,6.642609 45.089036,6.70440700000001 45.05121,6.980438 45.089036,7.00653099999999 45.092914,7.21939099999999 45.094853,7.223511 45.089036,7.378693 44.871443,7.32136143270145 44.8385424366672)' 14808,2379,2383,103628.671904731,f,f,,'LINESTRING(9.37725891362686 42.7057449479108,9.79980499999999 42.799431,10.5931379465185 42.9693952059839)' 5575,2379,2380,24868.4258834249,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"32521", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>" ", "symbol"=>"DC-Line", "country"=>"FR", "t9_code"=>"0", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"0", "oneCircuit"=>"0", "CreatedDate"=>"None", "DeletedDate"=>"None", "ModifiedDate"=>"None", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(9.37679000208623 42.7053229039427,9.357605 42.552069,9.45054814341409 42.5389781005166)' @@ -15,7 +15,7 @@ link_id,bus0,bus1,length,underground,under_construction,tags,geometry 5583,2382,7428,11623.019620339,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"32529", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>" ", "symbol"=>"DC-Line", "country"=>"IT", "t9_code"=>"FR-IT-01", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"1", "oneCircuit"=>"0", "CreatedDate"=>"None", "DeletedDate"=>"None", "ModifiedDate"=>"1.555323123e+12", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(9.17008474107272 41.2967639130447,9.168091 41.303603,9.18319700000001 41.250968,9.1995514318356 41.2089447559651)' 14825,2476,2585,45367.7245799963,f,f,,'LINESTRING(2.98259070757654 42.2776059846425,2.90313700000001 42.397094,2.867432 42.467032,2.77404800000001 42.655172)' 8745,3611,8302,9361.61122972312,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"120591", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>"None", "symbol"=>"DC-Line", "country"=>"CH", "t9_code"=>"None", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"0", "oneCircuit"=>"1", "CreatedDate"=>"1.556535027e+12", "DeletedDate"=>"None", "ModifiedDate"=>"None", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(7.95410166666667 47.5542867377085,7.928009 47.555214,7.937622 47.526475,7.96895162362761 47.4961125343931)' -14801,4709,4781,50206.4589537583,f,t,,'LINESTRING(6.43068069229957 50.8136946409214,6.020508 50.766865,5.925751 50.755572,5.73118285928413 50.7304278585398)' +14801,4709,4781,50206.4589537583,f,f,,'LINESTRING(6.43068069229957 50.8136946409214,6.020508 50.766865,5.925751 50.755572,5.73118285928413 50.7304278585398)' 14814,4972,5062,232745.802729813,f,f,,'LINESTRING(4.04528166772434 51.9611233898246,2.41561900000001 51.702353,0.794192405058928 51.4189824547604)' 5558,4975,7427,45665.1050240866,f,t,'"MW"=>"None", "TSO"=>"None", "oid"=>"32502", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>" ", "symbol"=>"DC-Line", "country"=>"UK", "t9_code"=>" BE-UK-01", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"1", "oneCircuit"=>"0", "CreatedDate"=>"None", "DeletedDate"=>"None", "ModifiedDate"=>"1.555407949e+12", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(1.92947399999999 51.251601,1.27623412238205 51.2327009391635)' 14826,4977,4983,52725.5506558225,f,f,,'LINESTRING(1.75051314494826 50.9186901861196,1.43508900000001 50.970535,1.02353536683349 51.0370060560335)' @@ -33,16 +33,16 @@ link_id,bus0,bus1,length,underground,under_construction,tags,geometry 5571,5743,7074,89346.6337548304,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"32517", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>"HelWin1", "symbol"=>"DC-Line", "country"=>"DE", "t9_code"=>"0", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"0", "oneCircuit"=>"0", "CreatedDate"=>"None", "DeletedDate"=>"None", "ModifiedDate"=>"1.545224101e+12", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(8.12610708224912 54.310749538123,8.238373 54.256401,9.32699442549698 53.9319562532009)' 5567,5744,5787,139209.866527364,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"32512", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>"DolWin1", "symbol"=>"DC-Line", "country"=>"DE", "t9_code"=>"0", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"0", "oneCircuit"=>"0", "CreatedDate"=>"None", "DeletedDate"=>"None", "ModifiedDate"=>"1.545224147e+12", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(6.84493115764205 53.880869,6.909027 53.880869,7.116394 53.835512,7.36358600000001 53.396432,7.32101399999999 53.112163,7.33612100000001 52.893992,7.16075117704058 52.8485079587114)' 5570,5745,8272,99066.5793764307,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"32515", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>"DolWin3", "symbol"=>"DC-Line", "country"=>"DE", "t9_code"=>"0", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"0", "oneCircuit"=>"0", "CreatedDate"=>"None", "DeletedDate"=>"None", "ModifiedDate"=>"1.545224133e+12", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(6.84423599483409 53.8134043878533,6.71127300000001 53.693454,6.65634200000001 53.59821,6.73461900000001 53.55581,7.112274 53.45126,7.05596900000001 53.340713,7.237244 53.26932,7.223511 53.18135,7.223511 53.1805270078955)' -14803,5751,5803,280301.445474794,f,t,,'LINESTRING(6.75668661933496 53.437616158174,6.838989 53.664171,6.96258499999999 53.785238,7.34298700000001 53.882488,7.80029300000001 54.517096,8.20678699999999 55.297102,8.86005375885099 55.4336013425692)' +14803,5751,5803,280301.445474794,f,f,,'LINESTRING(6.75668661933496 53.437616158174,6.838989 53.664171,6.96258499999999 53.785238,7.34298700000001 53.882488,7.80029300000001 54.517096,8.20678699999999 55.297102,8.86005375885099 55.4336013425692)' 14821,5749,6363,575352.425009444,f,f,,'LINESTRING(6.83036734046461 53.4374933986115,6.253967 53.645452,6.33636499999999 55.776573,6.34597800000001 56.029855,6.34597800000001 56.030622,6.43661500000001 58.130121,6.90176957000565 58.2653404287817)' 5568,5768,5787,131420.09609615,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"32513", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>"DolWin2", "symbol"=>"DC-Line", "country"=>"DE", "t9_code"=>"0", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"0", "oneCircuit"=>"0", "CreatedDate"=>"None", "DeletedDate"=>"None", "ModifiedDate"=>"1.545224159e+12", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(7.11083415172816 53.9630966319811,7.07107499999999 53.80795,7.301788 53.39807,7.267456 53.110514,7.29354899999999 52.907246,7.16070024970726 52.8485606886388)' 12932,5770,5773,6905.52230262641,f,t,,'LINESTRING(7.15460523215685 53.4027398808691,7.24823000000001 53.375956)' -14848,5858,6358,574884.998052791,f,t,,'LINESTRING(6.81690675921544 58.6338502746805,6.63024900000001 58.249559,6.78268399999999 57.579197,7.17544599999999 56.532986,7.17407200000001 56.5345,7.46521000000001 55.776573,7.46521000000001 55.776573,7.64099100000001 55.312736,8.458099 54.316523,9.394684 53.934262)' +14848,5858,6358,574884.998052791,f,f,,'LINESTRING(6.81690675921544 58.6338502746805,6.63024900000001 58.249559,6.78268399999999 57.579197,7.17544599999999 56.532986,7.17407200000001 56.5345,7.46521000000001 55.776573,7.46521000000001 55.776573,7.64099100000001 55.312736,8.458099 54.316523,9.394684 53.934262)' 5581,5893,6072,59184.4227659405,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"32527", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>" ", "symbol"=>"DC-Line", "country"=>"UK", "t9_code"=>"222.1.2", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"1", "oneCircuit"=>"0", "CreatedDate"=>"None", "DeletedDate"=>"None", "ModifiedDate"=>"None", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(-4.94702447012386 55.0727948492206,-5.137482 55.042188,-5.62500000000001 54.890036,-5.631866 54.887667,-5.7332134509551 54.813550429852)' 5580,5893,6072,58741.4601812995,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"32526", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>" ", "symbol"=>"DC-Line", "country"=>"UK", "t9_code"=>"222.1.1", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"1", "oneCircuit"=>"0", "CreatedDate"=>"None", "DeletedDate"=>"None", "ModifiedDate"=>"None", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(-4.94689333475508 55.0726735779237,-5.045471 55.009914,-5.59616099999999 54.840245,-5.62500000000001 54.834709,-5.73306677066227 54.8134313531551)' 8009,5897,5936,363085.503577327,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"70191", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>"Western HVDC link", "symbol"=>"DC-Line", "country"=>"UK", "t9_code"=>"None", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"0", "oneCircuit"=>"0", "CreatedDate"=>"1.514994622e+12", "DeletedDate"=>"None", "ModifiedDate"=>"1.51499467e+12", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(-3.18595885129092 53.213699479605,-3.158569 53.308724,-3.40988200000001 53.511735,-4.081421 53.803084,-5.158081 54.013418,-5.28442399999999 54.866334,-5.177307 55.345546,-4.88616899999999 55.586883,-4.8806877889882 55.7044245716822)' 14815,5937,6086,242400.41935291,f,f,,'LINESTRING(-3.12293971810515 53.2087645354697,-3.13934300000001 53.266034,-3.368683 53.377594,-5.18280000000001 53.495399,-5.62500000000001 53.519084,-5.62500000000001 53.519084,-6.101532 53.503568,-6.61057668606004 53.483977180569)' -14804,5949,6684,695432.776022422,f,t,,'LINESTRING(6.64773945778347 59.5995729910866,6.483307 59.539192,6.374817 59.538495,6.24847399999999 59.510636,6.196289 59.448566,5.898285 59.321981,5.64697299999999 59.234284,5.62500000000001 59.223042,4.81338500000001 58.813742,2.03384400000001 57.374679,0 56.170023,-0.650940000000012 55.776573,-1.55838055228731 55.2221613174321)' +14804,5949,6684,695432.776022422,f,f,,'LINESTRING(6.64773945778347 59.5995729910866,6.483307 59.539192,6.374817 59.538495,6.24847399999999 59.510636,6.196289 59.448566,5.898285 59.321981,5.64697299999999 59.234284,5.62500000000001 59.223042,4.81338500000001 58.813742,2.03384400000001 57.374679,0 56.170023,-0.650940000000012 55.776573,-1.55838055228731 55.2221613174321)' 5635,6300,6348,93313.2906756649,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"32585", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>"150", "symbol"=>"DC-Line", "country"=>"SE", "t9_code"=>"0", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"0", "oneCircuit"=>"0", "CreatedDate"=>"None", "DeletedDate"=>"None", "ModifiedDate"=>"None", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(18.2272491895352 57.5711315582343,17.274628 57.645401,16.875 57.674052,16.6818074486274 57.692364166947)' 14819,6311,6416,122337.134741418,f,f,,'LINESTRING(10.2163282994747 57.1311139024238,10.567474 57.20771,10.737762 57.192832,10.972595 57.230016,11.25 57.33171,11.532898 57.436081,11.867981 57.556366,12.0227165657676 57.561507168045)' 14809,6311,6416,122935.90852816,f,f,,'LINESTRING(10.2163571716117 57.1310010356663,10.366974 57.123569,10.578461 57.16678,10.740509 57.15263,11.001434 57.197296,11.174469 57.255281,11.25 57.282754,11.56723 57.399104,12.0227887239052 57.5613889668514)' @@ -58,6 +58,6 @@ link_id,bus0,bus1,length,underground,under_construction,tags,geometry 14818,6586,6618,257364.279393886,f,f,,'LINESTRING(21.3559064590049 61.0800030227353,21.303864 61.005076,20.946808 60.801394,18.153534 60.501202,18.007965 60.483615,17.171631 60.503906,17.0593630437863 60.5503864910584)' 14817,6589,6618,197128.229552834,f,f,,'LINESTRING(21.3557421230034 61.0800501553429,20.902863 60.846249,18.224945 60.556604,18.0193872312079 60.533018071939)' 14812,6620,6623,140169.735736189,f,f,,'LINESTRING(22.3045576957813 60.4368452717433,21.404114 60.329667,19.8472351583549 60.129935739173)' -8394,6684,6696,21158.5735245602,f,t,'"MW"=>"None", "TSO"=>"None", "oid"=>"89791", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>"None", "symbol"=>"DC-Line", "country"=>"NO", "t9_code"=>"None", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"0", "oneCircuit"=>"0", "CreatedDate"=>"1.518010133e+12", "DeletedDate"=>"None", "ModifiedDate"=>"None", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(6.64851407057135 59.5996162767494,6.99238592942864 59.5246589234811)' +8394,6684,6696,21158.5735245602,f,f,'"MW"=>"None", "TSO"=>"None", "oid"=>"89791", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>"None", "symbol"=>"DC-Line", "country"=>"NO", "t9_code"=>"None", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"0", "oneCircuit"=>"0", "CreatedDate"=>"1.518010133e+12", "DeletedDate"=>"None", "ModifiedDate"=>"None", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(6.64851407057135 59.5996162767494,6.99238592942864 59.5246589234811)' 5569,5787,8272,38561.1931761179,f,t,'"MW"=>"None", "TSO"=>"None", "oid"=>"32514", "ext1"=>"None", "EIC_2"=>"None", "EIC_3"=>"None", "EIC_4"=>"None", "text_"=>"DolWin 3", "symbol"=>"DC-Line", "country"=>"DE", "t9_code"=>"0", "visible"=>"1", "EIC_code"=>"None", "tie_line"=>"0", "oneCircuit"=>"0", "CreatedDate"=>"None", "DeletedDate"=>"None", "ModifiedDate"=>"1.489072219e+12", "Internalcomments"=>"None", "visible_on_printed"=>"1"','LINESTRING(7.223511 53.1805270078955,7.223511 53.179704,7.21527100000001 53.121229,7.24273699999999 52.932086,7.16056753068224 52.8486333236236)' 14813,7053,7430,192856.020480538,f,f,,'LINESTRING(10.8823542109264 53.948125809387,11.25 54.061,11.657867 54.186548,12.208557 54.386955,12.236023 54.402946,12.43515 54.541003,12.602692 54.684153,12.745514 54.844199,12.744141 54.842618,12.87735 54.979978,12.947388 55.077581,12.9299984288384 55.0630403498842)' diff --git a/data/parameter_corrections.yaml b/data/parameter_corrections.yaml index df15738a..3d19bed8 100644 --- a/data/parameter_corrections.yaml +++ b/data/parameter_corrections.yaml @@ -15,6 +15,7 @@ Link: "115000": 1200 # Caithness Moray HVDC index: "14804": 1400 # North-Sea link (NSN Link) + "8394": 1400 # North-Sea Link (NSN Link) continuation "14822": 700 # NO-DK Skagerrak 4 "14827": 440 # NO-DK Skagerrak 3 "14810": 500 # NO-DK Skagerrak 1-2 From d58a848f97f52916730f794c35131d67c6671012 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 13 Aug 2024 16:46:34 +0200 Subject: [PATCH 49/55] [pre-commit.ci] pre-commit autoupdate (#1208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black-pre-commit-mirror: 24.4.2 → 24.8.0](https://github.com/psf/black-pre-commit-mirror/compare/24.4.2...24.8.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 72eabf9e..8b9bf2e0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,7 +51,7 @@ repos: # Formatting with "black" coding style - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.4.2 + rev: 24.8.0 hooks: # Format Python files - id: black From 6ca8e80095c9304cb5c88804956ac486f584d83d Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 13 Aug 2024 18:13:32 +0200 Subject: [PATCH 50/55] add option to use atlite to smooth wind turbine power curves (#1209) --- config/config.default.yaml | 5 +++++ doc/configtables/offwind-ac.csv | 1 + doc/configtables/offwind-dc.csv | 1 + doc/configtables/onwind.csv | 1 + doc/release_notes.rst | 3 +++ 5 files changed, 11 insertions(+) diff --git a/config/config.default.yaml b/config/config.default.yaml index 306624d0..52512136 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -155,6 +155,7 @@ renewable: resource: method: wind turbine: Vestas_V112_3MW + smooth: true add_cutout_windspeed: true capacity_per_sqkm: 3 # correction_factor: 0.93 @@ -174,6 +175,7 @@ renewable: resource: method: wind turbine: NREL_ReferenceTurbine_2020ATB_5.5MW + smooth: true add_cutout_windspeed: true capacity_per_sqkm: 2 correction_factor: 0.8855 @@ -190,6 +192,7 @@ renewable: resource: method: wind turbine: NREL_ReferenceTurbine_2020ATB_5.5MW + smooth: true add_cutout_windspeed: true capacity_per_sqkm: 2 correction_factor: 0.8855 @@ -206,6 +209,8 @@ renewable: resource: method: wind turbine: NREL_ReferenceTurbine_5MW_offshore + smooth: true + add_cutout_windspeed: true # ScholzPhd Tab 4.3.1: 10MW/km^2 capacity_per_sqkm: 2 correction_factor: 0.8855 diff --git a/doc/configtables/offwind-ac.csv b/doc/configtables/offwind-ac.csv index 9ba2fa7e..c9c7c843 100644 --- a/doc/configtables/offwind-ac.csv +++ b/doc/configtables/offwind-ac.csv @@ -3,6 +3,7 @@ cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` ( resource,,, -- method,--,"Must be 'wind'","A superordinate technology type." -- turbine,--,"One of turbine types included in `atlite `_. Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve." +-- smooth,--,"{True, False}","Switch to apply a gaussian kernel density smoothing to the power curve." capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of wind turbine placement." correction_factor,--,float,"Correction factor for capacity factor time series." excluder_resolution,m,float,"Resolution on which to perform geographical elibility analysis." diff --git a/doc/configtables/offwind-dc.csv b/doc/configtables/offwind-dc.csv index e55d8944..86bf0cdc 100644 --- a/doc/configtables/offwind-dc.csv +++ b/doc/configtables/offwind-dc.csv @@ -3,6 +3,7 @@ cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` ( resource,,, -- method,--,"Must be 'wind'","A superordinate technology type." -- turbine,--,"One of turbine types included in `atlite `_. Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve." +-- smooth,--,"{True, False}","Switch to apply a gaussian kernel density smoothing to the power curve." capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of wind turbine placement." correction_factor,--,float,"Correction factor for capacity factor time series." excluder_resolution,m,float,"Resolution on which to perform geographical elibility analysis." diff --git a/doc/configtables/onwind.csv b/doc/configtables/onwind.csv index a801d83c..676494f4 100644 --- a/doc/configtables/onwind.csv +++ b/doc/configtables/onwind.csv @@ -3,6 +3,7 @@ cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` ( resource,,, -- method,--,"Must be 'wind'","A superordinate technology type." -- turbine,--,"One of turbine types included in `atlite `_. Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve." +-- smooth,--,"{True, False}","Switch to apply a gaussian kernel density smoothing to the power curve." capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of wind turbine placement." corine,,, -- grid_codes,--,"Any subset of the `CORINE Land Cover code list `_","Specifies areas according to CORINE Land Cover codes which are generally eligible for wind turbine placement." diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 5d95d4da..274231c6 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -9,6 +9,9 @@ Release Notes Upcoming Release ================ + +* Add option to apply a gaussian kernel density smoothing to wind turbine power curves. + * Update JRC-IDEES-2015 to `JRC-IDEES-2021 `__. The reference year is changed from 2015 to 2019. * Added unsustainable biomass potentials for solid, gaseous, and liquid biomass. The potentials can be phased-out and/or From f263862455248291366dd615fa41dd90646e3246 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 13 Aug 2024 21:03:07 +0200 Subject: [PATCH 51/55] use lower resolution EEZ shapes to reduce excessive RAM use (#1210) --- rules/build_electricity.smk | 2 +- rules/retrieve.smk | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 34472f27..97e1e16a 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -92,7 +92,7 @@ rule build_shapes: countries=config_provider("countries"), input: naturalearth=ancient("data/naturalearth/ne_10m_admin_0_countries_deu.shp"), - eez=ancient("data/eez/World_EEZ_v12_20231025_gpkg/eez_v12.gpkg"), + eez=ancient("data/eez/World_EEZ_v12_20231025_LR/eez_v12_lowres.gpkg"), nuts3=ancient("data/bundle/NUTS_2013_60M_SH/data/NUTS_RG_60M_2013.shp"), nuts3pop=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"), nuts3gdp=ancient("data/bundle/nama_10r_3gdp.tsv.gz"), diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 3a46a076..b2382f51 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -226,9 +226,9 @@ if config["enable"]["retrieve"]: rule retrieve_eez: params: - zip="data/eez/World_EEZ_v12_20231025_gpkg.zip", + zip="data/eez/World_EEZ_v12_20231025_LR.zip", output: - gpkg="data/eez/World_EEZ_v12_20231025_gpkg/eez_v12.gpkg", + gpkg="data/eez/World_EEZ_v12_20231025_LR/eez_v12_lowres.gpkg", run: import os import requests @@ -239,7 +239,7 @@ if config["enable"]["retrieve"]: response = requests.post( "https://www.marineregions.org/download_file.php", - params={"name": "World_EEZ_v12_20231025_gpkg.zip"}, + params={"name": "World_EEZ_v12_20231025_LR.zip"}, data={ "name": name, "organisation": org, From 3ef6d519d715efb2aec863f9c09ee2ec605c711b Mon Sep 17 00:00:00 2001 From: Bobby Xiong <36541459+bobbyxng@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:24:38 +0200 Subject: [PATCH 52/55] Fix simplify_network.py to handle more complex topologies (#1211) * Fixed simplify_network.py to handle more complex topologies, i.e. if two links are connected to nearby AC buses separated by an AC line. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- scripts/simplify_network.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 558e4cf2..2b759a73 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -298,13 +298,24 @@ def simplify_links( _, labels = connected_components(adjacency_matrix, directed=False) labels = pd.Series(labels, n.buses.index) - G = n.graph() + # Only span graph over the DC link components + G = n.graph(branch_components=["Link"]) def split_links(nodes): nodes = frozenset(nodes) seen = set() - supernodes = {m for m in nodes if len(G.adj[m]) > 2 or (set(G.adj[m]) - nodes)} + + # Supernodes are endpoints of links, identified by having lass then two neighbours or being an AC Bus + # An example for the latter is if two different links are connected to the same AC bus. + supernodes = { + m + for m in nodes + if ( + (len(G.adj[m]) < 2 or (set(G.adj[m]) - nodes)) + or (n.buses.loc[m, "carrier"] == "AC") + ) + } for u in supernodes: for m, ls in G.adj[u].items(): From 1cf85b0be889683935ec3898b5eac494aff5dd1b Mon Sep 17 00:00:00 2001 From: Bobby Xiong <36541459+bobbyxng@users.noreply.github.com> Date: Wed, 14 Aug 2024 14:22:03 +0200 Subject: [PATCH 53/55] Fix for Corsica in simplify_network: Include local substation (#1215) * Fixed simplify_network.py to handle more complex topologies, i.e. if two links are connected to nearby AC buses separated by an AC line. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Added fix for Corsica node: If region containing Corsica is modelled, include substation on Corsica as supernode (end point). * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- scripts/simplify_network.py | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 2b759a73..c54e3418 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -306,6 +306,14 @@ def simplify_links( seen = set() + # Corsica substation + node_corsica = find_closest_bus( + n, + x=9.44802, + y=42.52842, + tol=2000, # Tolerance needed to only return the bus if the region is actually modelled + ) + # Supernodes are endpoints of links, identified by having lass then two neighbours or being an AC Bus # An example for the latter is if two different links are connected to the same AC bus. supernodes = { @@ -314,6 +322,7 @@ def simplify_links( if ( (len(G.adj[m]) < 2 or (set(G.adj[m]) - nodes)) or (n.buses.loc[m, "carrier"] == "AC") + or (m == node_corsica) ) } @@ -530,6 +539,42 @@ def cluster( return clustering.network, clustering.busmap +def find_closest_bus(n, x, y, tol=2000): + """ + Find the index of the closest bus to the given coordinates within a specified tolerance. + Parameters: + n (pypsa.Network): The network object. + x (float): The x-coordinate (longitude) of the target location. + y (float): The y-coordinate (latitude) of the target location. + tol (float): The distance tolerance in meters. Default is 2000 meters. + + Returns: + int: The index of the closest bus to the target location within the tolerance. + Returns None if no bus is within the tolerance. + """ + # Conversion factors + meters_per_degree_lat = 111139 # Meters per degree of latitude + meters_per_degree_lon = 111139 * np.cos( + np.radians(y) + ) # Meters per degree of longitude at the given latitude + + x0 = np.array(n.buses.x) + y0 = np.array(n.buses.y) + + # Calculate distances in meters + dist = np.sqrt( + ((x - x0) * meters_per_degree_lon) ** 2 + + ((y - y0) * meters_per_degree_lat) ** 2 + ) + + # Find the closest bus within the tolerance + min_dist = dist.min() + if min_dist <= tol: + return n.buses.index[dist.argmin()] + else: + return None + + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake From f72afe7d102a2c605a9d8b7e00ebc3cd41ce284c Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 14 Aug 2024 16:27:06 +0200 Subject: [PATCH 54/55] bugfix: make sure to include Ukraine offshore shapes with new EEZ files --- scripts/build_shapes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 411d56a4..4de5370a 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -132,7 +132,8 @@ def countries(naturalearth, country_list): def eez(eez, country_list): df = gpd.read_file(eez) iso3_list = cc.convert(country_list, src="ISO2", to="ISO3") - df = df.query("ISO_TER1 in @iso3_list and POL_TYPE == '200NM'").copy() + pol_type = ["200NM", "Overlapping claim"] + df = df.query("ISO_TER1 in @iso3_list and POL_TYPE in @pol_type").copy() df["name"] = cc.convert(df.ISO_TER1, src="ISO3", to="ISO2") s = df.set_index("name").geometry.map( lambda s: _simplify_polys(s, filterremote=False) From a9f67b313f63df1b2d550fc04bac634416116752 Mon Sep 17 00:00:00 2001 From: Philipp Glaum <95913147+p-glaum@users.noreply.github.com> Date: Thu, 15 Aug 2024 11:42:21 +0200 Subject: [PATCH 55/55] handle new and upgraded TYNDP&NEP lines/links in base network (OSM compatible) (#1085) * add general implementation to add tyndp and nep. TODO: Remove duplicate? * pre_osm_merge * clean and update base_network.py update default config settings * clean and update base_network.py update default config settings * base_network.py:remove adding of transmission projects add_transmission_project.py: add new script for creating lines and link csv from transmission projects add_electricity.py: add new projects from created csv files * cluster_network.py: do not allow to group lines with different build years together-> requires pypsa update simplify_network.py: fix bug of simplify links * remove legacies of removing transmission projects from base_network * restructure folders:new folder for tranmission project csv add_transmission_projects: improve logging, cleanup * update lines csvs and add default line type for upgraded and new lines * remove duplicate lines which were already in gridkit * allow to connect ac lines only to ac buses * add manual links csv (adjusted links_tyndp.csv) and update default config * add realease note * remove links_tyndp.csv file, references in build_elec.smk and function in base_network.py to add them * add configuration options for transmission projects to documentation and add template folder for transmission projects * update pypsa version in environments * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * integrate Fabian's review comments 2 * add electricity:add import pathlib and duplicate printing out of adding line projects * update NEP line csv * address Fabian's comments * build_transmission_projects: address Fabian's final comments simplify_network: use modus to get line type which appears most often * build_transmission_project: change from .geometry to ["geometry"] * build_transmission_projects: remove redundanty line * build_transmission_projects: remove buffer for europe shape because of higher resolution default config: fix wrong key for skip argument in transmission_projects * update configtables and default config * update manual links csv and delete undetected duplicate links in tyndp2020 * final adjustments --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Fabian Neumann --- config/config.default.yaml | 23 +- data/links_tyndp.csv | 41 -- .../manual/new_links.csv | 21 + data/transmission_projects/nep/new_lines.csv | 16 + data/transmission_projects/nep/new_links.csv | 12 + .../template/new_lines.csv | 5 + .../template/new_links.csv | 5 + .../template/upgraded_lines.csv | 5 + .../template/upgraded_links.csv | 5 + .../tyndp2020/new_lines.csv | 119 ++++ .../tyndp2020/new_links.csv | 49 ++ .../tyndp2020/upgraded_lines.csv | 27 + .../tyndp2020/upgraded_links.csv | 8 + doc/configtables/links.csv | 1 - doc/configtables/transmission_projects.csv | 9 + doc/configuration.rst | 17 + doc/release_notes.rst | 1 + envs/environment.fixed.yaml | 2 +- envs/environment.yaml | 2 +- rules/build_electricity.smk | 48 +- scripts/add_electricity.py | 23 + scripts/base_network.py | 112 ---- scripts/build_transmission_projects.py | 564 ++++++++++++++++++ scripts/cluster_network.py | 1 + scripts/simplify_network.py | 5 +- 25 files changed, 958 insertions(+), 163 deletions(-) delete mode 100644 data/links_tyndp.csv create mode 100644 data/transmission_projects/manual/new_links.csv create mode 100644 data/transmission_projects/nep/new_lines.csv create mode 100644 data/transmission_projects/nep/new_links.csv create mode 100644 data/transmission_projects/template/new_lines.csv create mode 100644 data/transmission_projects/template/new_links.csv create mode 100644 data/transmission_projects/template/upgraded_lines.csv create mode 100644 data/transmission_projects/template/upgraded_links.csv create mode 100644 data/transmission_projects/tyndp2020/new_lines.csv create mode 100644 data/transmission_projects/tyndp2020/new_links.csv create mode 100644 data/transmission_projects/tyndp2020/upgraded_lines.csv create mode 100644 data/transmission_projects/tyndp2020/upgraded_links.csv create mode 100644 doc/configtables/transmission_projects.csv create mode 100644 scripts/build_transmission_projects.py diff --git a/config/config.default.yaml b/config/config.default.yaml index 52512136..14221dac 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -286,7 +286,7 @@ lines: max_extension: 20000 #MW length_factor: 1.25 reconnect_crimea: true - under_construction: 'keep' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity + under_construction: 'keep' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity for lines in grid extract dynamic_line_rating: activate: false cutout: europe-2013-sarah3-era5 @@ -299,8 +299,25 @@ links: p_max_pu: 1.0 p_nom_max: .inf max_extension: 30000 #MW - include_tyndp: true - under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity + under_construction: 'keep' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity for lines in grid extract + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#transmission_projects +transmission_projects: + enable: true + include: + tyndp2020: true + nep: true + manual: true + skip: + - upgraded_lines + - upgraded_links + status: + - under_construction + - in_permitting + - confirmed + #- planned_not_yet_permitted + #- under_consideration + new_link_capacity: zero #keep or zero # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#transformers transformers: diff --git a/data/links_tyndp.csv b/data/links_tyndp.csv deleted file mode 100644 index 43030be5..00000000 --- a/data/links_tyndp.csv +++ /dev/null @@ -1,41 +0,0 @@ -Name,Converterstation 1,Converterstation 2,Length (given) (km),Length (distance*1.2) (km),Power (MW),status,replaces,Ref,x1,y1,x2,y2 -Biscay Gulf,Gatica (ES),Cubnezais (FR),370,,2200,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/16,-2.867,43.367,-0.408943,45.074191 -Italy-France,Piossasco (IT),Grand Ile (FR),190,,1000,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/21,7.468,44.9898,6.045,45.472 -IFA2,Tourbe (FR),Chilling (GB),,247.2,1000,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/25,-0.172042,49.083593,-1.277269,50.839338 -Italy-Montenegro,Villanova (IT),Latsva (MT),445,,1200,under construction,Link.14539,https://tyndp.entsoe.eu/tyndp2018/projects/projects/28,14.125,42.3947222222222,18.7947222222222,42.3175 -NordLink,Tonstad (NO),Wilster (DE),514,,1400,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/37,6.716948,58.662631,9.373979,53.922479 -COBRA cable,Endrup (DK),Eemshaven (NL),325,,700,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/71,8.718392,55.523115,6.835494,53.438589 -Thames Estuary Cluster (NEMO-Link),Richborough (GB),Gezelle (BE),140,,1000,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/74,1.324854,51.295891,3.23043,51.24902 -Anglo-Scottish -1,Hunterston (UK),Deeside (UK),422,,2400,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/77,-4.898329,55.723331,-3.032972,53.199735 -ALEGrO,Lixhe (BE),Oberzier (DE),100,,1000,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/92,5.67933,50.7567965,6.474704,50.867532 -North Sea Link,Kvilldal (NO),Blythe (GB),720,,1400,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/110,6.637527,59.515096,-1.510277,55.126957 -HVDC SuedOstLink,Wolmirstedt (DE),Isar (DE),,557,2000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/130,11.629014,52.252137,12.091596,48.080837 -HVDC Line A-North,Emden East (DE),Osterath (DE),,284,2000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/132,7.206009,53.359403,6.619451,51.272935 -France-Alderney-Britain,Exeter (UK),Menuel (FR),220,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/153,-3.533899,50.718412,-1.469216,49.509594 -Viking DKW-GB,Bicker Fen (GB),Revsing (DK),,807,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/167,-0.203587,52.93979,9.178363,55.509166 -ElecLink,Sellindge (UK),Mandarins (FR),,72,1000,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/172,0.975555555555556,51.1058333333333,1.78472222222222,50.9030555555556 -Greenconnector,Verderio (IT),Sils i.D. (CH),150,,1000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/174,9.439781,45.668539,9.76569,46.432156 -Hansa PowerBridge I,Hurva (SE),Guestrow (DE),,283,700,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/176,13.6022222222222,55.8330555555556,12.189538,53.803155 -NorthConnect,Simadalen (NO),Peterhead (UK),650,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/190,7.16027,60.500527,-1.784066,57.508123 -HVDC SuedLink,Wilster (DE),Großgartach (DE),,637,4000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/235,9.373979,53.922479,9.117193,49.145157 -AQUIND Interconnector,Lovedean (GB),Barnabos (FR),254,,2000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/247,-1.020512,50.908244,0.991736,49.656631 -HVDC Ultranet,Osterath (DE),Philippsburg (DE),,314,600,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/254,6.619451,51.272935,8.458036,49.235253 -Gridlink,Kingsnorth (UK),Warande (FR),160,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/285,0.596111111111111,51.41972,2.376776,51.034368 -NeuConnect,Grain (UK),Fedderwarden (DE),680,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/309,0.716666666666667,51.44,8.046524,53.562763 -NordBalt,Klaipeda (LT),Nybro (SE),450,,700,built,,https://en.wikipedia.org/wiki/NordBalt,21.256667,55.681667,15.854167,56.767778 -Estlink 1,Harku (EE),Espoo (FI),105,,350,built,,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889 -Greenlink,Waterford (IE),Pembroke (UK),,180,500,under construction,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/286,-6.987,52.260,-4.986,51.686 -Celtic Interconnector,Aghada (IE),La Martyre (FR),,572,700,under consideration,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/107,-8.16642,51.91413,-4.184,48.459 -GiLA,Bordeaux (FR),Nantes (FR),,312,640,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-1.209,46.901,-0.576,44.960 -HG North Tyrrhenian Corridor,Milan (IT),Viterbo (IT),,500,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,9.409,45.553,12.015,42.244 -HG Adriatic Corridor,Ferrara (IT),Foggia (IT),,582,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,11.661,44.855,15.550,41.513 -SAPEI 2,Fioumesanto (IT),Montalto (IT),,390,1000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,8.283,40.790,11.602,42.331 -HG Ionian-Tyrrhenian Corridor,Rossano (IT),Latina (IT),,496,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,16.629,39.568,12.779,41.430 -HG Ionian-Tyrrhenian Corridor 2,Rossano (IT),Catania (IT),,330,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,16.629,39.568,15.049,37.408 -Germany-UK Hybrid Interconnector,Fetteresso (UK),Emden (DE),800,,2000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-2.383,56.991,7.207,53.376 -NU-Link Interconnector,Hornsea (UK),Moerdijk (NL),,460,1200,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-0.261,53.655,4.586,51.661 -APOLLO-LINK,La Farga (ES),La Spezia (IT),,725,2091,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,2.883,42.062,9.884,44.107 -Baltic WindConnector (BWC),Lubmin (DE),Lihula (EE),,960,2000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,13.686,54.139,23.818,58.675 -High-Voltage Direct Current Interconnector Project Romania-Hungary,Constanta (RO),Albertirsa (HU),,930,2500,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,28.588,44.201,19.584,47.224 -Rhine-Main-Link,Ovelgönne (DE),Marxheim (DE),,433,4000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,8.379,53.315,8.435,50.078 -Green Aegean Interconnector,Arachthos (GR),Ottenhofen (DE),,600,3000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,20.967,39.185,11.868,48.207 diff --git a/data/transmission_projects/manual/new_links.csv b/data/transmission_projects/manual/new_links.csv new file mode 100644 index 00000000..901f9593 --- /dev/null +++ b/data/transmission_projects/manual/new_links.csv @@ -0,0 +1,21 @@ +,bus0,bus1,length,p_nom,project_status,tags,x0,y0,x1,y1 +TYNDP2018_154,Exeter (UK),Menuel (FR),220,1400,in_permitting,"{name:France-Alderney-Britain, url:https://tyndp.entsoe.eu/tyndp2018/projects/projects/153}",-3.533899,50.718412,-1.469216,49.509594 +TYNDP2018_25,Tourbe (FR),Chilling (GB),247.2,1000,under_construction,"{name:IFA2, url:https://tyndp.entsoe.eu/tyndp2018/projects/projects/25]",-0.172042,49.083593,-1.277269,50.839338 +TYNDP2018_285,Kingsnorth (UK),Warande (FR),160,1400,in_permitting,"{name:Gridlink, url:https://tyndp.entsoe.eu/tyndp2018/projects/projects/285}",0.596111111111111,51.41972,2.376776,51.034368 +TYNDP2018_190,Simadalen (NO),Peterhead (UK),650,1400,in_permitting,"{name:NorthConnect, url:https://tyndp.entsoe.eu/tyndp2018/projects/projects/190}",7.16027,60.500527,-1.784066,57.508123 +TYNDP2018_16,Gatica (ES),Cubnezais (FR),370,2200,in_permitting,"{name:Biscay Gulf, url:https://tyndp.entsoe.eu/tyndp2018/projects/projects/16}",-2.867,43.367,-0.408943,45.074191 +TYNDP2018_21,Piossasco (IT),Grand Ile (FR),190,1000,under_construction,"{name:Italy-France, url:https://tyndp.entsoe.eu/tyndp2018/projects/projects/21}",7.468,44.9898,6.045,45.472 +TYNDP2022_286,Waterford (IE),Pembroke (UK),180,500,under_construction,"{name:Greenlink,url:{name:TYNDP2022_286,url:https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/286}}",-6.987,52.26,-4.986,51.686 +TYNDP2024_1134,Bordeaux (FR),Nantes (FR),312,640,under_consideration,"{name:GiLA,url:{name:TYNDP2024_1134,url:https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx}}",-1.209,46.901,-0.576,44.96 +TYNDP2024_1157,Milan (IT),Viterbo (IT),500,2000,in_permitting,"{name:HG North Tyrrhenian Corridor,url:{name:TYNDP2024_1157,url:https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx}}",9.409,45.553,12.015,42.244 +TYNDP2024_1166,Ferrara (IT),Foggia (IT),582,2000,in_permitting,"{name:HG Adriatic Corridor,url:{name:TYNDP2024_1166,url:https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx}}",11.661,44.855,15.55,41.513 +TYNDP2024_1169,Fioumesanto (IT),Montalto (IT),390,1000,in_permitting,"{name:SAPEI 2,url:{name:TYNDP2024_1169,url:https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx}}",8.283,40.79,11.602,42.331 +TYNDP2024_1168,Rossano (IT),Latina (IT),496,2000,in_permitting,"{name:HG Ionian-Tyrrhenian Corridor,url:{name:TYNDP2024_1168,url:https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx}}",16.629,39.568,12.779,41.43 +TYNDP2024_1168_1,Rossano (IT),Catania (IT),330,2000,in_permitting,"{name:HG Ionian-Tyrrhenian Corridor 2,url:{name:TYNDP2024_1168_1,url:https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx}}",16.629,39.568,15.049,37.408 +TYNDP2024_1192,Fetteresso (UK),Emden (DE),800,2000,under_consideration,"{name:Germany-UK Hybrid Interconnector,url:{name:TYNDP2024_1192,url:https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx}}",-2.383,56.991,7.207,53.376 +TYNDP2024_1197,Hornsea (UK),Moerdijk (NL),460,1200,under_consideration,"{name:NU-Link Interconnector,url:{name:TYNDP2024_1197,url:https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx}}",-0.261,53.655,4.586,51.661 +TYNDP2024_1210,La Farga (ES),La Spezia (IT),725,2091,under_consideration,"{name:APOLLO-LINK,url:{name:TYNDP2024_1210,url:https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx}}",2.883,42.062,9.884,44.107 +TYNDP2024_1211,Lubmin (DE),Sindi (EE),960,2000,under_consideration,"{name:Baltic WindConnector (BWC),url:{name:TYNDP2024_1211,url:https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx}}",13.686,54.139,24.671903,58.429161 +TYNDP2024_1216,Constanta (RO),Albertirsa (HU),930,2500,under_consideration,"{name:High-Voltage Direct Current Interconnector Project Romania-Hungary,url:{name:TYNDP2024_1216,url:https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx}}",28.588,44.201,19.584,47.224 +TYNDP2024_1229,Ovelgönne (DE),Marxheim (DE),433,4000,in_permitting,"{name:Rhine-Main-Link,url:{name:TYNDP2024_1229,url:https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx}}",8.379,53.315,8.435,50.078 +TYNDP2024_1231,Arachthos (GR),Ottenhofen (DE),600,3000,under_consideration,"{name:Green Aegean Interconnector,url:{name:TYNDP2024_1231,url:https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx}}",20.967,39.185,11.868,48.207 diff --git a/data/transmission_projects/nep/new_lines.csv b/data/transmission_projects/nep/new_lines.csv new file mode 100644 index 00000000..19c50116 --- /dev/null +++ b/data/transmission_projects/nep/new_lines.csv @@ -0,0 +1,16 @@ +,bus0,bus1,length,v_nom,num_parallel,project_status,build_year,tags,x0,y0,x1,y1,type +P43,Dipperz/Hessen,Bergrheinfeld//Bayern,85,380.0,2.0,confirmed,2031,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Dipperz/Hessen"",""location1"":""Bergrheinfeld//Bayern""",9.807198712581199,50.54115855,10.1810033,50.0082136,Al/St 490/64 4-bundle 380.0 +P71,Kiel/Schleswig-Holstein,Göhl/Schleswig-Holstein,85,380.0,2.0,confirmed,2037,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Kiel/Schleswig-Holstein"",""location1"":""Göhl/Schleswig-Holstein""",10.135555,54.3227085,10.9399634,54.2861191,Al/St 490/64 4-bundle 380.0 +P84,Hamburg,Sahms/Schleswig-Holstein,35,380.0,2.0,confirmed,2031,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Hamburg"",""location1"":""Sahms/Schleswig-Holstein""",10.000654,53.550341,10.5331297,53.5252973,Al/St 490/64 4-bundle 380.0 +P227,Lübeck/Schleswig-Holstein,Sahms/Schleswig-Holstein,52,380.0,2.0,confirmed,2030,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Lübeck/Schleswig-Holstein"",""location1"":""Sahms/Schleswig-Holstein""",10.684738,53.866444,10.5331297,53.5252973,Al/St 490/64 4-bundle 380.0 +P402,Westerkappeln/Nordrhein-Westfalen,Gersteinwerk/Nordrhein-Westfalen,89,380.0,2.0,confirmed,2033,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Westerkappeln/Nordrhein-Westfalen"",""location1"":""Gersteinwerk/Nordrhein-Westfalen""",7.8772237,52.3141716,7.722207887811252,51.6749132,Al/St 490/64 4-bundle 380.0 +P470,Emden/Niedersachsen,Dörpen/West /Niedersachsen,66,380.0,2.0,confirmed,2037,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Emden/Niedersachsen"",""location1"":""Dörpen/West /Niedersachsen""",7.2058304,53.3670541,7.256622642835093,52.9807372,Al/St 490/64 4-bundle 380.0 +P476,Hochwöhrden/Schleswig-Holstein,Mehlbek/Schleswig-Holstein,39,380.0,2.0,confirmed,2032,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Hochwöhrden/Schleswig-Holstein"",""location1"":""Mehlbek/Schleswig-Holstein""",9.0323968,54.158602,9.4346536,54.0045912,Al/St 490/64 4-bundle 380.0 +P478,Pöschendorf/Schleswig-Holstein,Alfstedt/Niedersachsen,37,380.0,2.0,confirmed,2037,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Pöschendorf/Schleswig-Holstein"",""location1"":""Alfstedt/Niedersachsen""",9.4944931,54.0409041,9.0666915,53.5499782,Al/St 490/64 4-bundle 380.0 +P485,Herlasgrün/Sachsen,Marktleuthen/Bayern,69,380.0,2.0,confirmed,2037,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Herlasgrün/Sachsen"",""location1"":""Marktleuthen/Bayern""",12.2299537,50.5680975,11.9945062,50.1279651,Al/St 490/64 4-bundle 380.0 +P490,Petersgmünd /Bayern,Goldshöfe /Baden-Württemberg,104,380.0,2.0,confirmed,2037,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Petersgmünd /Bayern"",""location1"":""Goldshöfe /Baden-Württemberg""",11.0252473,49.185644,10.1279814,48.8940723,Al/St 490/64 4-bundle 380.0 +P540,Eisfeld/Thüringen,Grafenrheinfeld /Bayern,126,380.0,2.0,confirmed,2037,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Eisfeld/Thüringen"",""location1"":""Grafenrheinfeld /Bayern""",10.908994,50.4269984,10.1966808,50.0055317,Al/St 490/64 4-bundle 380.0 +P625,Streumen/Sachsen,Schmölln/Sachsen,92,380.0,2.0,confirmed,2035,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Streumen/Sachsen"",""location1"":""Schmölln/Sachsen""",13.4054041,51.3582767,14.2347005,51.1244043,Al/St 490/64 4-bundle 380.0 +P627,Schossin/Mecklenburg-Vorpommern,Perleberg/Brandenburg,89,380.0,2.0,confirmed,2037,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Schossin/Mecklenburg-Vorpommern"",""location1"":""Perleberg/Brandenburg""",11.2823593,53.5284043,11.8627933,53.0762716,Al/St 490/64 4-bundle 380.0 +P635,Grabowhöfe/Mecklenburg-Vorpommern,Marke/Sachsen-Anhalt,280,380.0,2.0,confirmed,2037,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Grabowhöfe/Mecklenburg-Vorpommern"",""location1"":""Marke/Sachsen-Anhalt""",12.594171,53.5683057,12.252458735300388,51.7301547,Al/St 490/64 4-bundle 380.0 +P636,Delitzsch/Sachsen,Eula/Sachsen,58,380.0,2.0,confirmed,2037,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Delitzsch/Sachsen"",""location1"":""Eula/Sachsen""",12.342857,51.5255661,12.5130329,51.1496022,Al/St 490/64 4-bundle 380.0 diff --git a/data/transmission_projects/nep/new_links.csv b/data/transmission_projects/nep/new_links.csv new file mode 100644 index 00000000..d237893f --- /dev/null +++ b/data/transmission_projects/nep/new_links.csv @@ -0,0 +1,12 @@ +,bus0,bus1,p_nom,length,project_status,build_year,underground,tags,x0,y0,x1,y1 +DC21,Wilhelmshaven/Niedersachsen,Lippetal/Nordrhein-Westfalen,2000.0,270,confirmed,2032,True,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Wilhelmshaven/Niedersachsen"",""location1"":""Lippetal/Nordrhein-Westfalen""",8.106301,53.5278793,8.0787633,51.6516157 +DC25,Heide/Schleswig-Holstein,Polsum/Nordrhein-Westfalen,2000.0,440,confirmed,2032,True,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Heide/Schleswig-Holstein"",""location1"":""Polsum/Nordrhein-Westfalen""",9.0929015,54.1948456,7.0528322,51.6264919 +DC31,Hemmingstedt/Schleswig-Holstein,Stralendorf/Mecklenburg-Vorpommern,2000.0,212,confirmed,2032,True,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Hemmingstedt/Schleswig-Holstein"",""location1"":""Stralendorf/Mecklenburg-Vorpommern""",9.08263,54.1506435,11.3010418,53.572637 +DC32,Pöschendorf/Schleswig-Holstein,Stralendorf/Mecklenburg-Vorpommern,2000.0,170,confirmed,2034,True,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Pöschendorf/Schleswig-Holstein"",""location1"":""Stralendorf/Mecklenburg-Vorpommern""",9.4944931,54.0409041,11.3010418,53.572637 +DC34,Ovelgönne/Niedersachsen,Bürstadt/Hessen,2000.0,523,confirmed,2033,True,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Ovelgönne/Niedersachsen"",""location1"":""Bürstadt/Hessen""",8.4209977,53.3427799,8.4747076,49.6531823 +DC35,Ovelgönne/Niedersachsen,Marxheim/Taunus,2000.0,461,confirmed,2035,True,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Ovelgönne/Niedersachsen"",""location1"":""Marxheim/Taunus""",8.4209977,53.3427799,8.4320722,50.0724011 +DC40,Nüttermoor/Niedersachsen,Streumen/Sachsen,2000.0,594,confirmed,2037,True,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Nüttermoor/Niedersachsen"",""location1"":""Streumen/Sachsen""",7.4360347,53.2625382,13.4054041,51.3582767 +DC40plus,Dörpen/West,Klostermansfeld/Sachsen-Anhalt,2000.0,426,confirmed,2037,True,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Dörpen/West"",""location1"":""Klostermansfeld/Sachsen-Anhalt""",7.256622642835093,52.9807372,11.4933395,51.5841525 +DC41,Alfstedt/Niedersachsen,Hüffenhardt/Baden-Württemberg,2000.0,607,confirmed,2037,True,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Alfstedt/Niedersachsen"",""location1"":""Hüffenhardt/Baden-Württemberg""",9.0666915,53.5499782,9.0827453,49.2908862 +DC42,Sahms/Schleswig-Holstein,Böblingen/Baden-Württemberg,2000.0,737,confirmed,2037,True,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Sahms/Schleswig-Holstein"",""location1"":""Böblingen/Baden-Württemberg""",10.5331297,53.5252973,9.0113444,48.684969 +DC42plus,Sahms/Schleswig-Holstein,Trennfeld/Bayern,2000.0,546,confirmed,2037,True,"{""url"":""https://data.netzausbau.de/2037-2023/NEP/NEP_2037_2045_Bestaetigung.pdf"", ""status"":""confirmed"", ""origin"":""NEP"",""version"":2024,""location0"":""Sahms/Schleswig-Holstein"",""location1"":""Trennfeld/Bayern""",10.5331297,53.5252973,9.6151453,49.7950676 diff --git a/data/transmission_projects/template/new_lines.csv b/data/transmission_projects/template/new_lines.csv new file mode 100644 index 00000000..2f75428c --- /dev/null +++ b/data/transmission_projects/template/new_lines.csv @@ -0,0 +1,5 @@ +,project_status,length,build_year,underground,v_nom,tags,x0,y0,x1,y1,num_parallel +New_line_1,in_permitting,100,2025,False,380,"{link to source, name of project, further custom information}",-7.793621146853301,41.52102394053344,-8.38943499999999,40.95916,2 +New_line_2,under_construction,100,2023,False,380,"{link to source, name of project, further custom information}",,,,, +New_line_3,in_planning,100,2040,False,380,"{link to source, name of project, further custom information}",,,,, +New_line_4,under_consideration,100,2050,true,380,"{link to source, name of project, further custom information}",,,,, diff --git a/data/transmission_projects/template/new_links.csv b/data/transmission_projects/template/new_links.csv new file mode 100644 index 00000000..ff2d652e --- /dev/null +++ b/data/transmission_projects/template/new_links.csv @@ -0,0 +1,5 @@ +,project_status,length,build_year,underground,p_nom,tags,x0,y0,x1,y1 +New_link_1,in_permitting,100,2025,False,1000,"{link to source, name of project, further custom information}",-7.793621,41.52145,-8.38944,40.95915 +New_link_2,under_construction,100,2023,False,600,"{link to source, name of project, further custom information}",-7.793621,41.52145,-8.38944,40.95915 +New_link_3,in_planning,100,2040,False,2000,"{link to source, name of project, further custom information}",-7.793621,41.52145,-8.38944,40.95915 +New_link_4,under_consideration,100,2050,true,600,"{link to source, name of project, further custom information}",-7.793621,41.52145,-8.38944,40.95915 diff --git a/data/transmission_projects/template/upgraded_lines.csv b/data/transmission_projects/template/upgraded_lines.csv new file mode 100644 index 00000000..86192a3f --- /dev/null +++ b/data/transmission_projects/template/upgraded_lines.csv @@ -0,0 +1,5 @@ +,project_status,length,build_year,underground,v_nom,tags,x0,y0,x1,y1,num_parallel +Upgraded_line_1,in_permitting,100,2025,False,380,"{link to source, name of project, further custom information}",-7.793621146853301,41.52102394053344,-8.38943499999999,40.95916,2 +Upgraded_line_2,under_construction,100,2023,False,380,"{link to source, name of project, further custom information}",,,,, +Upgraded_line_3,in_planning,100,2040,False,380,"{link to source, name of project, further custom information}",,,,, +Upgraded_line_4,under_consideration,100,2050,true,380,"{link to source, name of project, further custom information}",,,,, diff --git a/data/transmission_projects/template/upgraded_links.csv b/data/transmission_projects/template/upgraded_links.csv new file mode 100644 index 00000000..65270cc3 --- /dev/null +++ b/data/transmission_projects/template/upgraded_links.csv @@ -0,0 +1,5 @@ +,project_status,length,build_year,underground,p_nom,tags,x0,y0,x1,y1 +Upgraded_link_1,in_permitting,100,2025,False,1000,"{link to source, name of project, further custom information}",-7.793621,41.52145,-8.38944,40.95915 +Upgraded_link_2,under_construction,100,2023,False,600,"{link to source, name of project, further custom information}",-7.793621,41.52145,-8.38944,40.95915 +Upgraded_link_3,in_planning,100,2040,False,2000,"{link to source, name of project, further custom information}",-7.793621,41.52145,-8.38944,40.95915 +Upgraded_link_4,under_consideration,100,2050,true,600,"{link to source, name of project, further custom information}",-7.793621,41.52145,-8.38944,40.95915 diff --git a/data/transmission_projects/tyndp2020/new_lines.csv b/data/transmission_projects/tyndp2020/new_lines.csv new file mode 100644 index 00000000..24729655 --- /dev/null +++ b/data/transmission_projects/tyndp2020/new_lines.csv @@ -0,0 +1,119 @@ +,project_status,length,build_year,underground,v_nom,tags,x0,y0,x1,y1,num_parallel,type +TYNDP2020_0,in_permitting,131.0,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1"", ""tyndp2020_proj_id""=>""1"", ""tyndp2020_invest_id""=>""4"", ""tyndp_status""=>""in_permitting""",-7.793621146853301,41.52102394053344,-8.38943499999999,40.95916,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_1,in_permitting,30.0,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/4"", ""tyndp2020_proj_id""=>""4"", ""tyndp2020_invest_id""=>""18"", ""tyndp_status""=>""in_permitting""",-8.273869552394737,42.46748640601422,-7.94998199999999,42.383908,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_2,in_permitting,140.21,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/4"", ""tyndp2020_proj_id""=>""4"", ""tyndp2020_invest_id""=>""496"", ""tyndp_status""=>""in_permitting""",-7.94998199999999,42.383908,-8.582626164089064,41.764651842353146,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_3,in_permitting,140.21,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/4"", ""tyndp2020_proj_id""=>""4"", ""tyndp2020_invest_id""=>""496"", ""tyndp_status""=>""in_permitting""",-8.582626164089064,41.764651842353146,-8.67645299999999,41.303603,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_4,under_construction,23.0,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/23"", ""tyndp2020_proj_id""=>""23"", ""tyndp2020_invest_id""=>""60"", ""tyndp_status""=>""under_construction""",3.08990499999999,50.50906,3.429108,50.773813,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_6,in_permitting,80.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/33"", ""tyndp2020_proj_id""=>""33"", ""tyndp2020_invest_id""=>""90"", ""tyndp_status""=>""in_permitting""",11.126404,43.872158,11.413422,44.580687,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_8,in_permitting,120.5,2026,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/35"", ""tyndp2020_proj_id""=>""35"", ""tyndp2020_invest_id""=>""313"", ""tyndp_status""=>""in_permitting""",14.3524,49.1584,15.687103,49.531448,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_9,in_permitting,117.0,2028,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/35"", ""tyndp2020_proj_id""=>""35"", ""tyndp2020_invest_id""=>""315"", ""tyndp_status""=>""in_permitting""",14.3524,49.1584,13.282471,49.60804,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_11,under_construction,110.0,2020,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/39"", ""tyndp2020_proj_id""=>""39"", ""tyndp2020_invest_id""=>""144"", ""tyndp_status""=>""under_construction""",9.75860599999998,54.306108,9.29580699999999,55.075223,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_12,planned_not_yet_permitting,157.0,2027,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/47"", ""tyndp2020_proj_id""=>""47"", ""tyndp2020_invest_id""=>""689"", ""tyndp_status""=>""planned_not_yet_permitting""",10.06485,48.282279,10.897064,47.236355,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_13,under_construction,110.0,2020,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/48"", ""tyndp2020_proj_id""=>""48"", ""tyndp2020_invest_id""=>""1500"", ""tyndp_status""=>""under_construction""",17.539673,47.882276,17.786865,47.653363,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_14,under_construction,110.0,2020,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/48"", ""tyndp2020_proj_id""=>""48"", ""tyndp2020_invest_id""=>""1500"", ""tyndp_status""=>""under_construction""",17.539673,47.882276,18.540802,48.236565,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_15,under_construction,48.0,2020,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/48"", ""tyndp2020_proj_id""=>""48"", ""tyndp2020_invest_id""=>""1501"", ""tyndp_status""=>""under_construction""",19.958038,48.380882,20.55954,48.263084,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_16,in_permitting,60.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/78"", ""tyndp2020_proj_id""=>""78"", ""tyndp2020_invest_id""=>""458"", ""tyndp_status""=>""in_permitting""",-3.21075400000001,51.135416,-2.633972,51.533523,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_17,in_permitting,138.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/81"", ""tyndp2020_proj_id""=>""81"", ""tyndp2020_invest_id""=>""462"", ""tyndp_status""=>""in_permitting""",-6.61376999999999,53.441446,-6.65496799999999,54.509921,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_21,planned_not_yet_permitting,61.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/85"", ""tyndp2020_proj_id""=>""85"", ""tyndp2020_invest_id""=>""779"", ""tyndp_status""=>""planned_not_yet_permitting""",-8.046112,38.05458,-8.1897,37.6672,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_22,planned_not_yet_permitting,61.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/85"", ""tyndp2020_proj_id""=>""85"", ""tyndp2020_invest_id""=>""779"", ""tyndp_status""=>""planned_not_yet_permitting""",-8.1897,37.6672,-7.73025499999999,37.397437,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_23,planned_not_yet_permitting,75.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/85"", ""tyndp2020_proj_id""=>""85"", ""tyndp2020_invest_id""=>""1670"", ""tyndp_status""=>""planned_not_yet_permitting""",-7.55310100000001,38.17991,-7.89230299999999,38.808681,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_24,under_construction,75.0,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/103"", ""tyndp2020_proj_id""=>""103"", ""tyndp2020_invest_id""=>""1488"", ""tyndp_status""=>""under_construction""",4.9617,52.333661,5.763702,52.619725,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_25,in_permitting,200.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/111"", ""tyndp2020_proj_id""=>""111"", ""tyndp2020_invest_id""=>""396"", ""tyndp_status""=>""in_permitting""",24.612122,65.877531,20.232697,66.70191,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_26,in_permitting,156.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/111"", ""tyndp2020_proj_id""=>""111"", ""tyndp2020_invest_id""=>""1710"", ""tyndp_status""=>""in_permitting""",24.612122,65.877531,26.114502,64.778807,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_27,under_construction,108.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/123"", ""tyndp2020_proj_id""=>""123"", ""tyndp2020_invest_id""=>""373"", ""tyndp_status""=>""under_construction""",21.625214,53.051946,21.40686,52.213497,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_28,in_permitting,160.0,2026,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/124"", ""tyndp2020_proj_id""=>""124"", ""tyndp2020_invest_id""=>""733"", ""tyndp_status""=>""in_permitting""",16.659393,57.43682,14.749146,56.326437,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_29,in_permitting,178.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/127"", ""tyndp2020_proj_id""=>""127"", ""tyndp2020_invest_id""=>""86"", ""tyndp_status""=>""in_permitting""",15.636292,41.359288,14.028168,42.409263,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_30,under_construction,30.0,2021,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/127"", ""tyndp2020_proj_id""=>""127"", ""tyndp2020_invest_id""=>""96"", ""tyndp_status""=>""under_construction""",15.555267,41.165216,15.3286,40.9749,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_31,under_construction,159.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/138"", ""tyndp2020_proj_id""=>""138"", ""tyndp2020_invest_id""=>""273"", ""tyndp_status""=>""under_construction""",28.02475,44.323848,26.7166,45.0762,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_32,in_permitting,140.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/138"", ""tyndp2020_proj_id""=>""138"", ""tyndp2020_invest_id""=>""275"", ""tyndp_status""=>""in_permitting""",27.960205,45.447607,26.872559,46.265341,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_33,under_construction,86.0,2021,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/138"", ""tyndp2020_proj_id""=>""138"", ""tyndp2020_invest_id""=>""800"", ""tyndp_status""=>""under_construction""",27.610016,43.326177,27.358704,42.576344,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_34,in_permitting,151.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/142"", ""tyndp2020_proj_id""=>""142"", ""tyndp2020_invest_id""=>""256"", ""tyndp_status""=>""in_permitting""",26.003265,42.200038,23.083964803314,40.8967964275543,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_35,under_construction,13.0,2021,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/142"", ""tyndp2020_proj_id""=>""142"", ""tyndp2020_invest_id""=>""258"", ""tyndp_status""=>""under_construction""",26.003265,42.200038,25.804138,41.964596,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_36,under_construction,150.0,2021,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/142"", ""tyndp2020_proj_id""=>""142"", ""tyndp2020_invest_id""=>""262"", ""tyndp_status""=>""under_construction""",26.003265,42.200038,27.358704,42.576344,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_37,under_construction,131.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/144"", ""tyndp2020_proj_id""=>""144"", ""tyndp2020_invest_id""=>""238"", ""tyndp_status""=>""under_construction""",21.88777,45.30413,20.706482,44.86755,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_38,under_construction,116.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/144"", ""tyndp2020_proj_id""=>""144"", ""tyndp2020_invest_id""=>""269"", ""tyndp_status""=>""under_construction""",22.578278,44.679395,21.88777,45.30413,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_39,in_permitting,274.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/144"", ""tyndp2020_proj_id""=>""144"", ""tyndp2020_invest_id""=>""270"", ""tyndp_status""=>""in_permitting""",21.2815,45.7426,21.2815,45.7426,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_43,planned_not_yet_permitting,57.11,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/170"", ""tyndp2020_proj_id""=>""170"", ""tyndp2020_invest_id""=>""1661"", ""tyndp_status""=>""planned_not_yet_permitting""",16.11557,54.132674,16.6857237480894,54.0342010066989,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_44,planned_not_yet_permitting,119.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/170"", ""tyndp2020_proj_id""=>""170"", ""tyndp2020_invest_id""=>""1662"", ""tyndp_status""=>""planned_not_yet_permitting""",16.858521,53.16571,16.6857237480894,54.0342010066989,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_45,planned_not_yet_permitting,288.5,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/170"", ""tyndp2020_proj_id""=>""170"", ""tyndp2020_invest_id""=>""1664"", ""tyndp_status""=>""planned_not_yet_permitting""",14.831543,53.330873,18.104095,54.714309,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_46,planned_not_yet_permitting,77.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/170"", ""tyndp2020_proj_id""=>""170"", ""tyndp2020_invest_id""=>""1665"", ""tyndp_status""=>""planned_not_yet_permitting""",18.104095,54.714309,18.684998,54.266026,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_47,in_permitting,92.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/183"", ""tyndp2020_proj_id""=>""183"", ""tyndp2020_invest_id""=>""1018"", ""tyndp_status""=>""in_permitting""",8.86459400000001,55.074436,8.66683999999999,55.481966,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_48,under_construction,60.0,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/186"", ""tyndp2020_proj_id""=>""186"", ""tyndp2020_invest_id""=>""886"", ""tyndp_status""=>""under_construction""",16.399841,48.323387,16.820068,48.943249,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_49,in_permitting,60.0,2028,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/187"", ""tyndp2020_proj_id""=>""187"", ""tyndp2020_invest_id""=>""997"", ""tyndp_status""=>""in_permitting""",13.080597,48.256684,13.080597,48.256684,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_50,under_construction,300.0,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/197"", ""tyndp2020_proj_id""=>""197"", ""tyndp2020_invest_id""=>""742"", ""tyndp_status""=>""under_construction""",26.114502,64.778807,25.180664,62.262171,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_51,in_permitting,83.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/200"", ""tyndp2020_proj_id""=>""200"", ""tyndp2020_invest_id""=>""308"", ""tyndp_status""=>""in_permitting""",13.191833,50.341955,12.675274,50.160683,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_52,under_construction,87.0,2021,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/200"", ""tyndp2020_proj_id""=>""200"", ""tyndp2020_invest_id""=>""309"", ""tyndp_status""=>""under_construction""",12.675274,50.160683,13.282471,49.60804,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_53,under_construction,60.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/207"", ""tyndp2020_proj_id""=>""207"", ""tyndp2020_invest_id""=>""939"", ""tyndp_status""=>""under_construction""",8.09143099999999,53.313647,7.23587000000001,53.374317,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_54,under_construction,30.0,2020,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/207"", ""tyndp2020_proj_id""=>""207"", ""tyndp2020_invest_id""=>""1684"", ""tyndp_status""=>""under_construction""",8.09143099999999,53.313647,8.00079299999999,53.55581,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_56,under_construction,182.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/208"", ""tyndp2020_proj_id""=>""208"", ""tyndp2020_invest_id""=>""156"", ""tyndp_status""=>""under_construction""",6.615143,51.630805,7.38830599999999,53.076703,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_57,in_permitting,45.2,2026,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/227"", ""tyndp2020_proj_id""=>""227"", ""tyndp2020_invest_id""=>""627"", ""tyndp_status""=>""in_permitting""",19.500315,43.953021,19.388123,43.723475,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_58,in_permitting,100.0,2026,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/227"", ""tyndp2020_proj_id""=>""227"", ""tyndp2020_invest_id""=>""630"", ""tyndp_status""=>""in_permitting""",19.500315,43.953021,19.315338,43.347152,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_59,under_construction,151.0,2019,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/227"", ""tyndp2020_proj_id""=>""227"", ""tyndp2020_invest_id""=>""1526"", ""tyndp_status""=>""under_construction""",18.772888,42.315909,19.315338,43.347152,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_60,under_construction,,2020,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/227"", ""tyndp2020_proj_id""=>""227"", ""tyndp2020_invest_id""=>""1527"", ""tyndp_status""=>""under_construction""",20.928955,44.056999,20.739984,43.680615,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_61,planned_not_yet_permitting,32.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/228"", ""tyndp2020_proj_id""=>""228"", ""tyndp2020_invest_id""=>""1231"", ""tyndp_status""=>""planned_not_yet_permitting""",7.50366199999999,47.891485,7.71240200000001,48.103763,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_62,under_consideration,40.0,2035,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/229"", ""tyndp2020_proj_id""=>""229"", ""tyndp2020_invest_id""=>""1270"", ""tyndp_status""=>""under_consideration""",15.038505,52.76751,15.5,51.93333,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_63,under_consideration,40.0,2035,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/229"", ""tyndp2020_proj_id""=>""229"", ""tyndp2020_invest_id""=>""1271"", ""tyndp_status""=>""under_consideration""",15.5,51.93333,16.729431,52.32359,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_64,under_consideration,90.0,2035,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/229"", ""tyndp2020_proj_id""=>""229"", ""tyndp2020_invest_id""=>""1673"", ""tyndp_status""=>""under_consideration""",15.5,51.93333,16.056519,51.449728,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_65,under_consideration,80.0,2035,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/229"", ""tyndp2020_proj_id""=>""229"", ""tyndp2020_invest_id""=>""1674"", ""tyndp_status""=>""under_consideration""",15.5,51.93333,14.615936,52.125058,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_66,in_permitting,100.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/230"", ""tyndp2020_proj_id""=>""230"", ""tyndp2020_invest_id""=>""355"", ""tyndp_status""=>""in_permitting""",15.086975,51.069017,16.365509,50.816348,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_67,under_consideration,11.0,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/231"", ""tyndp2020_proj_id""=>""231"", ""tyndp2020_invest_id""=>""1282"", ""tyndp_status""=>""under_consideration""",8.16970799999999,47.523693,8.205414,47.649662,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_68,under_consideration,91.0,2033,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/241"", ""tyndp2020_proj_id""=>""241"", ""tyndp2020_invest_id""=>""1276"", ""tyndp_status""=>""under_consideration""",18.420959,45.289022,18.538055,44.455349,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_69,under_consideration,46.2,2033,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/241"", ""tyndp2020_proj_id""=>""241"", ""tyndp2020_invest_id""=>""1277"", ""tyndp_status""=>""under_consideration""",18.420959,45.289022,18.436344,44.904884,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_70,under_consideration,25.0,2033,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/241"", ""tyndp2020_proj_id""=>""241"", ""tyndp2020_invest_id""=>""1279"", ""tyndp_status""=>""under_consideration""",18.420959,45.289022,18.420959,45.289022,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_71,under_consideration,47.8,2032,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/241"", ""tyndp2020_proj_id""=>""241"", ""tyndp2020_invest_id""=>""1530"", ""tyndp_status""=>""under_consideration""",18.436344,44.904884,18.538055,44.455349,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_72,under_consideration,70.0,2035,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/243"", ""tyndp2020_proj_id""=>""243"", ""tyndp2020_invest_id""=>""1269"", ""tyndp_status""=>""under_consideration""",18.663025,45.454351,19.175262,45.75411,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_73,planned_not_yet_permitting,65.0,2028,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/244"", ""tyndp2020_proj_id""=>""244"", ""tyndp2020_invest_id""=>""1245"", ""tyndp_status""=>""planned_not_yet_permitting""",6.25122100000001,49.182601,7.028503,49.340336,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_74,under_construction,27.0,2021,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/245"", ""tyndp2020_proj_id""=>""245"", ""tyndp2020_invest_id""=>""1246"", ""tyndp_status""=>""under_construction""",6.936493,53.145947,7.389679,53.097323,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_76,planned_not_yet_permitting,90.0,2029,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/252"", ""tyndp2020_proj_id""=>""252"", ""tyndp2020_invest_id""=>""1050"", ""tyndp_status""=>""planned_not_yet_permitting""",5.71701000000001,51.094036,5.35583499999999,50.557942,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_77,in_permitting,90.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/252"", ""tyndp2020_proj_id""=>""252"", ""tyndp2020_invest_id""=>""1456"", ""tyndp_status""=>""in_permitting""",4.603271,51.209464,5.71701000000001,51.094036,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_78,planned_not_yet_permitting,60.0,2033,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/252"", ""tyndp2020_proj_id""=>""252"", ""tyndp2020_invest_id""=>""1515"", ""tyndp_status""=>""planned_not_yet_permitting""",5.35583499999999,50.557942,4.35470599999999,50.529143,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_79,planned_not_yet_permitting,47.0,2035,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/252"", ""tyndp2020_proj_id""=>""252"", ""tyndp2020_invest_id""=>""1516"", ""tyndp_status""=>""planned_not_yet_permitting""",4.22012299999999,50.864911,4.35470599999999,50.529143,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_80,planned_not_yet_permitting,16.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/252"", ""tyndp2020_proj_id""=>""252"", ""tyndp2020_invest_id""=>""1517"", ""tyndp_status""=>""planned_not_yet_permitting""",4.177551,51.162122,4.22012299999999,50.864911,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_81,planned_not_yet_permitting,36.0,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/252"", ""tyndp2020_proj_id""=>""252"", ""tyndp2020_invest_id""=>""1676"", ""tyndp_status""=>""planned_not_yet_permitting""",4.177551,51.162122,4.603271,51.209464,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_82,under_construction,125.0,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/258"", ""tyndp2020_proj_id""=>""258"", ""tyndp2020_invest_id""=>""667"", ""tyndp_status""=>""under_construction""",9.24087500000001,53.906765,8.86459400000001,55.074436,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_83,under_consideration,120.0,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/259"", ""tyndp2020_proj_id""=>""259"", ""tyndp2020_invest_id""=>""1205"", ""tyndp_status""=>""under_consideration""",21.566162,47.659838,22.026215,47.044861,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_84,under_consideration,1.0,2035,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/263"", ""tyndp2020_proj_id""=>""263"", ""tyndp2020_invest_id""=>""1258"", ""tyndp_status""=>""under_consideration""",9.556022,47.302744,8.298798,47.148634,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_85,under_consideration,1.0,2035,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/263"", ""tyndp2020_proj_id""=>""263"", ""tyndp2020_invest_id""=>""1258"", ""tyndp_status""=>""under_consideration""",9.556022,47.302744,9.253235,46.824496,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_86,under_consideration,,2035,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/263"", ""tyndp2020_proj_id""=>""263"", ""tyndp2020_invest_id""=>""1583"", ""tyndp_status""=>""under_consideration""",9.556022,47.302744,9.80228979023356,47.6377345779987,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_87,under_consideration,,2035,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/263"", ""tyndp2020_proj_id""=>""263"", ""tyndp2020_invest_id""=>""1583"", ""tyndp_status""=>""under_consideration""",9.556022,47.302744,9.790192,47.138359,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_88,in_permitting,64.0,2029,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/264"", ""tyndp2020_proj_id""=>""264"", ""tyndp2020_invest_id""=>""1259"", ""tyndp_status""=>""in_permitting""",8.16970799999999,47.523693,8.298798,47.148634,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_89,in_permitting,45.4,2027,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/264"", ""tyndp2020_proj_id""=>""264"", ""tyndp2020_invest_id""=>""1287"", ""tyndp_status""=>""in_permitting""",7.19192499999999,47.364874,7.22213700000001,47.007416,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_90,in_permitting,87.1,2035,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/264"", ""tyndp2020_proj_id""=>""264"", ""tyndp2020_invest_id""=>""1288"", ""tyndp_status""=>""in_permitting""",8.298798,47.148634,8.32488999999999,46.532414,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_92,in_permitting,124.0,2029,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/266"", ""tyndp2020_proj_id""=>""266"", ""tyndp2020_invest_id""=>""1286"", ""tyndp_status""=>""in_permitting""",7.555847,46.336499,8.795929,46.401882,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_93,in_permitting,106.6,2027,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/266"", ""tyndp2020_proj_id""=>""266"", ""tyndp2020_invest_id""=>""1733"", ""tyndp_status""=>""in_permitting""",7.64236499999999,47.079475,7.555847,46.336499,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_94,under_construction,30.0,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/266"", ""tyndp2020_proj_id""=>""266"", ""tyndp2020_invest_id""=>""1734"", ""tyndp_status""=>""under_construction""",7.165833,46.200745,7.555847,46.336499,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_95,planned_not_yet_permitting,110.0,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/270"", ""tyndp2020_proj_id""=>""270"", ""tyndp2020_invest_id""=>""1212"", ""tyndp_status""=>""planned_not_yet_permitting""",-1.14818,42.10892,-1.671123,41.952398,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_96,planned_not_yet_permitting,80.0,2029,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/276"", ""tyndp2020_proj_id""=>""276"", ""tyndp2020_invest_id""=>""1208"", ""tyndp_status""=>""planned_not_yet_permitting""",-0.812988000000009,43.979969,-0.575409000000004,43.345155,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_97,under_construction,1.0,2020,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/297"", ""tyndp2020_proj_id""=>""297"", ""tyndp2020_invest_id""=>""445"", ""tyndp_status""=>""under_construction""",4.307132,51.301299,4.307132,51.301299,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_98,under_construction,128.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/312"", ""tyndp2020_proj_id""=>""312"", ""tyndp2020_invest_id""=>""1472"", ""tyndp_status""=>""under_construction""",13.080597,48.256684,12.803192,47.244746,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_99,in_permitting,90.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/313"", ""tyndp2020_proj_id""=>""313"", ""tyndp2020_invest_id""=>""1473"", ""tyndp_status""=>""in_permitting""",12.172852,48.638354,13.080597,48.256684,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_100,under_construction,80.0,2021,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/320"", ""tyndp2020_proj_id""=>""320"", ""tyndp2020_invest_id""=>""1558"", ""tyndp_status""=>""under_construction""",15.724182,46.386728,16.689606,46.633408,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_101,in_permitting,85.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/322"", ""tyndp2020_proj_id""=>""322"", ""tyndp2020_invest_id""=>""1476"", ""tyndp_status""=>""in_permitting""",10.06485,48.282279,9.73251299999999,47.623752,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_104,planned_not_yet_permitting,40.0,2026,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/328"", ""tyndp2020_proj_id""=>""328"", ""tyndp2020_invest_id""=>""1630"", ""tyndp_status""=>""planned_not_yet_permitting""",6.590884,49.789617,6.140853,49.688695,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_105,under_consideration,78.0,2035,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/330"", ""tyndp2020_proj_id""=>""330"", ""tyndp2020_invest_id""=>""1498"", ""tyndp_status""=>""under_consideration""",17.440796,49.236431,18.287416,49.038186,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_107,planned_not_yet_permitting,100.0,2028,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/340"", ""tyndp2020_proj_id""=>""340"", ""tyndp2020_invest_id""=>""1519"", ""tyndp_status""=>""planned_not_yet_permitting""",3.429108,50.773813,4.35470599999999,50.529143,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_108,under_consideration,60.0,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/341"", ""tyndp2020_proj_id""=>""341"", ""tyndp2020_invest_id""=>""1538"", ""tyndp_status""=>""under_consideration""",20.9729,44.874362,20.378265,44.819838,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_109,planned_not_yet_permitting,180.0,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/343"", ""tyndp2020_proj_id""=>""343"", ""tyndp2020_invest_id""=>""1532"", ""tyndp_status""=>""planned_not_yet_permitting""",17.201843,44.772087,15.26017,44.886816,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_110,planned_not_yet_permitting,68.0,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/343"", ""tyndp2020_proj_id""=>""343"", ""tyndp2020_invest_id""=>""1533"", ""tyndp_status""=>""planned_not_yet_permitting""",15.26017,44.886816,14.558258,45.323186,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_111,under_consideration,165.0,2040,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/344"", ""tyndp2020_proj_id""=>""344"", ""tyndp2020_invest_id""=>""1541"", ""tyndp_status""=>""under_consideration""",6.14410399999999,52.52207,5.671692,51.908696,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_113,under_construction,161.0,2020,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/350"", ""tyndp2020_proj_id""=>""350"", ""tyndp2020_invest_id""=>""1622"", ""tyndp_status""=>""under_construction""",20.725963,41.239159,19.966278,41.16108,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_115,under_consideration,38.8,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/379"", ""tyndp2020_proj_id""=>""379"", ""tyndp2020_invest_id""=>""1593"", ""tyndp_status""=>""under_consideration""",-2.880286,43.349776,-3.05282599999999,43.265206,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_116,under_consideration,69.12,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/379"", ""tyndp2020_proj_id""=>""379"", ""tyndp2020_invest_id""=>""1595"", ""tyndp_status""=>""under_consideration""",-2.880286,43.349776,-2.24395799999999,43.047816,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_117,under_consideration,57.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/379"", ""tyndp2020_proj_id""=>""379"", ""tyndp2020_invest_id""=>""1597"", ""tyndp_status""=>""under_consideration""",-2.880286,43.349776,-2.20687899999999,43.098977,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_120,under_construction,14.0,2021,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1043"", ""tyndp2020_proj_id""=>""1043"", ""tyndp2020_invest_id""=>""1685"", ""tyndp_status""=>""under_construction""",9.48532099999999,53.629168,9.48532099999999,53.629168,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_121,under_construction,60.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1043"", ""tyndp2020_proj_id""=>""1043"", ""tyndp2020_invest_id""=>""1686"", ""tyndp_status""=>""under_construction""",10.299683,52.286643,9.988131,51.954234,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_122,under_construction,45.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1043"", ""tyndp2020_proj_id""=>""1043"", ""tyndp2020_invest_id""=>""1686"", ""tyndp_status""=>""under_construction""",9.988131,51.954234,9.852428,51.649318,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_123,under_construction,105.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1043"", ""tyndp2020_proj_id""=>""1043"", ""tyndp2020_invest_id""=>""1686"", ""tyndp_status""=>""under_construction""",9.852428,51.649318,9.621277,50.85971,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_124,in_permitting,290.0,2028,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1046"", ""tyndp2020_proj_id""=>""1046"", ""tyndp2020_invest_id""=>""1688"", ""tyndp_status""=>""in_permitting""",26.842346,64.556701,27.766571,62.262171,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_125,in_permitting,100.0,2029,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1046"", ""tyndp2020_proj_id""=>""1046"", ""tyndp2020_invest_id""=>""1689"", ""tyndp_status""=>""in_permitting""",25.692902,65.279688,26.842346,64.556701,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_126,planned_not_yet_permitting,175.0,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1046"", ""tyndp2020_proj_id""=>""1046"", ""tyndp2020_invest_id""=>""1697"", ""tyndp_status""=>""planned_not_yet_permitting""",25.180664,62.262171,24.915619,60.90707,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_127,under_consideration,31.0,2035,True,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1047"", ""tyndp2020_proj_id""=>""1047"", ""tyndp2020_invest_id""=>""1691"", ""tyndp_status""=>""under_consideration""",7.23587000000001,53.374317,6.86370800000001,53.418536,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_128,planned_not_yet_permitting,170.0,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1052"", ""tyndp2020_proj_id""=>""1052"", ""tyndp2020_invest_id""=>""1713"", ""tyndp_status""=>""planned_not_yet_permitting""",12.693329,46.798179,14.5912685597917,46.536256583087,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_129,planned_not_yet_permitting,105.0,2027,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1054"", ""tyndp2020_proj_id""=>""1054"", ""tyndp2020_invest_id""=>""1715"", ""tyndp_status""=>""planned_not_yet_permitting""",10.897064,47.236355,11.964111,47.292271,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_130,under_consideration,90.0,2035,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1056"", ""tyndp2020_proj_id""=>""1056"", ""tyndp2020_invest_id""=>""1723"", ""tyndp_status""=>""under_consideration""",16.78425,43.503952,17.436701,43.052743,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_132,in_permitting,50.0,2027,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1059"", ""tyndp2020_proj_id""=>""1059"", ""tyndp2020_invest_id""=>""645"", ""tyndp_status""=>""in_permitting""",15.996094,39.990799,16.020813,39.780047,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_133,in_permitting,,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1059"", ""tyndp2020_proj_id""=>""1059"", ""tyndp2020_invest_id""=>""1727"", ""tyndp_status""=>""in_permitting""",15.03067,40.66814,15.058136,40.941527,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_134,in_permitting,,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1059"", ""tyndp2020_proj_id""=>""1059"", ""tyndp2020_invest_id""=>""1727"", ""tyndp_status""=>""in_permitting""",15.058136,40.941527,14.957886,41.143501,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_135,under_construction,40.0,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1063"", ""tyndp2020_proj_id""=>""1063"", ""tyndp2020_invest_id""=>""1731"", ""tyndp_status""=>""under_construction""",4.218787,51.424034,4.218787,51.424034,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_136,under_consideration,50.6,2036,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1074"", ""tyndp2020_proj_id""=>""1074"", ""tyndp2020_invest_id""=>""1742"", ""tyndp_status""=>""under_consideration""",19.720459,46.020807,20.170898,46.29856,2,Al/St 240/40 4-bundle 380.0 +TYNDP2020_137,under_consideration,80.0,2036,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1074"", ""tyndp2020_proj_id""=>""1074"", ""tyndp2020_invest_id""=>""1743"", ""tyndp_status""=>""under_consideration""",19.175262,45.75411,19.756165,45.527517,2,Al/St 240/40 4-bundle 380.0 diff --git a/data/transmission_projects/tyndp2020/new_links.csv b/data/transmission_projects/tyndp2020/new_links.csv new file mode 100644 index 00000000..902cc9b3 --- /dev/null +++ b/data/transmission_projects/tyndp2020/new_links.csv @@ -0,0 +1,49 @@ +,project_status,length,build_year,underground,p_nom,tags,x0,y0,x1,y1 +TYNDP2020_0,under_consideration,330.0,2033,True,600.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/234"", ""tyndp2020_proj_id""=>""234"", ""tyndp2020_invest_id""=>""1236"", ""tyndp_status""=>""under_consideration""",12.492828,55.609384,16.11557,54.132674 +TYNDP2020_1,in_permitting,700.0,2026,True,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/235"", ""tyndp2020_proj_id""=>""235"", ""tyndp2020_invest_id""=>""664"", ""tyndp_status""=>""in_permitting""",9.24087500000001,53.906765,9.12211621382025,49.0939516665424 +TYNDP2020_2,in_permitting,558.0,2026,True,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/235"", ""tyndp2020_proj_id""=>""235"", ""tyndp2020_invest_id""=>""664"", ""tyndp_status""=>""in_permitting""",9.327393,53.931837,10.081329,49.950336 +TYNDP2020_3,under_consideration,200.0,2035,True,800.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/239"", ""tyndp2020_proj_id""=>""239"", ""tyndp2020_invest_id""=>""1241"", ""tyndp_status""=>""under_consideration""",17.166138,63.184727,21.901245,63.03068 +TYNDP2020_4,planned_not_yet_permitting,267.0,2030,False,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1034"", ""tyndp2020_proj_id""=>""1034"", ""tyndp2020_invest_id""=>""1692"", ""tyndp_status""=>""planned_not_yet_permitting""",8.00079299999999,53.55581,8.06808499999999,51.653815 +TYNDP2020_5,planned_not_yet_permitting,407.0,2030,False,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1034"", ""tyndp2020_proj_id""=>""1034"", ""tyndp2020_invest_id""=>""1693"", ""tyndp_status""=>""planned_not_yet_permitting""",9.13101199999999,54.20101,7.06420899999999,51.66063 +TYNDP2020_6,under_construction,445.0,2026,True,600.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/28"", ""tyndp2020_proj_id""=>""28"", ""tyndp2020_invest_id""=>""1503"", ""tyndp_status""=>""under_construction""",14.028168,42.409263,18.772888,42.315909 +TYNDP2020_7,in_permitting,240.0,2023,True,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/247"", ""tyndp2020_proj_id""=>""247"", ""tyndp2020_invest_id""=>""1381"", ""tyndp_status""=>""in_permitting""",-1.05331400000001,50.900435,1.11373900000001,49.712937 +TYNDP2020_8,under_consideration,133.0,2027,True,700.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1040"", ""tyndp2020_proj_id""=>""1040"", ""tyndp2020_invest_id""=>""1677"", ""tyndp_status""=>""under_consideration""",-5.80902099999999,54.726206,-4.880676,55.704677 +TYNDP2020_10,in_permitting,340.0,2024,False,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/254"", ""tyndp2020_proj_id""=>""254"", ""tyndp2020_invest_id""=>""660"", ""tyndp_status""=>""in_permitting""",6.58493,51.245584,8.478699,49.283036 +TYNDP2020_11,under_consideration,230.0,2025,True,1400.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1049"", ""tyndp2020_proj_id""=>""1049"", ""tyndp2020_invest_id""=>""1706"", ""tyndp_status""=>""under_consideration""",0.763549999999995,51.403489,3.18054200000001,51.330612 +TYNDP2020_12,under_consideration,610.0,2028,True,1400.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1050"", ""tyndp2020_proj_id""=>""1050"", ""tyndp2020_invest_id""=>""1707"", ""tyndp_status""=>""under_consideration""",0.482024999999987,51.519853,7.23587000000001,53.374317 +TYNDP2020_13,under_consideration,640.0,2028,True,1400.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1051"", ""tyndp2020_proj_id""=>""1051"", ""tyndp2020_invest_id""=>""1708"", ""tyndp_status""=>""under_consideration""",1.30737299999999,52.593038,9.22851599999999,55.481188 +TYNDP2020_14,under_consideration,300.0,2030,True,1400.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/267"", ""tyndp2020_proj_id""=>""267"", ""tyndp2020_invest_id""=>""1262"", ""tyndp_status""=>""under_consideration""",13.580475,55.86067,12.204437,53.760078 +TYNDP2020_15,planned_not_yet_permitting,230.0,2030,False,1500.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/270"", ""tyndp2020_proj_id""=>""270"", ""tyndp2020_invest_id""=>""1211"", ""tyndp_status""=>""planned_not_yet_permitting""",-1.671123,41.952398,-0.575409000000004,43.345155 +TYNDP2020_16,under_consideration,200.0,2040,False,1000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1058"", ""tyndp2020_proj_id""=>""1058"", ""tyndp2020_invest_id""=>""1726"", ""tyndp_status""=>""under_consideration""",9.43585963014029,48.6383575221442,8.298798,47.148634 +TYNDP2020_17,in_permitting,500.0,2026,True,700.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/107"", ""tyndp2020_proj_id""=>""107"", ""tyndp2020_invest_id""=>""810"", ""tyndp_status""=>""in_permitting""",-8.36471600000001,51.892597,-4.237976,48.518424 +TYNDP2020_19,planned_not_yet_permitting,225.0,2029,False,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/276"", ""tyndp2020_proj_id""=>""276"", ""tyndp2020_invest_id""=>""1206"", ""tyndp_status""=>""planned_not_yet_permitting""",-1.645402,42.818157,-0.812988000000009,43.979969 +TYNDP2020_20,under_consideration,195.0,2036,True,500.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1068"", ""tyndp2020_proj_id""=>""1068"", ""tyndp2020_invest_id""=>""1740"", ""tyndp_status""=>""under_consideration""",18.227692,57.571097,21.100616,56.489036 +TYNDP2020_21,planned_not_yet_permitting,156.0,2022,True,1500.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/285"", ""tyndp2020_proj_id""=>""285"", ""tyndp2020_invest_id""=>""1383"", ""tyndp_status""=>""planned_not_yet_permitting""",0.575409000000004,51.430896,2.16293300000001,50.923813 +TYNDP2020_22,in_permitting,540.0,2025,True,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/130"", ""tyndp2020_proj_id""=>""130"", ""tyndp2020_invest_id""=>""665"", ""tyndp_status""=>""in_permitting""",11.637268,52.27404,12.292328,48.659222 +TYNDP2020_23,in_permitting,300.0,2025,False,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/132"", ""tyndp2020_proj_id""=>""132"", ""tyndp2020_invest_id""=>""661"", ""tyndp_status""=>""in_permitting""",7.23587000000001,53.374317,6.58493,51.245584 +TYNDP2020_24,under_consideration,110.0,2036,False,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1077"", ""tyndp2020_proj_id""=>""1077"", ""tyndp2020_invest_id""=>""1747"", ""tyndp_status""=>""under_consideration""",22.9422,40.662931,22.111359,41.50035 +TYNDP2020_25,under_consideration,170.0,2036,False,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1077"", ""tyndp2020_proj_id""=>""1077"", ""tyndp2020_invest_id""=>""1748"", ""tyndp_status""=>""under_consideration""",22.111359,41.50035,23.03833,42.549034 +TYNDP2020_26,under_consideration,180.0,2036,False,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1081"", ""tyndp2020_proj_id""=>""1081"", ""tyndp2020_invest_id""=>""1749"", ""tyndp_status""=>""under_consideration""",21.137695,39.279042,19.966278,41.16108 +TYNDP2020_27,under_consideration,665.0,2026,True,1800.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/296"", ""tyndp2020_proj_id""=>""296"", ""tyndp2020_invest_id""=>""1437"", ""tyndp_status""=>""under_consideration""",-1.972046,43.32318,-1.847076,47.30531 +TYNDP2020_28,under_consideration,665.0,2026,True,1800.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/296"", ""tyndp2020_proj_id""=>""296"", ""tyndp2020_invest_id""=>""1437"", ""tyndp_status""=>""under_consideration""",-1.847076,47.30531,-3.967438,50.489842 +TYNDP2020_29,in_permitting,100.0,2024,True,400.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/299"", ""tyndp2020_proj_id""=>""299"", ""tyndp2020_invest_id""=>""1458"", ""tyndp_status""=>""in_permitting""",8.63662699999999,40.644178,9.45098899999999,42.538916 +TYNDP2020_30,in_permitting,50.0,2024,False,400.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/299"", ""tyndp2020_proj_id""=>""299"", ""tyndp2020_invest_id""=>""1458"", ""tyndp_status""=>""in_permitting""",9.45098899999999,42.538916,10.671844,43.014689 +TYNDP2020_31,in_permitting,170.0,2028,True,1000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/150"", ""tyndp2020_proj_id""=>""150"", ""tyndp2020_invest_id""=>""616"", ""tyndp_status""=>""in_permitting""",12.516174,45.825928,14.615936,46.084662 +TYNDP2020_32,in_permitting,700.0,2024,True,1400.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/309"", ""tyndp2020_proj_id""=>""309"", ""tyndp2020_invest_id""=>""1628"", ""tyndp_status""=>""in_permitting""",0.763549999999995,51.403489,8.00079299999999,53.55581 +TYNDP2020_33,in_permitting,400.0,2026,True,750.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/170"", ""tyndp2020_proj_id""=>""170"", ""tyndp2020_invest_id""=>""1034"", ""tyndp_status""=>""in_permitting""",21.295624,56.032924,18.104095,54.714309 +TYNDP2020_34,under_consideration,328.0,2035,True,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/335"", ""tyndp2020_proj_id""=>""335"", ""tyndp2020_invest_id""=>""1505"", ""tyndp_status""=>""under_consideration""",8.66683999999999,55.481966,3.590555,55.343106 +TYNDP2020_35,under_consideration,360.0,2035,True,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/335"", ""tyndp2020_proj_id""=>""335"", ""tyndp2020_invest_id""=>""1506"", ""tyndp_status""=>""under_consideration""",4.05120799999999,51.942572,3.590555,55.343106 +TYNDP2020_36,under_construction,69.0,2021,True,1000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/172"", ""tyndp2020_proj_id""=>""172"", ""tyndp2020_invest_id""=>""1487"", ""tyndp_status""=>""under_construction""",0.95855700000001,51.059523,1.833344,50.902167 +TYNDP2020_37,in_permitting,165.0,2024,True,1000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/174"", ""tyndp2020_proj_id""=>""174"", ""tyndp2020_invest_id""=>""1014"", ""tyndp_status""=>""in_permitting""",9.449615,45.620761,9.253235,46.824496 +TYNDP2020_38,in_permitting,300.0,2026,True,700.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/176"", ""tyndp2020_proj_id""=>""176"", ""tyndp2020_invest_id""=>""995"", ""tyndp_status""=>""in_permitting""",13.580475,55.86067,12.204437,53.760078 +TYNDP2020_39,under_consideration,170.0,2030,True,600.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/179"", ""tyndp2020_proj_id""=>""179"", ""tyndp2020_invest_id""=>""1016"", ""tyndp_status""=>""under_consideration""",11.969604,55.468735,12.271729,54.068253 +TYNDP2020_40,under_consideration,290.0,2035,True,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/335"", ""tyndp2020_proj_id""=>""335"", ""tyndp2020_invest_id""=>""1507"", ""tyndp_status""=>""under_consideration""",6.86370800000001,53.418536,3.590555,55.343106 +TYNDP2020_41,under_consideration,390.0,2035,True,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/335"", ""tyndp2020_proj_id""=>""335"", ""tyndp2020_invest_id""=>""1508"", ""tyndp_status""=>""under_consideration""",9.24087500000001,53.906765,3.590555,55.343106 +TYNDP2020_42,under_consideration,400.0,2035,True,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/335"", ""tyndp2020_proj_id""=>""335"", ""tyndp2020_invest_id""=>""1509"", ""tyndp_status""=>""under_consideration""",8.418274,53.182996,3.590555,55.343106 +TYNDP2020_43,under_consideration,490.0,2035,True,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/335"", ""tyndp2020_proj_id""=>""335"", ""tyndp2020_invest_id""=>""1511"", ""tyndp_status""=>""under_consideration""",10.458984,53.435719,3.590555,55.343106 +TYNDP2020_44,planned_not_yet_permitting,400.0,2030,True,1000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/338"", ""tyndp2020_proj_id""=>""338"", ""tyndp2020_invest_id""=>""1521"", ""tyndp_status""=>""planned_not_yet_permitting""",14.028168,42.409263,12.811432,43.915702 +TYNDP2020_45,planned_not_yet_permitting,400.0,2025,True,1000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/339"", ""tyndp2020_proj_id""=>""339"", ""tyndp2020_invest_id""=>""1557"", ""tyndp_status""=>""planned_not_yet_permitting""",9.18319700000001,39.374649,13.304443,38.159396 +TYNDP2020_46,planned_not_yet_permitting,500.0,2025,True,1000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/339"", ""tyndp2020_proj_id""=>""339"", ""tyndp2020_invest_id""=>""1557"", ""tyndp_status""=>""planned_not_yet_permitting""",9.18319700000001,39.374649,14.639282,40.688969 +TYNDP2020_47,under_consideration,150.0,2035,False,1000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/225"", ""tyndp2020_proj_id""=>""225"", ""tyndp2020_invest_id""=>""1107"", ""tyndp_status""=>""under_consideration""",5.35583499999999,50.557942,6.568451,50.403266 +TYNDP2020_48,planned_not_yet_permitting,211.0,2025,False,750.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/349"", ""tyndp2020_proj_id""=>""349"", ""tyndp2020_invest_id""=>""1638"", ""tyndp_status""=>""planned_not_yet_permitting""",-3.474151,53.250084,-6.602175,53.348929 +TYNDP2020_49,planned_not_yet_permitting,255.0,2025,False,750.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/349"", ""tyndp2020_proj_id""=>""349"", ""tyndp2020_invest_id""=>""1640"", ""tyndp_status""=>""planned_not_yet_permitting""",-6.602175,53.348929,-9.578832,54.120411 diff --git a/data/transmission_projects/tyndp2020/upgraded_lines.csv b/data/transmission_projects/tyndp2020/upgraded_lines.csv new file mode 100644 index 00000000..ecf07b6a --- /dev/null +++ b/data/transmission_projects/tyndp2020/upgraded_lines.csv @@ -0,0 +1,27 @@ +,project_status,length,build_year,underground,v_nom,tags,x0,y0,x1,y1,num_parallel,type +6031,in_permitting,32.0,2024,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/103"", ""tyndp2020_proj_id""=>""103"", ""tyndp2020_invest_id""=>""1490"", ""tyndp_status""=>""in_permitting""",5.763702,52.619725,6.14410399999999,52.52207,2,Al/St 240/40 4-bundle 380.0 +5123,in_permitting,35.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/103"", ""tyndp2020_proj_id""=>""103"", ""tyndp2020_invest_id""=>""1539"", ""tyndp_status""=>""in_permitting""",4.592285,51.916321,4.83398400000001,51.704055,2,Al/St 240/40 4-bundle 380.0 +8435,planned_not_yet_permitting,41.5,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/170"", ""tyndp2020_proj_id""=>""170"", ""tyndp2020_invest_id""=>""1663"", ""tyndp_status""=>""planned_not_yet_permitting""",14.440155,53.16571,14.831543,53.330873,2,Al/St 240/40 4-bundle 380.0 +12697,in_permitting,70.0,2021,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/230"", ""tyndp2020_proj_id""=>""230"", ""tyndp2020_invest_id""=>""353"", ""tyndp_status""=>""in_permitting""",14.440155,53.16571,15.038505,52.76751,2,Al/St 240/40 4-bundle 380.0 +14734,under_construction,49.0,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/264"", ""tyndp2020_proj_id""=>""264"", ""tyndp2020_invest_id""=>""1284"", ""tyndp_status""=>""under_construction""",10.362854,46.809459,9.79568500000001,46.561693,2,Al/St 240/40 4-bundle 380.0 +6348,under_construction,17.0,2020,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/297"", ""tyndp2020_proj_id""=>""297"", ""tyndp2020_invest_id""=>""445"", ""tyndp_status""=>""under_construction""",4.26544200000001,51.358062,4.307132,51.301299,2,Al/St 240/40 4-bundle 380.0 +6347,in_permitting,4.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/297"", ""tyndp2020_proj_id""=>""297"", ""tyndp2020_invest_id""=>""604"", ""tyndp_status""=>""in_permitting""",4.307132,51.301299,4.27917500000001,51.25504,2,Al/St 240/40 4-bundle 380.0 +10813,in_permitting,10.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/322"", ""tyndp2020_proj_id""=>""322"", ""tyndp2020_invest_id""=>""1476"", ""tyndp_status""=>""in_permitting""",9.73251299999999,47.623752,9.80228979023356,47.6377345779987,2,Al/St 240/40 4-bundle 380.0 +14499,under_consideration,2.0,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/341"", ""tyndp2020_proj_id""=>""341"", ""tyndp2020_invest_id""=>""1536"", ""tyndp_status""=>""under_consideration""",22.578278,44.679395,22.542572,44.612956,2,Al/St 240/40 4-bundle 380.0 +9296,planned_not_yet_permitting,203.0,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/343"", ""tyndp2020_proj_id""=>""343"", ""tyndp2020_invest_id""=>""1534"", ""tyndp_status""=>""planned_not_yet_permitting""",15.26017,44.886816,16.500092,43.603267,2,Al/St 240/40 4-bundle 380.0 +6044,in_permitting,80.0,2029,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/346"", ""tyndp2020_proj_id""=>""346"", ""tyndp2020_invest_id""=>""1544"", ""tyndp_status""=>""in_permitting""",4.218787,51.424034,5.061511,51.608344,2,Al/St 240/40 4-bundle 380.0 +6087,under_construction,40.0,2023,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/348"", ""tyndp2020_proj_id""=>""348"", ""tyndp2020_invest_id""=>""1546"", ""tyndp_status""=>""under_construction""",6.86370800000001,53.418536,6.474759,53.213488,2,Al/St 240/40 4-bundle 380.0 +14697,under_consideration,35.512,2036,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/376"", ""tyndp2020_proj_id""=>""376"", ""tyndp2020_invest_id""=>""1559"", ""tyndp_status""=>""under_consideration""",21.474152,40.839788,21.475525,41.045181,2,Al/St 240/40 4-bundle 380.0 +14758,under_consideration,12.0,2030,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/377"", ""tyndp2020_proj_id""=>""377"", ""tyndp2020_invest_id""=>""1561"", ""tyndp_status""=>""under_consideration""",5.71701000000001,51.094036,5.89691199999999,51.140586,2,Al/St 240/40 4-bundle 380.0 +8621,in_permitting,131.0,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1"", ""tyndp2020_proj_id""=>""1"", ""tyndp2020_invest_id""=>""4"", ""tyndp_status""=>""in_permitting""",-7.72338899999999,41.614416,-7.793621146853301,41.52102394053344,2,Al/St 240/40 4-bundle 380.0 +6036,in_permitting,50.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/103"", ""tyndp2020_proj_id""=>""103"", ""tyndp2020_invest_id""=>""1540"", ""tyndp_status""=>""in_permitting""",5.46295200000001,51.422333,5.89691199999999,51.140586,2,Al/St 240/40 4-bundle 380.0 +3715,in_permitting,94.0,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/142"", ""tyndp2020_proj_id""=>""142"", ""tyndp2020_invest_id""=>""257"", ""tyndp_status""=>""in_permitting""",26.003265,42.200038,24.483032,42.09007,2,Al/St 240/40 4-bundle 380.0 +8260,in_permitting,74.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/144"", ""tyndp2020_proj_id""=>""144"", ""tyndp2020_invest_id""=>""270"", ""tyndp_status""=>""in_permitting""",21.88777,45.30413,21.2815,45.7426,2,Al/St 240/40 4-bundle 380.0 +8367,in_permitting,274.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/144"", ""tyndp2020_proj_id""=>""144"", ""tyndp2020_invest_id""=>""270"", ""tyndp_status""=>""in_permitting""",21.2815,45.7426,21.409607,46.176978,2,Al/St 240/40 4-bundle 380.0 +1693,in_permitting,,2026,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/227"", ""tyndp2020_proj_id""=>""227"", ""tyndp2020_invest_id""=>""630"", ""tyndp_status""=>""in_permitting""",20.187378,44.664746,19.500315,43.953021,2,Al/St 240/40 4-bundle 380.0 +12696,in_permitting,147.0,2022,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/230"", ""tyndp2020_proj_id""=>""230"", ""tyndp2020_invest_id""=>""1232"", ""tyndp_status""=>""in_permitting""",15.038505,52.76751,16.729431,52.32359,2,Al/St 240/40 4-bundle 380.0 +3966,planned_not_yet_permitting,70.0,2029,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/276"", ""tyndp2020_proj_id""=>""276"", ""tyndp2020_invest_id""=>""1207"", ""tyndp_status""=>""planned_not_yet_permitting""",-0.812988000000009,43.979969,-0.547943,44.563077,2,Al/St 240/40 4-bundle 380.0 +14277,in_permitting,15.0,2025,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/297"", ""tyndp2020_proj_id""=>""297"", ""tyndp2020_invest_id""=>""604"", ""tyndp_status""=>""in_permitting""",4.27917500000001,51.25504,4.177551,51.162122,2,Al/St 240/40 4-bundle 380.0 +14594,under_consideration,70.0,2034,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/325"", ""tyndp2020_proj_id""=>""325"", ""tyndp2020_invest_id""=>""1483"", ""tyndp_status""=>""under_consideration""",14.5912685597917,46.536256583087,15.119934,46.263443,2,Al/St 240/40 4-bundle 380.0 +12655,under_construction,161.0,2020,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/350"", ""tyndp2020_proj_id""=>""350"", ""tyndp2020_invest_id""=>""1622"", ""tyndp_status""=>""under_construction""",21.475525,41.045181,20.725963,41.239159,2,Al/St 240/40 4-bundle 380.0 +6355,under_consideration,90.0,2036,False,380,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1074"", ""tyndp2020_proj_id""=>""1074"", ""tyndp2020_invest_id""=>""1744"", ""tyndp_status""=>""under_consideration""",19.756165,45.527517,19.609222,45.021127,2,Al/St 240/40 4-bundle 380.0 diff --git a/data/transmission_projects/tyndp2020/upgraded_links.csv b/data/transmission_projects/tyndp2020/upgraded_links.csv new file mode 100644 index 00000000..5fe56694 --- /dev/null +++ b/data/transmission_projects/tyndp2020/upgraded_links.csv @@ -0,0 +1,8 @@ +,project_status,length,build_year,underground,p_nom,tags,x0,y0,x1,y1 +T0,in_permitting,370.0,2027,True,2000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/16"", ""tyndp2020_proj_id""=>""16"", ""tyndp2020_invest_id""=>""38"", ""tyndp_status""=>""in_permitting""",-2.880286,43.349776,-0.341949000000001,45.030833 +14801,under_construction,90.0,2020,True,1000.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/92"", ""tyndp2020_proj_id""=>""92"", ""tyndp2020_invest_id""=>""146"", ""tyndp_status""=>""under_construction""",6.46545399999999,50.831096,5.688171,50.729502 +T17,in_permitting,665.0,2024,True,1400.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/190"", ""tyndp2020_proj_id""=>""190"", ""tyndp2020_invest_id""=>""1382"", ""tyndp_status""=>""in_permitting""",7.23331357825616,60.5160767478361,-1.815491,57.484833 +T25,planned_not_yet_permitting,195.0,2023,True,500.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/286"", ""tyndp2020_proj_id""=>""286"", ""tyndp2020_invest_id""=>""1385"", ""tyndp_status""=>""planned_not_yet_permitting""",-6.96670499999999,52.27404,-4.850464,51.728729 +T12,in_permitting,220.0,2025,True,1400.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/153"", ""tyndp2020_proj_id""=>""153"", ""tyndp2020_invest_id""=>""987"", ""tyndp_status""=>""in_permitting""",-1.47628799999999,49.460984,-3.357697,50.739062 +T13,under_construction,770.0,2023,True,1400.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/167"", ""tyndp2020_proj_id""=>""167"", ""tyndp2020_invest_id""=>""998"", ""tyndp_status""=>""under_construction""",9.22851599999999,55.481188,-0.208961173685554,52.9600676434916 +5635,under_consideration,125.0,2036,True,700.0,"""url""=>""https://tyndp2020-project-platform.azurewebsites.net/projectsheets/transmission/1068"", ""tyndp2020_proj_id""=>""1068"", ""tyndp2020_invest_id""=>""1741"", ""tyndp_status""=>""under_consideration""",16.659393,57.43682,18.227692,57.571097 diff --git a/doc/configtables/links.csv b/doc/configtables/links.csv index c1ffb427..9dedc36a 100644 --- a/doc/configtables/links.csv +++ b/doc/configtables/links.csv @@ -2,5 +2,4 @@ p_max_pu,--,"Value in [0.,1.]","Correction factor for link capacities ``p_nom``." p_nom_max,MW,"float","Global upper limit for the maximum capacity of each extendable DC link." max_extension,MW,"float","Upper limit for the extended capacity of each extendable DC link." -include_tyndp,bool,"{'true', 'false'}","Specifies whether to add HVDC link projects from the `TYNDP 2018 `_ which are at least in permitting." under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}","Specifies how to handle lines which are currently under construction." diff --git a/doc/configtables/transmission_projects.csv b/doc/configtables/transmission_projects.csv new file mode 100644 index 00000000..acba4842 --- /dev/null +++ b/doc/configtables/transmission_projects.csv @@ -0,0 +1,9 @@ +,Unit,Values,Description +enable,bool,"{true,false}",Whether to integrate this transmission projects or not. +include,--,,"Name of the transmission projects. They should be unique and have to be provided in the `data/transmission_projects` folder." +-- tyndp2020,bool,"{true,false}",Whether to integrate the TYNDP 2020 dataset. +-- nep,bool,"{true,false}",Whether to integrate the German network development plan dataset. +-- manual,bool,"{true,false}",Whether to integrate the manually added transmission projects. They are taken from the previously existing links_tyndp.csv file. +skip,list,,"Type of lines to skip from all transmission projects. Possible values are: ``upgraded_lines``, ``upgraded_links``, ``new_lines``, ``new_links``." +status,list or dict,,"Status to include into the model as list or as dict with name of project and status to include. Possible values for status are ``under_construction``, ``in_permitting``, ``confirmed``, ``planned_not_yet_permitted``, ``under_consideration``." +new_link_capacity,--,"{zero,keep}",Whether to set the new link capacity to the provided capacity or set it to zero. diff --git a/doc/configuration.rst b/doc/configuration.rst index 8695d4bb..e140261a 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -379,6 +379,23 @@ overwrite the existing values. .. _transformers_cf: +``transmission projects`` +======================= + +Allows to define additional transmission projects that will be added to the base network, e.g., from the TYNDP 2020 dataset. The projects are read in from the CSV files in the subfolder of ``data/transmission_projects/``. New transmission projects, e.g. from TYNDP 2024, can be added in a new subfolder of transmission projects, e.g. ``data/transmission_projects/tyndp2024`` while extending the list of ``transmission_projects`` in the ``config.yaml`` by ``tyndp2024``. The CSV files in the project folder should have the same columns as the CSV files in the template folder ``data/transmission_projects/template``. + +.. literalinclude:: ../config/config.default.yaml + :language: yaml + :start-at: transmission_projects: + :end-before: # docs + +.. csv-table:: + :header-rows: 1 + :widths: 22,7,22,33 + :file: configtables/transmission_projects.csv + +.. _transformers_cf: + ``transformers`` ================ diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 274231c6..ac8c2559 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -9,6 +9,7 @@ Release Notes Upcoming Release ================ +* More modular and flexible handling of transmission projects. One can now add new transmission projects in a subfolder of `data/transmission projects` similar to the files in the template folder. After adding the new files and updating the config section `transmission_projects:`, transmission projects will be included if they are not duplicates of existing lines or other projects. * Add option to apply a gaussian kernel density smoothing to wind turbine power curves. diff --git a/envs/environment.fixed.yaml b/envs/environment.fixed.yaml index 67fff66f..917deab7 100644 --- a/envs/environment.fixed.yaml +++ b/envs/environment.fixed.yaml @@ -335,7 +335,7 @@ dependencies: - pyomo=6.6.1 - pyparsing=3.1.2 - pyproj=3.6.1 -- pypsa=0.28.0 +- pypsa=0.29.0 - pyqt=5.15.9 - pyqt5-sip=12.12.2 - pyscipopt=5.0.1 diff --git a/envs/environment.yaml b/envs/environment.yaml index c8d8a633..c87de77a 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -11,7 +11,7 @@ dependencies: - pip - atlite>=0.2.9 -- pypsa>=0.28 +- pypsa>=0.29 - linopy - dask diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 97e1e16a..f3505091 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -56,7 +56,6 @@ rule base_network: snapshots=config_provider("snapshots"), drop_leap_day=config_provider("enable", "drop_leap_day"), lines=config_provider("lines"), - links=config_provider("links"), transformers=config_provider("transformers"), input: eg_buses="data/entsoegridkit/buses.csv", @@ -66,7 +65,6 @@ rule base_network: eg_transformers="data/entsoegridkit/transformers.csv", parameter_corrections="data/parameter_corrections.yaml", links_p_nom="data/links_p_nom.csv", - links_tyndp="data/links_tyndp.csv", country_shapes=resources("country_shapes.geojson"), offshore_shapes=resources("offshore_shapes.geojson"), europe_shape=resources("europe_shape.geojson"), @@ -342,6 +340,40 @@ rule build_line_rating: "../scripts/build_line_rating.py" +rule build_transmission_projects: + params: + transmission_projects=config_provider("transmission_projects"), + line_factor=config_provider("lines", "length_factor"), + input: + base_network=resources("networks/base.nc"), + offshore_shapes=resources("offshore_shapes.geojson"), + europe_shape=resources("europe_shape.geojson"), + transmission_projects=lambda w: [ + "data/transmission_projects/" + name + for name, include in config_provider("transmission_projects", "include")( + w + ).items() + if include + ], + output: + new_lines=resources("transmission_projects/new_lines.csv"), + new_links=resources("transmission_projects/new_links.csv"), + adjust_lines=resources("transmission_projects/adjust_lines.csv"), + adjust_links=resources("transmission_projects/adjust_links.csv"), + new_buses=resources("transmission_projects/new_buses.csv"), + log: + logs("build_transmission_projects.log"), + benchmark: + benchmarks("build_transmission_projects") + resources: + mem_mb=2000, + threads: 1 + conda: + "../envs/environment.yaml" + script: + "../scripts/build_transmission_projects.py" + + def input_profile_tech(w): return { f"profile_{tech}": resources(f"profile_{tech}.nc") @@ -402,6 +434,7 @@ rule add_electricity: costs=config_provider("costs"), foresight=config_provider("foresight"), drop_leap_day=config_provider("enable", "drop_leap_day"), + transmission_projects=config_provider("transmission_projects"), input: unpack(input_profile_tech), unpack(input_conventional), @@ -412,6 +445,17 @@ rule add_electricity: if config_provider("lines", "dynamic_line_rating", "activate")(w) else resources("networks/base.nc") ), + transmission_projects=lambda w: ( + [ + resources("transmission_projects/new_buses.csv"), + resources("transmission_projects/new_lines.csv"), + resources("transmission_projects/new_links.csv"), + resources("transmission_projects/adjust_lines.csv"), + resources("transmission_projects/adjust_links.csv"), + ] + if config_provider("transmission_projects", "enable")(w) + else [] + ), tech_costs=lambda w: resources( f"costs_{config_provider('costs', 'year')(w)}.csv" ), diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 49510953..076eb84e 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -84,6 +84,7 @@ It further adds extendable ``generators`` with **zero** capacity for import logging from itertools import product +from pathlib import Path from typing import Dict, List import geopandas as gpd @@ -799,6 +800,25 @@ def attach_line_rating( n.lines_t.s_max_pu *= s_max_pu +def add_transmission_projects(n, transmission_projects): + logger.info(f"Adding transmission projects to network.") + for path in transmission_projects: + path = Path(path) + df = pd.read_csv(path, index_col=0, dtype={"bus0": str, "bus1": str}) + if df.empty: + continue + if "new_buses" in path.name: + n.madd("Bus", df.index, **df) + elif "new_lines" in path.name: + n.madd("Line", df.index, **df) + elif "new_links" in path.name: + n.madd("Link", df.index, **df) + elif "adjust_lines": + n.lines.update(df) + elif "adjust_links": + n.links.update(df) + + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -811,6 +831,9 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.base_network) + if params["transmission_projects"]["enable"]: + add_transmission_projects(n, snakemake.input.transmission_projects) + time = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) n.set_snapshots(time) diff --git a/scripts/base_network.py b/scripts/base_network.py index 8172b332..78cf6984 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -25,7 +25,6 @@ Relevant Settings links: p_max_pu: under_construction: - include_tyndp: transformers: x: @@ -43,7 +42,6 @@ Inputs - ``data/entsoegridkit``: Extract from the geographical vector data of the online `ENTSO-E Interactive Map `_ by the `GridKit `_ toolkit dating back to March 2022. - ``data/parameter_corrections.yaml``: Corrections for ``data/entsoegridkit`` - ``data/links_p_nom.csv``: confer :ref:`links` -- ``data/links_tyndp.csv``: List of projects in the `TYNDP 2018 `_ that are at least *in permitting* with fields for start- and endpoint (names and coordinates), length, capacity, construction status, and project reference ID. - ``resources/country_shapes.geojson``: confer :ref:`shapes` - ``resources/offshore_shapes.geojson``: confer :ref:`shapes` - ``resources/europe_shape.geojson``: confer :ref:`shapes` @@ -222,112 +220,6 @@ def _load_links_from_eg(buses, eg_links): return links -def _add_links_from_tyndp(buses, links, links_tyndp, europe_shape): - links_tyndp = pd.read_csv(links_tyndp) - - # remove all links from list which lie outside all of the desired countries - europe_shape = gpd.read_file(europe_shape).loc[0, "geometry"] - europe_shape_prepped = shapely.prepared.prep(europe_shape) - x1y1_in_europe_b = links_tyndp[["x1", "y1"]].apply( - lambda p: europe_shape_prepped.contains(Point(p)), axis=1 - ) - x2y2_in_europe_b = links_tyndp[["x2", "y2"]].apply( - lambda p: europe_shape_prepped.contains(Point(p)), axis=1 - ) - is_within_covered_countries_b = x1y1_in_europe_b & x2y2_in_europe_b - - if not is_within_covered_countries_b.all(): - logger.info( - "TYNDP links outside of the covered area (skipping): " - + ", ".join(links_tyndp.loc[~is_within_covered_countries_b, "Name"]) - ) - - links_tyndp = links_tyndp.loc[is_within_covered_countries_b] - if links_tyndp.empty: - return buses, links - - has_replaces_b = links_tyndp.replaces.notnull() - oids = dict(Bus=_get_oid(buses), Link=_get_oid(links)) - keep_b = dict( - Bus=pd.Series(True, index=buses.index), Link=pd.Series(True, index=links.index) - ) - for reps in links_tyndp.loc[has_replaces_b, "replaces"]: - for comps in reps.split(":"): - oids_to_remove = comps.split(".") - c = oids_to_remove.pop(0) - keep_b[c] &= ~oids[c].isin(oids_to_remove) - buses = buses.loc[keep_b["Bus"]] - links = links.loc[keep_b["Link"]] - - links_tyndp["j"] = _find_closest_links( - links, links_tyndp, distance_upper_bound=0.20 - ) - # Corresponds approximately to 20km tolerances - - if links_tyndp["j"].notnull().any(): - logger.info( - "TYNDP links already in the dataset (skipping): " - + ", ".join(links_tyndp.loc[links_tyndp["j"].notnull(), "Name"]) - ) - links_tyndp = links_tyndp.loc[links_tyndp["j"].isnull()] - if links_tyndp.empty: - return buses, links - - tree_buses = buses.query("carrier=='AC'") - tree = KDTree(tree_buses[["x", "y"]]) - _, ind0 = tree.query(links_tyndp[["x1", "y1"]]) - ind0_b = ind0 < len(tree_buses) - links_tyndp.loc[ind0_b, "bus0"] = tree_buses.index[ind0[ind0_b]] - - _, ind1 = tree.query(links_tyndp[["x2", "y2"]]) - ind1_b = ind1 < len(tree_buses) - links_tyndp.loc[ind1_b, "bus1"] = tree_buses.index[ind1[ind1_b]] - - links_tyndp_located_b = ( - links_tyndp["bus0"].notnull() & links_tyndp["bus1"].notnull() - ) - if not links_tyndp_located_b.all(): - logger.warning( - "Did not find connected buses for TYNDP links (skipping): " - + ", ".join(links_tyndp.loc[~links_tyndp_located_b, "Name"]) - ) - links_tyndp = links_tyndp.loc[links_tyndp_located_b] - - logger.info("Adding the following TYNDP links: " + ", ".join(links_tyndp["Name"])) - - links_tyndp = links_tyndp[["bus0", "bus1"]].assign( - carrier="DC", - p_nom=links_tyndp["Power (MW)"], - length=links_tyndp["Length (given) (km)"].fillna( - links_tyndp["Length (distance*1.2) (km)"] - ), - under_construction=True, - underground=False, - geometry=( - links_tyndp[["x1", "y1", "x2", "y2"]].apply( - lambda s: str(LineString([[s.x1, s.y1], [s.x2, s.y2]])), axis=1 - ) - ), - tags=( - '"name"=>"' - + links_tyndp["Name"] - + '", ' - + '"ref"=>"' - + links_tyndp["Ref"] - + '", ' - + '"status"=>"' - + links_tyndp["status"] - + '"' - ), - ) - - links_tyndp.index = "T" + links_tyndp.index.astype(str) - - links = pd.concat([links, links_tyndp], sort=True) - - return buses, links - - def _load_lines_from_eg(buses, eg_lines): lines = ( pd.read_csv( @@ -731,7 +623,6 @@ def base_network( eg_lines, eg_links, links_p_nom, - links_tyndp, europe_shape, country_shapes, offshore_shapes, @@ -741,8 +632,6 @@ def base_network( buses = _load_buses_from_eg(eg_buses, europe_shape, config["electricity"]) links = _load_links_from_eg(buses, eg_links) - if config["links"].get("include_tyndp"): - buses, links = _add_links_from_tyndp(buses, links, links_tyndp, europe_shape) converters = _load_converters_from_eg(buses, eg_converters) @@ -911,7 +800,6 @@ if __name__ == "__main__": snakemake.input.eg_lines, snakemake.input.eg_links, snakemake.input.links_p_nom, - snakemake.input.links_tyndp, snakemake.input.europe_shape, snakemake.input.country_shapes, snakemake.input.offshore_shapes, diff --git a/scripts/build_transmission_projects.py b/scripts/build_transmission_projects.py new file mode 100644 index 00000000..ba185d17 --- /dev/null +++ b/scripts/build_transmission_projects.py @@ -0,0 +1,564 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +# coding: utf-8 +""" +Gets the transmission projects defined in the config file, concatenates and +deduplicates them. Projects are later included in :mod:`add_electricity.py`. + +Relevant Settings +----------------- + +.. code:: yaml + +transmission_projects: + include: + #tyndp: true # For later, when other TYNDP projects are combined with new version + nep: true + status: + - confirmed + - in_permitting + - under_construction + #- under_consideration + link_under_construction: zero + +.. seealso:: + Documentation of the configuration file ``config/config.yaml`` at + :ref:`transmission_projects` +Inputs +------ + +- ``networks/base_network.nc``: Base network topology for the electricity grid. This is processed in :mod:`base_network.py`. +- ``data/transmission_projects/"project_name"/``: Takes the transmission projects from the subfolder of data/transmission_projects. The subfolder name is the project name. +- ``offshore_shapes.geojson``: Shapefile containing the offshore regions. Used to determine if a new bus should be added for a new line or link. +- ``europe_shape.geojson``: Shapefile containing the shape of Europe. Used to determine if a project is within the considered countries. + +Outputs +------- + +- ``transmission_projects/new_lines.csv``: New project lines to be added to the network. This includes new lines and upgraded lines. +- ``transmission_projects/new_links.csv``: New project links to be added to the network. This includes new links and upgraded links. +- ``transmission_projects/adjust_lines.csv``: For lines which are upgraded, the decommissioning year of the existing line is adjusted to the build year of the upgraded line. +- ``transmission_projects/adjust_links.csv``: For links which are upgraded, the decommissioning year of the existing link is adjusted to the build year of the upgraded link. +- ``transmission_projects/new_buses.csv``: For some links, we have to add new buses (e.g. North Sea Wind Power Hub). +""" +import logging +import os +from pathlib import Path + +import geopandas as gpd +import numpy as np +import pandas as pd +import pypsa +import shapely +from _helpers import configure_logging, set_scenario_config +from pypsa.descriptors import nominal_attrs +from scipy import spatial +from shapely.geometry import LineString, Point + +logger = logging.getLogger(__name__) + + +def add_new_buses(n, new_ports): + # Add new buses for the ports which do not have an existing bus close by. If there are multiple ports at the same location, only one bus is added. + duplicated = new_ports.duplicated(subset=["x", "y"], keep="first") + to_add = new_ports[~duplicated] + added_buses = n.madd( + "Bus", + names=to_add.index, + suffix=" bus", + x=to_add.x, + y=to_add.y, + v_nom=380, + under_construction=True, + symbol="substation", + substation_off=True, + substation_lv=False, + carrier="AC", + ) + new_buses = n.buses.loc[added_buses].copy().dropna(axis=1, how="all") + new_ports.loc[to_add.index, "neighbor"] = added_buses + new_ports["neighbor"] = new_ports.groupby(["x", "y"])["neighbor"].transform("first") + return new_ports, new_buses + + +def find_country_for_bus(bus, shapes): + """ + Find the country of a bus based on its coordinates and the provided + shapefile. + + Shapefile must contain a column "country" with the country names. + """ + point = Point(bus.x, bus.y) + country = shapes.loc[shapes.contains(point), "country"] + return country.values[0] + + +def connect_new_lines( + lines, + n, + new_buses_df, + offshore_shapes=None, + distance_upper_bound=np.inf, + bus_carrier="AC", +): + """ + Find the closest existing bus to the port of each line. + + If closest bus is further away than distance_upper_bound and is + inside an offshore region, a new bus is created. and the line is + connected to it. + """ + bus_carrier = np.atleast_1d(bus_carrier) + buses = n.buses.query("carrier in @bus_carrier").copy() + bus_tree = spatial.KDTree(buses[["x", "y"]]) + + for port in [0, 1]: + lines_port = lines["geometry"].apply( + lambda x: pd.Series( + get_bus_coords_from_port(x, port=port), index=["x", "y"] + ) + ) + distances, indices = bus_tree.query(lines_port) + # Series of lines with closest bus in the existing network and whether they match the distance criterion + lines_port["neighbor"] = buses.iloc[indices].index + lines_port["match_distance"] = distances < distance_upper_bound + # For buses which are not close to any existing bus, only add a new bus if the line is going offshore (e.g. North Sea Wind Power Hub) + if not lines_port.match_distance.all() and offshore_shapes.union_all(): + potential_new_buses = lines_port[~lines_port.match_distance] + is_offshore = potential_new_buses.apply( + lambda x: offshore_shapes.union_all().contains(Point(x.x, x.y)), axis=1 + ) + new_buses = potential_new_buses[is_offshore] + if not new_buses.empty: + new_port, new_buses = add_new_buses(n, new_buses) + new_buses["country"] = new_buses.apply( + lambda bus: find_country_for_bus(bus, offshore_shapes), axis=1 + ) + lines_port.loc[new_port.index, "match_distance"] = True + lines_port.loc[new_port.index, "neighbor"] = new_port.neighbor + new_buses_df = pd.concat([new_buses_df, new_buses]) + + if not lines_port.match_distance.all(): + logging.warning( + "Could not find bus close enough to connect the the following lines:\n" + + str(lines_port[~lines_port.match_distance].index.to_list()) + + "\n Lines will be ignored." + ) + lines.drop(lines_port[~lines_port.match_distance].index, inplace=True) + lines_port = lines_port[lines_port.match_distance] + + lines.loc[lines_port.index, f"bus{port}"] = lines_port["neighbor"] + + lines = lines.assign(under_construction=True) + + return lines, new_buses_df + + +def get_branch_coords_from_geometry(linestring, reversed=False): + """ + Reduces a linestring to its start and end points. Used to simplify the + linestring which can have more than two points. + + Parameters: + linestring: Shapely linestring + reversed (bool, optional): If True, returns the end and start points instead of the start and end points. + Defaults to False. + + Returns: + numpy.ndarray: Flattened array of start and end coordinates. + """ + coords = np.asarray(linestring.coords) + ind = [0, -1] if not reversed else [-1, 0] + start_end_coords = coords[ind] + return start_end_coords.flatten() + + +def get_branch_coords_from_buses(line): + """ + Gets line string for branch component in an pypsa network. + + Parameters: + linestring: shapely linestring + reversed (bool, optional): If True, returns the end and start points instead of the start and end points. + Defaults to False. + + Returns: + numpy.ndarray: Flattened array of start and end coordinates. + """ + start_coords = n.buses.loc[line.bus0, ["x", "y"]].values + end_coords = n.buses.loc[line.bus1, ["x", "y"]].values + return np.array([start_coords, end_coords]).flatten() + + +def get_bus_coords_from_port(linestring, port=0): + """ + Extracts the coordinates of a specified port from a given linestring. + + Parameters: + linestring: The shapely linestring. + port (int): The index of the port to extract coordinates from. Default is 0. + + Returns: + tuple: The coordinates of the specified port as a tuple (x, y). + """ + coords = np.asarray(linestring.coords) + ind = [0, -1] + coords = coords[ind] + coords = coords[port] + return coords + + +def find_closest_lines(lines, new_lines, distance_upper_bound=0.1, type="new"): + """ + Find the closest lines in the existing set of lines to a set of new lines. + + Parameters: + lines (pandas.DataFrame): DataFrame of the existing lines. + new_lines (pandas.DataFrame): DataFrame with column geometry containing the new lines. + distance_upper_bound (float, optional): Maximum distance to consider a line as a match. Defaults to 0.1 which corresponds to approximately 15 km. + + Returns: + pandas.Series: Series containing with index the new lines and values providing closest existing line. + """ + + # get coordinates of start and end points of all lines, for new lines we need to check both directions + treelines = lines.apply(get_branch_coords_from_buses, axis=1) + querylines = pd.concat( + [ + new_lines["geometry"].apply(get_branch_coords_from_geometry), + new_lines["geometry"].apply(get_branch_coords_from_geometry, reversed=True), + ] + ) + treelines = np.vstack(treelines) + querylines = np.vstack(querylines) + tree = spatial.KDTree(treelines) + dist, ind = tree.query(querylines, distance_upper_bound=distance_upper_bound) + found_b = ind < len(lines) + # since the new lines are checked in both directions, we need to find the correct index of the new line + found_i = np.arange(len(querylines))[found_b] % len(new_lines) + # create a DataFrame with the distances, new line and its closest existing line + line_map = pd.DataFrame( + dict(D=dist[found_b], existing_line=lines.index[ind[found_b] % len(lines)]), + index=new_lines.index[found_i].rename("new_lines"), + ) + if type == "new": + if len(found_i) != 0: + logger.warning( + "Found new lines similar to existing lines:\n" + + str(line_map["existing_line"].to_dict()) + + "\n Lines are assumed to be duplicated and will be ignored." + ) + elif type == "upgraded": + if len(found_i) < len(new_lines): + not_found = new_lines.index.difference(line_map.index) + logger.warning( + "Could not find upgraded lines close enough to existing lines:\n" + + str(not_found.to_list()) + + "\n Lines will be ignored." + ) + # only keep the closer line of the new line pair (since lines are checked in both directions) + line_map = line_map.sort_values(by="D")[ + lambda ds: ~ds.index.duplicated(keep="first") + ].sort_index()["existing_line"] + return line_map + + +def adjust_decommissioning(upgraded_lines, line_map): + """ + Adjust the decommissioning year of the existing lines to the built year of + the upgraded lines. + """ + to_update = pd.DataFrame(index=line_map) + to_update["build_year"] = ( + 1990 # dummy build_year to make existing lines decommissioned when upgraded lines are built + ) + to_update["lifetime"] = ( + upgraded_lines.rename(line_map)["build_year"] - 1990 + ) # set lifetime to the difference between build year of upgraded line and existing line + return to_update + + +def get_upgraded_lines(branch_component, n, upgraded_lines, line_map): + """ + Get upgraded lines by merging information of existing line and upgraded + line. + """ + # get first the information of the existing lines which will be upgraded + lines_to_add = n.df(branch_component).loc[line_map].copy() + # get columns of upgraded lines which are not in existing lines + new_columns = upgraded_lines.columns.difference(lines_to_add.columns) + # rename upgraded lines to match existing lines + upgraded_lines = upgraded_lines.rename(line_map) + # set the same index names to be able to merge + upgraded_lines.index.name = lines_to_add.index.name + # merge upgraded lines with existing lines + lines_to_add.update(upgraded_lines) + # add column which was added in upgraded lines + lines_to_add = pd.concat([lines_to_add, upgraded_lines[new_columns]], axis=1) + # only consider columns of original upgraded lines and bus0 and bus1 + lines_to_add = lines_to_add.loc[:, ["bus0", "bus1", *upgraded_lines.columns]] + # set capacity of upgraded lines to capacity of existing lines + lines_to_add[nominal_attrs[branch_component]] = n.df(branch_component).loc[ + line_map, nominal_attrs[branch_component] + ] + # change index of new lines to avoid duplicates + lines_to_add.index = lines_to_add.index.astype(str) + "_upgraded" + return lines_to_add + + +def get_project_files(path, skip=[]): + path = Path(path) + lines = {} + files = [ + p + for p in path.iterdir() + if p.is_file() + and p.suffix == ".csv" + and not any(substring in p.name for substring in skip) + ] + if not files: + logger.warning(f"No projects found for {path.parent.name}") + return lines + for file in files: + df = pd.read_csv(file, index_col=0) + df["geometry"] = df.apply( + lambda x: LineString([[x.x0, x.y0], [x.x1, x.y1]]), axis=1 + ) + df.drop(columns=["x0", "y0", "x1", "y1"], inplace=True) + lines[file.stem] = df + return lines + + +def remove_projects_outside_countries(lines, europe_shape): + """ + Remove projects which are not in the considered countries. + """ + europe_shape_prepped = shapely.prepared.prep(europe_shape) + is_within_covered_countries = lines["geometry"].apply( + lambda x: europe_shape_prepped.contains(x) + ) + + if not is_within_covered_countries.all(): + logger.warning( + "Project lines outside of the covered area (skipping): " + + ", ".join(str(i) for i in lines.loc[~is_within_covered_countries].index) + ) + + lines = lines.loc[is_within_covered_countries] + return lines + + +def is_similar(ds1, ds2, percentage=10): + """ + Check if values in series ds2 are within a specified percentage of series + ds1. + + Returns: + - A boolean series where True indicates ds2 values are within the percentage range of ds2. + """ + lower_bound = ds1 * (1 - percentage / 100) + upper_bound = ds1 * (1 + percentage / 100) + return np.logical_and(ds2 >= lower_bound, ds2 <= upper_bound) + + +def set_underwater_fraction(new_links, offshore_shapes): + new_links_gds = gpd.GeoSeries(new_links["geometry"]) + new_links["underwater_fraction"] = ( + new_links_gds.intersection(offshore_shapes.union_all()).length + / new_links_gds.length + ).round(2) + + +def add_projects( + n, + new_lines_df, + new_links_df, + adjust_lines_df, + adjust_links_df, + new_buses_df, + europe_shape, + offshore_shapes, + path, + plan, + status=["confirmed", "under construction"], + skip=[], +): + lines_dict = get_project_files(path, skip=skip) + for key, lines in lines_dict.items(): + logging.info(f"Processing {key.replace('_', ' ')} projects from {plan}.") + lines = remove_projects_outside_countries(lines, europe_shape) + if isinstance(status, dict): + status = status[plan] + lines = lines.loc[lines.project_status.isin(status)] + if lines.empty: + continue + if key == "new_lines": + new_lines, new_buses_df = connect_new_lines( + lines, n, new_buses_df, bus_carrier="AC" + ) + duplicate_lines = find_closest_lines( + n.lines, new_lines, distance_upper_bound=0.10, type="new" + ) + # TODO: think about using build_year instead of v_nom + # ignore duplicates where v_nom is not within a tolerance of 10% + to_ignore = is_similar( + new_lines.loc[duplicate_lines.index, "v_nom"], + duplicate_lines.map(n.lines["v_nom"]), + ) + duplicate_lines = duplicate_lines[~to_ignore] + new_lines = new_lines.drop(duplicate_lines.index, errors="ignore") + new_lines_df = pd.concat([new_lines_df, new_lines]) + # add new lines to network to be able to find added duplicates + n.madd("Line", new_lines.index, **new_lines) + elif key == "new_links": + new_links, new_buses_df = connect_new_lines( + lines, + n, + new_buses_df, + offshore_shapes=offshore_shapes, + distance_upper_bound=0.4, + bus_carrier=["AC", "DC"], + ) + duplicate_links = find_closest_lines( + n.links, new_links, distance_upper_bound=0.10, type="new" + ) + # TODO: think about using build_year instead of p_nom + # ignore duplicates where p_nom is not within a tolerance of 10% + to_ignore = is_similar( + new_links.loc[duplicate_links.index, "p_nom"], + duplicate_links.map(n.links["p_nom"]), + ) + duplicate_links = duplicate_links[~to_ignore] + new_links = new_links.drop(duplicate_links.index, errors="ignore") + set_underwater_fraction(new_links, offshore_shapes) + new_links_df = pd.concat([new_links_df, new_links]) + # add new links to network to be able to find added duplicates + n.madd("Link", new_links.index, **new_links) + elif key == "upgraded_lines": + line_map = find_closest_lines( + n.lines, lines, distance_upper_bound=0.30, type="upgraded" + ) + upgraded_lines = lines.loc[line_map.index] + lines_to_adjust = adjust_decommissioning(upgraded_lines, line_map) + adjust_lines_df = pd.concat([adjust_lines_df, lines_to_adjust]) + upgraded_lines = get_upgraded_lines("Line", n, upgraded_lines, line_map) + new_lines_df = pd.concat([new_lines_df, upgraded_lines]) + elif key == "upgraded_links": + line_map = find_closest_lines( + n.links.query("carrier=='DC'"), + lines, + distance_upper_bound=0.30, + type="upgraded", + ) + upgraded_links = lines.loc[line_map.index] + links_to_adjust = adjust_decommissioning(upgraded_links, line_map) + adjust_links_df = pd.concat([adjust_links_df, links_to_adjust]) + upgraded_links = get_upgraded_lines("Link", n, upgraded_links, line_map) + new_links_df = pd.concat([new_links_df, upgraded_links]) + set_underwater_fraction(new_links_df, offshore_shapes) + else: + logger.warning(f"Unknown project type {key}") + continue + return new_lines_df, new_links_df, adjust_lines_df, adjust_links_df, new_buses_df + + +def fill_length_from_geometry(line, line_factor=1.2): + if not pd.isna(line.length): + return line.length + length = gpd.GeoSeries(line["geometry"], crs=4326).to_crs(3035).length.values[0] + length = length / 1000 * line_factor + return round(length, 1) + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("build_transmission_projects", run="all") + configure_logging(snakemake) + set_scenario_config(snakemake) + + line_factor = snakemake.params.line_factor + + n = pypsa.Network(snakemake.input.base_network) + + new_lines_df = pd.DataFrame() + new_links_df = pd.DataFrame() + adjust_lines_df = pd.DataFrame() + adjust_links_df = pd.DataFrame() + new_buses_df = pd.DataFrame() + + europe_shape = gpd.read_file(snakemake.input.europe_shape).loc[0, "geometry"] + offshore_shapes = gpd.read_file(snakemake.input.offshore_shapes).rename( + {"name": "country"}, axis=1 + ) + + transmission_projects = snakemake.params.transmission_projects + projects = [ + project + for project, include in transmission_projects["include"].items() + if include + ] + paths = snakemake.input.transmission_projects + for project in projects: + path = list(filter(lambda path: project in path, paths))[0] + new_lines_df, new_links_df, adjust_lines_df, adjust_links_df, new_buses_df = ( + add_projects( + n, + new_lines_df, + new_links_df, + adjust_lines_df, + adjust_links_df, + new_buses_df, + europe_shape, + offshore_shapes, + path=path, + plan=project, + status=transmission_projects["status"], + skip=transmission_projects["skip"], + ) + ) + if not new_lines_df.empty: + line_type = "Al/St 240/40 4-bundle 380.0" + # Add new line type for new lines + new_lines_df["type"] = new_lines_df["type"].fillna(line_type) + new_lines_df["num_parallel"] = new_lines_df["num_parallel"].fillna(2) + if "underground" in new_lines_df.columns: + new_lines_df["underground"] = ( + new_lines_df["underground"].astype("bool").fillna(False) + ) + # Add carrier types of lines + new_lines_df["carrier"] = "AC" + # Fill empty length values with length calculated from geometry + new_lines_df["length"] = new_lines_df.apply( + fill_length_from_geometry, args=(line_factor,), axis=1 + ) + # get s_nom from line type + new_lines_df["s_nom"] = ( + np.sqrt(3) + * n.line_types.loc[new_lines_df["type"], "i_nom"].values + * new_lines_df["v_nom"] + * new_lines_df["num_parallel"] + ).round(2) + if not new_links_df.empty: + # Add carrier types of lines and links + new_links_df["carrier"] = "DC" + # Fill empty length values with length calculated from geometry + new_links_df["length"] = new_links_df.apply( + fill_length_from_geometry, args=(line_factor,), axis=1 + ) + # Whether to keep existing link capacity or set to zero + not_upgraded = ~new_links_df.index.str.contains("upgraded") + if transmission_projects["new_link_capacity"] == "keep": + new_links_df.loc[not_upgraded, "p_nom"] = new_links_df["p_nom"].fillna(0) + elif transmission_projects["new_link_capacity"] == "zero": + new_links_df.loc[not_upgraded, "p_nom"] = 0 + # export csv files for new buses, lines, links and adjusted lines and links + new_lines_df.to_csv(snakemake.output.new_lines) + new_links_df.to_csv(snakemake.output.new_links) + adjust_lines_df.to_csv(snakemake.output.adjust_lines) + adjust_links_df.to_csv(snakemake.output.adjust_links) + new_buses_df.to_csv(snakemake.output.new_buses) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index da7bd178..79184167 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -410,6 +410,7 @@ def clustering_for_n_clusters( generator_strategies=generator_strategies, one_port_strategies=one_port_strategies, scale_link_capital_costs=False, + custom_line_groupers=["build_year"], ) if not n.links.empty: diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index c54e3418..9ab0cb23 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -124,7 +124,7 @@ def simplify_network_to_380(n): n.buses["v_nom"] = 380.0 - (linetype_380,) = n.lines.loc[n.lines.v_nom == 380.0, "type"].unique() + linetype_380 = n.lines["type"].mode()[0] n.lines["type"] = linetype_380 n.lines["v_nom"] = 380 n.lines["i_nom"] = n.line_types.i_nom[linetype_380] @@ -579,7 +579,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("simplify_network", simpl="") + snakemake = mock_snakemake("simplify_network", simpl="", run="all") configure_logging(snakemake) set_scenario_config(snakemake) @@ -655,6 +655,7 @@ if __name__ == "__main__": "substation_off", "geometry", "underground", + "project_status", ] n.buses.drop(remove, axis=1, inplace=True, errors="ignore") n.lines.drop(remove, axis=1, errors="ignore", inplace=True)