From fa03c61187a232c452714979515967910474f14b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 31 Jul 2023 10:52:37 +0200 Subject: [PATCH 1/5] gas_input: switch production data from scigrid to gem --- rules/build_sector.smk | 3 +- scripts/build_gas_input_locations.py | 56 +++++++++++++++++++++------- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index dd49fc6f..1e8c70ba 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -85,12 +85,11 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule build_gas_input_locations: input: - lng=HTTP.remote( + gem=HTTP.remote( "https://globalenergymonitor.org/wp-content/uploads/2023/07/Europe-Gas-Tracker-2023-03-v3.xlsx", keep_local=True, ), entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson", - production="data/gas_network/scigrid-gas/data/IGGIELGN_Productions.geojson", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", regions_offshore=RESOURCES diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index a3b945ab..07707658 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -23,11 +23,10 @@ def read_scigrid_gas(fn): return df -def build_gem_lng_data(lng_fn): - df = pd.read_excel(lng_fn[0], sheet_name="LNG terminals - data") +def build_gem_lng_data(fn): + df = pd.read_excel(fn[0], sheet_name="LNG terminals - data") df = df.set_index("ComboID") - remove_status = ["Cancelled"] remove_country = ["Cyprus", "Turkey"] remove_terminal = ["Puerto de la Luz LNG Terminal", "Gran Canaria LNG Terminal"] @@ -42,9 +41,43 @@ def build_gem_lng_data(lng_fn): return gpd.GeoDataFrame(df, geometry=geometry, crs="EPSG:4326") -def build_gas_input_locations(lng_fn, entry_fn, prod_fn, countries): +def build_gem_prod_data(fn): + df = pd.read_excel(fn[0], sheet_name="Gas extraction - main") + df = df.set_index("GEM Unit ID") + + remove_country = ["Cyprus", "Türkiye"] + remove_fuel_type = ["oil"] + + df = df.query( + "Status != 'shut in' \ + & 'Fuel type' != 'oil' \ + & Country != @remove_country \ + & ~Latitude.isna() \ + & ~Longitude.isna()" + ).copy() + + p = pd.read_excel(fn[0], sheet_name="Gas extraction - production") + p = p.set_index("GEM Unit ID") + p = p[p["Fuel description"] == 'gas' ] + + capacities = pd.DataFrame(index=df.index) + for key in ["production", "production design capacity", "reserves"]: + cap = p.loc[p["Production/reserves"] == key, "Quantity (converted)"].groupby("GEM Unit ID").sum().reindex(df.index) + # assume capacity such that 3% of reserves can be extracted per year (25% quantile) + annualization_factor = 0.03 if key == "reserves" else 1. + capacities[key] = cap * annualization_factor + + df["mcm_per_year"] = capacities["production"] \ + .combine_first(capacities["production design capacity"]) \ + .combine_first(capacities["reserves"]) + + geometry = gpd.points_from_xy(df["Longitude"], df["Latitude"]) + return gpd.GeoDataFrame(df, geometry=geometry, crs="EPSG:4326") + + +def build_gas_input_locations(gem_fn, entry_fn, countries): # LNG terminals - lng = build_gem_lng_data(lng_fn) + lng = build_gem_lng_data(gem_fn) # Entry points from outside the model scope entry = read_scigrid_gas(entry_fn) @@ -56,16 +89,14 @@ def build_gas_input_locations(lng_fn, entry_fn, prod_fn, countries): ] # production sites inside the model scope - prod = read_scigrid_gas(prod_fn) - prod = prod.loc[ - (prod.geometry.y > 35) & (prod.geometry.x < 30) & (prod.country_code != "DE") - ] + prod = build_gem_prod_data(gem_fn) mcm_per_day_to_mw = 437.5 # MCM/day to MWh/h + mcm_per_year_to_mw = 1.199 # MCM/year to MWh/h mtpa_to_mw = 1649.224 # mtpa to MWh/h lng["p_nom"] = lng["CapacityInMtpa"] * mtpa_to_mw entry["p_nom"] = entry["max_cap_from_to_M_m3_per_d"] * mcm_per_day_to_mw - prod["p_nom"] = prod["max_supply_M_m3_per_d"] * mcm_per_day_to_mw + prod["p_nom"] = prod["mcm_per_year"] * mcm_per_year_to_mw lng["type"] = "lng" entry["type"] = "pipeline" @@ -83,7 +114,7 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_gas_input_locations", simpl="", - clusters="37", + clusters="128", ) logging.basicConfig(level=snakemake.config["logging"]["level"]) @@ -104,9 +135,8 @@ if __name__ == "__main__": countries = regions.index.str[:2].unique().str.replace("GB", "UK") gas_input_locations = build_gas_input_locations( - snakemake.input.lng, + snakemake.input.gem, snakemake.input.entry, - snakemake.input.production, countries, ) From 7c058f1ed333d41703e62d3d406d0d61a803da7d Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 31 Jul 2023 12:20:43 +0200 Subject: [PATCH 2/5] add locations, capacities and costs of existing gas storage --- rules/build_sector.smk | 1 + scripts/build_gas_input_locations.py | 24 ++++++++++++++++-------- scripts/prepare_sector_network.py | 17 ++++++++++++----- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 1e8c70ba..ab8ff4ed 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -90,6 +90,7 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: keep_local=True, ), entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson", + storage="data/gas_network/scigrid-gas/data/IGGIELGN_Storages.geojson", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", regions_offshore=RESOURCES diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index 07707658..ad449202 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -75,7 +75,7 @@ def build_gem_prod_data(fn): return gpd.GeoDataFrame(df, geometry=geometry, crs="EPSG:4326") -def build_gas_input_locations(gem_fn, entry_fn, countries): +def build_gas_input_locations(gem_fn, entry_fn, sto_fn, countries): # LNG terminals lng = build_gem_lng_data(gem_fn) @@ -88,23 +88,30 @@ def build_gas_input_locations(gem_fn, entry_fn, countries): | (entry.from_country == "NO") # malformed datapoint # entries from NO to GB ] + sto = read_scigrid_gas(sto_fn) + remove_country = ["RU", "UA", "TR", "BY"] + sto = sto.query("country_code != @remove_country") + # production sites inside the model scope prod = build_gem_prod_data(gem_fn) mcm_per_day_to_mw = 437.5 # MCM/day to MWh/h mcm_per_year_to_mw = 1.199 # MCM/year to MWh/h mtpa_to_mw = 1649.224 # mtpa to MWh/h - lng["p_nom"] = lng["CapacityInMtpa"] * mtpa_to_mw - entry["p_nom"] = entry["max_cap_from_to_M_m3_per_d"] * mcm_per_day_to_mw - prod["p_nom"] = prod["mcm_per_year"] * mcm_per_year_to_mw + mcm_to_gwh = 11.36 # MCM to GWh + lng["capacity"] = lng["CapacityInMtpa"] * mtpa_to_mw + entry["capacity"] = entry["max_cap_from_to_M_m3_per_d"] * mcm_per_day_to_mw + prod["capacity"] = prod["mcm_per_year"] * mcm_per_year_to_mw + sto["capacity"] = sto["max_cushionGas_M_m3"] * mcm_to_gwh lng["type"] = "lng" entry["type"] = "pipeline" prod["type"] = "production" + sto["type"] = "storage" - sel = ["geometry", "p_nom", "type"] + sel = ["geometry", "capacity", "type"] - return pd.concat([prod[sel], entry[sel], lng[sel]], ignore_index=True) + return pd.concat([prod[sel], entry[sel], lng[sel], sto[sel]], ignore_index=True) if __name__ == "__main__": @@ -137,6 +144,7 @@ if __name__ == "__main__": gas_input_locations = build_gas_input_locations( snakemake.input.gem, snakemake.input.entry, + snakemake.input.storage, countries, ) @@ -147,8 +155,8 @@ if __name__ == "__main__": gas_input_nodes.to_file(snakemake.output.gas_input_nodes, driver="GeoJSON") gas_input_nodes_s = ( - gas_input_nodes.groupby(["bus", "type"])["p_nom"].sum().unstack() + gas_input_nodes.groupby(["bus", "type"])["capacity"].sum().unstack() ) - gas_input_nodes_s.columns.name = "p_nom" + gas_input_nodes_s.columns.name = "capacity" gas_input_nodes_s.to_csv(snakemake.output.gas_input_nodes_simplified) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index e9d97ade..9387d4b1 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -454,10 +454,11 @@ def add_carrier_buses(n, carrier, nodes=None): n.add("Carrier", carrier) unit = "MWh_LHV" if carrier == "gas" else "MWh_th" + # preliminary value for non-gas carriers to avoid zeros + capital_cost = costs.at["gas storage", "fixed"] if carrier == "gas" else 0.02 n.madd("Bus", nodes, location=location, carrier=carrier, unit=unit) - # capital cost could be corrected to e.g. 0.2 EUR/kWh * annuity and O&M n.madd( "Store", nodes + " Store", @@ -465,8 +466,7 @@ def add_carrier_buses(n, carrier, nodes=None): e_nom_extendable=True, e_cyclic=True, carrier=carrier, - capital_cost=0.2 - * costs.at[carrier, "discount rate"], # preliminary value to avoid zeros + capital_cost=capital_cost, ) n.madd( @@ -1162,7 +1162,7 @@ def add_storage_and_grids(n, costs): if options["gas_network"]: logger.info( - "Add natural gas infrastructure, incl. LNG terminals, production and entry-points." + "Add natural gas infrastructure, incl. LNG terminals, production, storage and entry-points." ) if options["H2_retrofit"]: @@ -1207,10 +1207,17 @@ def add_storage_and_grids(n, costs): remove_i = n.generators[gas_i & internal_i].index n.generators.drop(remove_i, inplace=True) - p_nom = gas_input_nodes.sum(axis=1).rename(lambda x: x + " gas") + input_types = ["lng", "pipeline", "production"] + p_nom = gas_input_nodes[input_types].sum(axis=1).rename(lambda x: x + " gas") n.generators.loc[gas_i, "p_nom_extendable"] = False n.generators.loc[gas_i, "p_nom"] = p_nom + # add existing gas storage capacity + gas_i = n.stores.carrier == "gas" + e_nom = gas_input_nodes["storage"].rename(lambda x: x + " gas Store").reindex(n.stores.index).fillna(0.) * 1e3 # MWh_LHV + e_nom.clip(upper=e_nom.quantile(0.98), inplace=True) # limit extremely large storage + n.stores.loc[gas_i, "e_nom_min"] = e_nom + # add candidates for new gas pipelines to achieve full connectivity G = nx.Graph() From 252f6d2c15838dc17ded00271f4edc05b417bec8 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 3 Jan 2024 08:13:01 +0100 Subject: [PATCH 3/5] pre-commit formatting --- .pre-commit-config.yaml | 2 +- scripts/build_gas_input_locations.py | 21 ++++++++++++++------- scripts/prepare_sector_network.py | 12 ++++++++++-- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7b9009c3..78e70b57 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,7 +50,7 @@ repos: - id: blackdoc # Formatting with "black" coding style -- repo: https://github.com/psf/black +- repo: https://github.com/psf/black-pre-commit-mirror rev: 23.12.1 hooks: # Format Python files diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index ad449202..2f967c75 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -47,7 +47,7 @@ def build_gem_prod_data(fn): remove_country = ["Cyprus", "Türkiye"] remove_fuel_type = ["oil"] - + df = df.query( "Status != 'shut in' \ & 'Fuel type' != 'oil' \ @@ -58,18 +58,25 @@ def build_gem_prod_data(fn): p = pd.read_excel(fn[0], sheet_name="Gas extraction - production") p = p.set_index("GEM Unit ID") - p = p[p["Fuel description"] == 'gas' ] + p = p[p["Fuel description"] == "gas"] capacities = pd.DataFrame(index=df.index) for key in ["production", "production design capacity", "reserves"]: - cap = p.loc[p["Production/reserves"] == key, "Quantity (converted)"].groupby("GEM Unit ID").sum().reindex(df.index) + cap = ( + p.loc[p["Production/reserves"] == key, "Quantity (converted)"] + .groupby("GEM Unit ID") + .sum() + .reindex(df.index) + ) # assume capacity such that 3% of reserves can be extracted per year (25% quantile) - annualization_factor = 0.03 if key == "reserves" else 1. + annualization_factor = 0.03 if key == "reserves" else 1.0 capacities[key] = cap * annualization_factor - df["mcm_per_year"] = capacities["production"] \ - .combine_first(capacities["production design capacity"]) \ + df["mcm_per_year"] = ( + capacities["production"] + .combine_first(capacities["production design capacity"]) .combine_first(capacities["reserves"]) + ) geometry = gpd.points_from_xy(df["Longitude"], df["Latitude"]) return gpd.GeoDataFrame(df, geometry=geometry, crs="EPSG:4326") @@ -88,7 +95,7 @@ def build_gas_input_locations(gem_fn, entry_fn, sto_fn, countries): | (entry.from_country == "NO") # malformed datapoint # entries from NO to GB ] - sto = read_scigrid_gas(sto_fn) + sto = read_scigrid_gas(sto_fn) remove_country = ["RU", "UA", "TR", "BY"] sto = sto.query("country_code != @remove_country") diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 9387d4b1..d5c979fa 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1214,8 +1214,16 @@ def add_storage_and_grids(n, costs): # add existing gas storage capacity gas_i = n.stores.carrier == "gas" - e_nom = gas_input_nodes["storage"].rename(lambda x: x + " gas Store").reindex(n.stores.index).fillna(0.) * 1e3 # MWh_LHV - e_nom.clip(upper=e_nom.quantile(0.98), inplace=True) # limit extremely large storage + e_nom = ( + gas_input_nodes["storage"] + .rename(lambda x: x + " gas Store") + .reindex(n.stores.index) + .fillna(0.0) + * 1e3 + ) # MWh_LHV + e_nom.clip( + upper=e_nom.quantile(0.98), inplace=True + ) # limit extremely large storage n.stores.loc[gas_i, "e_nom_min"] = e_nom # add candidates for new gas pipelines to achieve full connectivity From 4983a2e02178dfe501358ce24636f877ecd4f478 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 3 Jan 2024 08:19:52 +0100 Subject: [PATCH 4/5] add release note --- doc/release_notes.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 634209c7..36823791 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -65,6 +65,9 @@ Upcoming Release * Validate downloads from Zenodo using MD5 checksums. This identifies corrupted or incomplete downloads. +* Add locations, capacities and costs of existing gas storage using Global + Energy Monitor's `Europe Gas Tracker + `_. **Bugs and Compatibility** From 19b503d7580faf75dba539923d255c37f4038fd7 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 11 Aug 2023 12:07:03 +0200 Subject: [PATCH 5/5] retrieve.smk: add scigrid storages to files of interest --- rules/retrieve.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 4c9ca814..4ded2a46 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -169,6 +169,7 @@ if config["enable"]["retrieve"] and ( "IGGIELGN_LNGs.geojson", "IGGIELGN_BorderPoints.geojson", "IGGIELGN_Productions.geojson", + "IGGIELGN_Storages.geojson", "IGGIELGN_PipeSegments.geojson", ]