From 7535a0d38559b8b85cc1ddc3b4871ff07e786b5c Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 6 Jan 2022 10:29:59 +0100 Subject: [PATCH 01/45] update data bundle (#216) --- doc/release_notes.rst | 4 +++- scripts/retrieve_sector_databundle.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index d5ca8b80..7808d2ba 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -62,6 +62,8 @@ incorporates retrofitting options to hydrogen. * Add option to sweep the global CO2 sequestration potentials with keyword ``seq200`` in the ``{sector_opts}`` wildcard (for limit of 200 Mt CO2). +* Updated `data bundle `_ that includes the hydrogan salt cavern storage potentials. + **Bugfixes** * The CO2 sequestration limit implemented as GlobalConstraint (introduced in the previous version) @@ -433,4 +435,4 @@ To make a new release of the data bundle, make an archive of the files in ``data .. code:: bash - data % tar pczf pypsa-eur-sec-data-bundle.tar.gz eea/UNFCCC_v23.csv switzerland-sfoe biomass eurostat-energy_balances-* jrc-idees-2015 emobility WindWaveWEC_GLTB.xlsx myb1-2017-nitro.xls Industrial_Database.csv retro/tabula-calculator-calcsetbuilding.csv nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson + data % tar pczf pypsa-eur-sec-data-bundle.tar.gz eea/UNFCCC_v23.csv switzerland-sfoe biomass eurostat-energy_balances-* jrc-idees-2015 emobility WindWaveWEC_GLTB.xlsx myb1-2017-nitro.xls Industrial_Database.csv retro/tabula-calculator-calcsetbuilding.csv nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson h2_salt_caverns_GWh_per_sqkm.geojson diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py index 6e647ce5..9fba27ea 100644 --- a/scripts/retrieve_sector_databundle.py +++ b/scripts/retrieve_sector_databundle.py @@ -19,7 +19,7 @@ from _helpers import progress_retrieve, configure_logging if __name__ == "__main__": configure_logging(snakemake) - url = "https://zenodo.org/record/5546517/files/pypsa-eur-sec-data-bundle.tar.gz" + url = "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz" tarball_fn = Path("sector-bundle.tar.gz") to_fn = Path("data") From 372473d76cdbca598517be253b50ce9d138d72e5 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 7 Jan 2022 11:32:31 +0100 Subject: [PATCH 02/45] update mocksnakemake --- scripts/add_existing_baseyear.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index bb35e378..53f986c9 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -159,7 +159,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas inv_busmap = {} for k, v in busmap.iteritems(): inv_busmap[v] = inv_busmap.get(v, []) + [k] - + clustermaps = busmap_s.map(busmap) clustermaps.index = clustermaps.index.astype(int) @@ -197,7 +197,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas capacity = capacity[capacity > snakemake.config['existing_capacities']['threshold_capacity']] if generator in ['solar', 'onwind', 'offwind']: - + suffix = '-ac' if generator == 'offwind' else '' name_suffix = f' {generator}{suffix}-{baseyear}' @@ -213,7 +213,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas p_max_pu = n.generators_t.p_max_pu[[i + name_suffix for i in inv_ind]] p_max_pu.columns=[i + name_suffix for i in inv_ind ] - + n.madd("Generator", [i + name_suffix for i in inv_ind], bus=ind, @@ -436,17 +436,17 @@ def add_heating_capacities_installed_before_baseyear(n, baseyear, grouping_years threshold = snakemake.config['existing_capacities']['threshold_capacity'] n.mremove("Link", [index for index in n.links.index.to_list() if str(grouping_year) in index and n.links.p_nom[index] < threshold]) - +#%% if __name__ == "__main__": if 'snakemake' not in globals(): from helper import mock_snakemake snakemake = mock_snakemake( 'add_existing_baseyear', simpl='', - clusters=45, + clusters="45", lv=1.0, opts='', - sector_opts='Co2L0-168H-T-H-B-I-solar+p3-dist1', + sector_opts='168H-T-H-B-I-A-solar+p3-dist1', planning_horizons=2020, ) From 953e1c883e335bbf885e5a35c9e509d745249d83 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 7 Jan 2022 11:38:25 +0100 Subject: [PATCH 03/45] adjust functions to new default lifetime np.inf instead of previous NaN --- scripts/add_brownfield.py | 7 ++++--- scripts/add_existing_baseyear.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 0952c752..0164e64a 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -8,6 +8,7 @@ idx = pd.IndexSlice import pypsa import yaml +import numpy as np from add_existing_baseyear import add_build_year_to_new_assets from helper import override_component_attrs @@ -25,7 +26,7 @@ def add_brownfield(n, n_p, year): # CO2 or global EU values since these are already in n n_p.mremove( c.name, - c.df.index[c.df.lifetime.isna()] + c.df.index[c.df.lifetime==np.inf] ) # remove assets whose build_year + lifetime < year @@ -44,7 +45,7 @@ def add_brownfield(n, n_p, year): )] threshold = snakemake.config['existing_capacities']['threshold_capacity'] - + if not chp_heat.empty: threshold_chp_heat = (threshold * c.df.efficiency[chp_heat.str.replace("heat", "electric")].values @@ -55,7 +56,7 @@ def add_brownfield(n, n_p, year): c.name, chp_heat[c.df.loc[chp_heat, attr + "_nom_opt"] < threshold_chp_heat] ) - + n_p.mremove( c.name, c.df.index[c.df[attr + "_nom_extendable"] & ~c.df.index.isin(chp_heat) & (c.df[attr + "_nom_opt"] < threshold)] diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 53f986c9..6955fb6c 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -28,7 +28,7 @@ def add_build_year_to_new_assets(n, baseyear): # Give assets with lifetimes and no build year the build year baseyear for c in n.iterate_components(["Link", "Generator", "Store"]): - assets = c.df.index[~c.df.lifetime.isna() & c.df.build_year==0] + assets = c.df.index[(c.df.lifetime!=np.inf) & (c.df.build_year==0)] c.df.loc[assets, "build_year"] = baseyear # add -baseyear to name From b5aa4234ef79490b7e4a45f3c92704fa1859a55d Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 7 Jan 2022 12:09:02 +0100 Subject: [PATCH 04/45] adjust capital cost of existing renewables to consider e.g. electricity grid connection costs as well --- scripts/add_existing_baseyear.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 6955fb6c..07eb3cda 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -201,6 +201,11 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas suffix = '-ac' if generator == 'offwind' else '' name_suffix = f' {generator}{suffix}-{baseyear}' + # to consider electricity grid connection costs or a split between + # solar utility and rooftop as well, rather take cost assumptions + # from existing network than from the cost database + capital_cost = n.generators[n.generators.carrier==generator+suffix].capital_cost.mean() + if 'm' in snakemake.wildcards.clusters: for ind in capacity.index: @@ -220,7 +225,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas carrier=generator, p_nom=capacity[ind] / len(inv_ind), # split among regions in a country marginal_cost=costs.at[generator,'VOM'], - capital_cost=costs.at[generator,'fixed'], + capital_cost=capital_cost, efficiency=costs.at[generator, 'efficiency'], p_max_pu=p_max_pu, build_year=grouping_year, @@ -238,7 +243,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas carrier=generator, p_nom=capacity, marginal_cost=costs.at[generator, 'VOM'], - capital_cost=costs.at[generator, 'fixed'], + capital_cost=capital_cost, efficiency=costs.at[generator, 'efficiency'], p_max_pu=p_max_pu.rename(columns=n.generators.bus), build_year=grouping_year, @@ -471,7 +476,7 @@ if __name__ == "__main__": snakemake.config['costs']['lifetime'] ) - grouping_years=snakemake.config['existing_capacities']['grouping_years'] + grouping_years = snakemake.config['existing_capacities']['grouping_years'] add_power_capacities_installed_before_baseyear(n, grouping_years, costs, baseyear) if "H" in opts: From c9ce9190a726086c5c33147e05b7002d4998691d Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 7 Jan 2022 12:40:17 +0100 Subject: [PATCH 05/45] adjust to spatital resolved gas nodes --- scripts/add_existing_baseyear.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 07eb3cda..74019b8c 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -251,11 +251,15 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas ) else: + bus0 = n.buses[(n.buses.carrier==carrier[generator])].index + if any(n.buses.loc[bus0,"location"]!="EU"): + bus0 = n.buses[n.buses.location.isin(capacity.index) & + (n.buses.carrier==carrier[generator])].index n.madd("Link", capacity.index, suffix= " " + generator +"-" + str(grouping_year), - bus0="EU " + carrier[generator], + bus0=bus0, bus1=capacity.index, bus2="co2 atmosphere", carrier=generator, @@ -404,10 +408,15 @@ def add_heating_capacities_installed_before_baseyear(n, baseyear, grouping_years lifetime=costs.at[costs_name, 'lifetime'] ) + bus0 = n.buses[(n.buses.carrier=="gas")].index + if any(n.buses.loc[bus0,"location"]!="EU"): + bus0 = n.buses[n.buses.location.isin(nodal_df[f'{heat_type} gas boiler'][nodes[name]].index) & + (n.buses.carrier=="gas")].index + n.madd("Link", nodes[name], suffix= f" {name} gas boiler-{grouping_year}", - bus0="EU gas", + bus0=bus0, bus1=nodes[name] + " " + name + " heat", bus2="co2 atmosphere", carrier=name + " gas boiler", From 030bf96d329a48a84e8153b0adfef21395d2cabb Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 7 Jan 2022 12:45:33 +0100 Subject: [PATCH 06/45] update mocksnakemake --- scripts/add_brownfield.py | 5 +++-- scripts/add_existing_baseyear.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 0164e64a..ddb0a626 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -83,9 +83,10 @@ if __name__ == "__main__": snakemake = mock_snakemake( 'add_brownfield', simpl='', - clusters=48, + clusters="45", + opts="", lv=1.0, - sector_opts='Co2L0-168H-T-H-B-I-solar3-dist1', + sector_opts='Co2L0-168H-T-H-B-I-A-solar3-dist1', planning_horizons=2030, ) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 74019b8c..fae64904 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -450,7 +450,7 @@ def add_heating_capacities_installed_before_baseyear(n, baseyear, grouping_years threshold = snakemake.config['existing_capacities']['threshold_capacity'] n.mremove("Link", [index for index in n.links.index.to_list() if str(grouping_year) in index and n.links.p_nom[index] < threshold]) -#%% + if __name__ == "__main__": if 'snakemake' not in globals(): from helper import mock_snakemake From 5b30f0ece6edc5c11d089ee62e04a1dac479f213 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 7 Jan 2022 16:53:37 +0100 Subject: [PATCH 07/45] avoid doubling of the existing natural gas grid --- scripts/add_brownfield.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index ddb0a626..937ba409 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -76,7 +76,12 @@ def add_brownfield(n, n_p, year): for tattr in n.component_attrs[c.name].index[selection]: n.import_series_from_dataframe(c.pnl[tattr], c.name, tattr) + # deal with gas network + pipe_carrier = ['gas pipeline'] + to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year!=year) + n.mremove("Link", n.links.loc[to_drop].index) +#%% if __name__ == "__main__": if 'snakemake' not in globals(): from helper import mock_snakemake @@ -86,7 +91,7 @@ if __name__ == "__main__": clusters="45", opts="", lv=1.0, - sector_opts='Co2L0-168H-T-H-B-I-A-solar3-dist1', + sector_opts='168H-T-H-B-I-A-solar+p3-dist1', planning_horizons=2030, ) From 9f8b54a3cea63f67b08c81fd85cc15a9edc2844c Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 7 Jan 2022 16:59:14 +0100 Subject: [PATCH 08/45] adjust pipe retrofitting constraint to work with myopic --- scripts/solve_network.py | 51 +++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index e23f3437..5108e2ee 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -33,14 +33,14 @@ def _add_land_use_constraint(n): existing = n.generators.loc[n.generators.carrier==carrier,"p_nom"].groupby(n.generators.bus.map(n.buses.location)).sum() existing.index += " " + carrier + "-" + snakemake.wildcards.planning_horizons n.generators.loc[existing.index,"p_nom_max"] -= existing - + n.generators.p_nom_max.clip(lower=0, inplace=True) def _add_land_use_constraint_m(n): # if generators clustering is lower than network clustering, land_use accounting is at generators clusters - planning_horizons = snakemake.config["scenario"]["planning_horizons"] + planning_horizons = snakemake.config["scenario"]["planning_horizons"] grouping_years = snakemake.config["existing_capacities"]["grouping_years"] current_horizon = snakemake.wildcards.planning_horizons @@ -48,9 +48,9 @@ def _add_land_use_constraint_m(n): existing = n.generators.loc[n.generators.carrier==carrier,"p_nom"] ind = list(set([i.split(sep=" ")[0] + ' ' + i.split(sep=" ")[1] for i in existing.index])) - + previous_years = [ - str(y) for y in + str(y) for y in planning_horizons + grouping_years if y < int(snakemake.wildcards.planning_horizons) ] @@ -59,13 +59,13 @@ def _add_land_use_constraint_m(n): ind2 = [i for i in ind if i + " " + carrier + "-" + p_year in existing.index] sel_current = [i + " " + carrier + "-" + current_horizon for i in ind2] sel_p_year = [i + " " + carrier + "-" + p_year for i in ind2] - n.generators.loc[sel_current, "p_nom_max"] -= existing.loc[sel_p_year].rename(lambda x: x[:-4] + current_horizon) - + n.generators.loc[sel_current, "p_nom_max"] -= existing.loc[sel_p_year].rename(lambda x: x[:-4] + current_horizon) + n.generators.p_nom_max.clip(lower=0, inplace=True) def prepare_network(n, solve_opts=None): - + if 'clip_p_max_pu' in solve_opts: for df in (n.generators_t.p_max_pu, n.generators_t.p_min_pu, n.storage_units_t.inflow): df.where(df>solve_opts['clip_p_max_pu'], other=0., inplace=True) @@ -189,36 +189,42 @@ def add_chp_constraints(n): def add_pipe_retrofit_constraint(n): """Add constraint for retrofitting existing CH4 pipelines to H2 pipelines.""" - gas_pipes_i = n.links[n.links.carrier=="gas pipeline"].index - h2_retrofitted_i = n.links[n.links.carrier=='H2 pipeline retrofitted'].index + gas_pipes_i = n.links[n.links.carrier=="gas pipeline"].query("p_nom_extendable").index + h2_retrofitted_i = n.links[n.links.carrier=='H2 pipeline retrofitted'].query("p_nom_extendable").index + h2_retrofitted_fixed_i = n.links[n.links.carrier=='H2 pipeline retrofitted'].index.difference(h2_retrofitted_i) if h2_retrofitted_i.empty or gas_pipes_i.empty: return link_p_nom = get_var(n, "Link", "p_nom") - pipe_capacity = n.links.loc[gas_pipes_i, 'p_nom'] - CH4_per_H2 = 1 / n.config["sector"]["H2_retrofit_capacity_per_CH4"] - fr = "H2 pipeline retrofitted" to = "gas pipeline" + + pipe_capacity = n.links.loc[gas_pipes_i, 'p_nom'].rename(index=lambda x: x.split("-2")[0]) + already_retrofitted = (n.links.loc[h2_retrofitted_fixed_i, 'p_nom'] + .rename(index= lambda x: x.split("-2")[0] + .replace(fr, to)).groupby(level=0).sum()) + remaining_capacity = pipe_capacity - CH4_per_H2 * already_retrofitted.reindex(index=pipe_capacity.index).fillna(0) + lhs = linexpr( (CH4_per_H2, link_p_nom.loc[h2_retrofitted_i].rename(index=lambda x: x.replace(fr, to))), (1, link_p_nom.loc[gas_pipes_i]) ) - define_constraints(n, lhs, "=", pipe_capacity, 'Link', 'pipe_retrofit') + lhs.rename(index=lambda x: x.split("-2")[0], inplace=True) + define_constraints(n, lhs, "=", remaining_capacity, 'Link', 'pipe_retrofit') def add_co2_sequestration_limit(n, sns): - + co2_stores = n.stores.loc[n.stores.carrier=='co2 stored'].index if co2_stores.empty or ('Store', 'e') not in n.variables.index: return - + vars_final_co2_stored = get_var(n, 'Store', 'e').loc[sns[-1], co2_stores] - + lhs = linexpr((1, vars_final_co2_stored)).sum() limit = n.config["sector"].get("co2_sequestration_potential", 200) * 1e6 @@ -226,7 +232,7 @@ def add_co2_sequestration_limit(n, sns): if not "seq" in o: continue limit = float(o[o.find("seq")+3:]) break - + name = 'co2_sequestration_limit' sense = "<=" @@ -258,7 +264,7 @@ def solve_network(n, config, opts='', **kwargs): if cf_solving.get('skip_iterations', False): network_lopf(n, solver_name=solver_name, solver_options=solver_options, - extra_functionality=extra_functionality, + extra_functionality=extra_functionality, keep_shadowprices=keep_shadowprices, **kwargs) else: ilopf(n, solver_name=solver_name, solver_options=solver_options, @@ -275,12 +281,13 @@ if __name__ == "__main__": if 'snakemake' not in globals(): from helper import mock_snakemake snakemake = mock_snakemake( - 'solve_network', + 'solve_network_myopic', simpl='', - clusters=48, + opts="", + clusters="45", lv=1.0, - sector_opts='Co2L0-168H-T-H-B-I-solar3-dist1', - planning_horizons=2050, + sector_opts='168H-T-H-B-I-A-solar+p3-dist1', + planning_horizons="2030", ) logging.basicConfig(filename=snakemake.log.python, From c40904a72708aadea19761eca64ec6c386d27cf1 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 7 Jan 2022 16:59:48 +0100 Subject: [PATCH 09/45] adjust h2_plot function to work with myopic, since build year is now in links.index --- scripts/plot_network.py | 42 ++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 9b8cddc3..752459a3 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -235,7 +235,7 @@ def plot_h2_map(network): bus_size_factor = 1e5 linewidth_factor = 1e4 # MW below which not drawn - line_lower_threshold = 1e3 + line_lower_threshold = 1e2 # Drop non-electric buses so they don't clutter the plot n.buses.drop(n.buses.index[n.buses.carrier != "AC"], inplace=True) @@ -249,7 +249,7 @@ def plot_h2_map(network): n.links.drop(n.links.index[~n.links.carrier.str.contains("H2 pipeline")], inplace=True) - h2_new = n.links.loc[n.links.carrier=="H2 pipeline", "p_nom_opt"] + h2_new = n.links.loc[n.links.carrier=="H2 pipeline"] h2_retro = n.links.loc[n.links.carrier=='H2 pipeline retrofitted'] @@ -264,10 +264,18 @@ def plot_h2_map(network): axis=1 ) - h2_retro = h2_retro["p_nom_opt"] + h2_new.index = h2_new.apply( + lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}" + ,axis=1) + h2_new = h2_new["p_nom_opt"].groupby(level=0).sum() + h2_retro = h2_retro["p_nom_opt"].groupby(level=0).sum() + h2_retro = h2_retro.groupby(level=0).sum().reindex(h2_new.index).fillna(0) + + n.links.rename(index=lambda x: x.split("-2")[0], inplace=True) + n.links = n.links.groupby(level=0).first() link_widths_total = (h2_new + h2_retro) / linewidth_factor - link_widths_total = link_widths_total.groupby(level=0).sum().reindex(n.links.index).fillna(0.) + link_widths_total = link_widths_total.reindex(n.links.index).fillna(0.) link_widths_total[n.links.p_nom_opt < line_lower_threshold] = 0. retro = n.links.p_nom_opt.where(n.links.carrier=='H2 pipeline retrofitted', other=0.) @@ -281,7 +289,7 @@ def plot_h2_map(network): figsize=(7, 6), subplot_kw={"projection": ccrs.PlateCarree()} ) - + n.plot( bus_sizes=bus_sizes, bus_colors=snakemake.config['plotting']['tech_colors'], @@ -365,7 +373,7 @@ def plot_ch4_map(network): # Drop non-electric buses so they don't clutter the plot n.buses.drop(n.buses.index[n.buses.carrier != "AC"], inplace=True) - fossil_gas_i = n.generators[n.generators.carrier=="gas"].index + fossil_gas_i = n.generators[n.generators.carrier=="gas"].index fossil_gas = n.generators_t.p.loc[:,fossil_gas_i].mul(n.snapshot_weightings.generators, axis=0).sum().groupby(n.generators.loc[fossil_gas_i,"bus"]).sum() / bus_size_factor fossil_gas.rename(index=lambda x: x.replace(" gas", ""), inplace=True) fossil_gas = fossil_gas.reindex(n.buses.index).fillna(0) @@ -390,10 +398,10 @@ def plot_ch4_map(network): to_remove = n.links.index[~n.links.carrier.str.contains("gas pipeline")] n.links.drop(to_remove, inplace=True) - link_widths_rem = n.links.p_nom_opt / linewidth_factor + link_widths_rem = n.links.p_nom_opt / linewidth_factor link_widths_rem[n.links.p_nom_opt < line_lower_threshold] = 0. - link_widths_orig = n.links.p_nom / linewidth_factor + link_widths_orig = n.links.p_nom / linewidth_factor link_widths_orig[n.links.p_nom < line_lower_threshold] = 0. max_usage = n.links_t.p0.abs().max(axis=0) @@ -422,7 +430,7 @@ def plot_ch4_map(network): link_colors='lightgrey', link_widths=link_widths_orig, branch_components=["Link"], - ax=ax, + ax=ax, **map_opts ) @@ -452,7 +460,7 @@ def plot_ch4_map(network): facecolor='grey' ) labels = ["{} TWh".format(s) for s in (10, 100)] - + l2 = ax.legend( handles, labels, loc="upper left", @@ -462,7 +470,7 @@ def plot_ch4_map(network): title='gas generation', handler_map=make_handler_map_to_scale_circles_as_in(ax) ) - + ax.add_artist(l2) handles = [] @@ -471,7 +479,7 @@ def plot_ch4_map(network): for s in (50, 10): handles.append(plt.Line2D([0], [0], color="grey", linewidth=s * 1e3 / linewidth_factor)) labels.append("{} GW".format(s)) - + l1_1 = ax.legend( handles, labels, loc="upper left", @@ -481,7 +489,7 @@ def plot_ch4_map(network): handletextpad=1.5, title='gas pipeline used capacity' ) - + ax.add_artist(l1_1) fig.savefig( @@ -695,11 +703,11 @@ if __name__ == "__main__": snakemake = mock_snakemake( 'plot_network', simpl='', - clusters=45, - lv=1.5, + clusters="45", + lv=1.0, opts='', - sector_opts='Co2L0-168H-T-H-B-I-solar+p3-dist1', - planning_horizons=2030, + sector_opts='168H-T-H-B-I-A-solar+p3-dist1', + planning_horizons="2050", ) overrides = override_component_attrs(snakemake.input.overrides) From 598b75358da8b14e24454a1a5864d7c1f7a69ac4 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 24 Jan 2022 12:42:53 +0100 Subject: [PATCH 10/45] Update installation.rst --- doc/installation.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/installation.rst b/doc/installation.rst index a9f353bd..6637cdfd 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -68,14 +68,14 @@ Data requirements ================= Small data files are included directly in the git repository, while -larger ones are archived in a data bundle on zenodo (`10.5281/zenodo.5546517 `_). +larger ones are archived in a data bundle on zenodo (`10.5281/zenodo.5824485 `_). The data bundle's size is around 640 MB. To download and extract the data bundle on the command line: .. code:: bash - projects/pypsa-eur-sec/data % wget "https://zenodo.org/record/5546517/files/pypsa-eur-sec-data-bundle.tar.gz" + projects/pypsa-eur-sec/data % wget "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz" projects/pypsa-eur-sec/data % tar -xvzf pypsa-eur-sec-data-bundle.tar.gz From 0d7ad643bc7fb9c22ced9b9741bbe8383e44893b Mon Sep 17 00:00:00 2001 From: Tim Pedersen Date: Mon, 24 Jan 2022 14:34:54 +0100 Subject: [PATCH 11/45] Resolved error whith /tmp (#224) --- scripts/solve_network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index e23f3437..2346d561 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -288,6 +288,7 @@ if __name__ == "__main__": tmpdir = snakemake.config['solving'].get('tmpdir') if tmpdir is not None: + from pathlib import Path Path(tmpdir).mkdir(parents=True, exist_ok=True) opts = snakemake.wildcards.opts.split('-') solve_opts = snakemake.config['solving']['options'] From 4e3314acd8f2ea90f1d63540ee89a34867c52321 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 25 Jan 2022 12:55:57 +0100 Subject: [PATCH 12/45] bugfix: remove also retrofitted H2 network --- scripts/prepare_sector_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index bd1c7038..9ad969fb 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -2357,7 +2357,7 @@ def decentral(n): def remove_h2_network(n): - n.links.drop(n.links.index[n.links.carrier == "H2 pipeline"], inplace=True) + n.links.drop(n.links.index[n.links.carrier.str.contains("H2 pipeline")], inplace=True) if "EU H2 Store" in n.stores.index: n.stores.drop("EU H2 Store", inplace=True) From b6cfcf6364e1037ce2bdad53504c8df7eb2b72b0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 25 Jan 2022 12:57:04 +0100 Subject: [PATCH 13/45] bugfix: mock_snakemake configfiles --- scripts/helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helper.py b/scripts/helper.py index b176ccee..51411c2e 100644 --- a/scripts/helper.py +++ b/scripts/helper.py @@ -66,7 +66,7 @@ def mock_snakemake(rulename, **wildcards): if os.path.exists(p): snakefile = p break - workflow = sm.Workflow(snakefile) + workflow = sm.Workflow(snakefile, overwrite_configfiles=[]) workflow.include(snakefile) workflow.global_resources = {} rule = workflow.get_rule(rulename) From aed81940b30aa0bd452a2b420a85a9206f8ad07d Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Thu, 17 Mar 2022 18:15:14 +0100 Subject: [PATCH 14/45] add_existing_baseyear.py: Style improvement Co-authored-by: Fabian Neumann --- scripts/add_existing_baseyear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index fae64904..9fbac0ce 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -204,7 +204,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas # to consider electricity grid connection costs or a split between # solar utility and rooftop as well, rather take cost assumptions # from existing network than from the cost database - capital_cost = n.generators[n.generators.carrier==generator+suffix].capital_cost.mean() + capital_cost = n.generators.loc[n.generators.carrier==generator+suffix, "capital_cost"].mean() if 'm' in snakemake.wildcards.clusters: From 5502943301aaa3126a2575a22db21f0bbcaf18ec Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Thu, 17 Mar 2022 18:15:59 +0100 Subject: [PATCH 15/45] scripts/solve_network.py: Style improvement Co-authored-by: Fabian Neumann --- scripts/solve_network.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 5108e2ee..7ea0f064 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -189,9 +189,9 @@ def add_chp_constraints(n): def add_pipe_retrofit_constraint(n): """Add constraint for retrofitting existing CH4 pipelines to H2 pipelines.""" - gas_pipes_i = n.links[n.links.carrier=="gas pipeline"].query("p_nom_extendable").index - h2_retrofitted_i = n.links[n.links.carrier=='H2 pipeline retrofitted'].query("p_nom_extendable").index - h2_retrofitted_fixed_i = n.links[n.links.carrier=='H2 pipeline retrofitted'].index.difference(h2_retrofitted_i) + gas_pipes_i = n.links.query("carrier == 'gas pipeline' and p_nom_extendable").index + h2_retrofitted_i = n.links.query("carrier == 'H2 pipeline retrofitted' and p_nom_extendable").index + h2_retrofitted_fixed_i = n.links.query("carrier == 'H2 pipeline retrofitted' and not p_nom_extendable").index if h2_retrofitted_i.empty or gas_pipes_i.empty: return From 08cef37e7e0c2e3cce74734731ce3fd2a31e284f Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 18 Mar 2022 10:18:02 +0100 Subject: [PATCH 16/45] improve code style --- scripts/plot_network.py | 44 ++++++++++++++++++++++------------------ scripts/solve_network.py | 11 +++++----- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 752459a3..416e4c4e 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -223,6 +223,26 @@ def plot_map(network, components=["links", "stores", "storage_units", "generator bbox_inches="tight" ) +def group_pipes(df, drop_direction=False): + """Group pipes which connect same buses and return overall capacity. + """ + if drop_direction: + positive_order = df.bus0 < df.bus1 + df_p = df[positive_order] + swap_buses = {"bus0": "bus1", "bus1": "bus0"} + df_n = df[~positive_order].rename(columns=swap_buses) + df = pd.concat([df_p, df_n]) + + # there are pipes for each investment period rename to AC buses name for plotting + df.index = df.apply( + lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}", + axis=1 + ) + # group pipe lines connecting the same buses and rename them for plotting + pipe_capacity = df["p_nom_opt"].groupby(level=0).sum() + + return pipe_capacity + def plot_h2_map(network): @@ -246,31 +266,15 @@ def plot_h2_map(network): # make a fake MultiIndex so that area is correct for legend bus_sizes.rename(index=lambda x: x.replace(" H2", ""), level=0, inplace=True) - + # frop all links which are not H2 pipelines n.links.drop(n.links.index[~n.links.carrier.str.contains("H2 pipeline")], inplace=True) h2_new = n.links.loc[n.links.carrier=="H2 pipeline"] - h2_retro = n.links.loc[n.links.carrier=='H2 pipeline retrofitted'] + # sum capacitiy for pipelines from different investment periods + h2_new = group_pipes(h2_new) + h2_retro = group_pipes(h2_retro, drop_direction=True).reindex(h2_new.index).fillna(0) - positive_order = h2_retro.bus0 < h2_retro.bus1 - h2_retro_p = h2_retro[positive_order] - swap_buses = {"bus0": "bus1", "bus1": "bus0"} - h2_retro_n = h2_retro[~positive_order].rename(columns=swap_buses) - h2_retro = pd.concat([h2_retro_p, h2_retro_n]) - - h2_retro.index = h2_retro.apply( - lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}", - axis=1 - ) - - h2_new.index = h2_new.apply( - lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}" - ,axis=1) - - h2_new = h2_new["p_nom_opt"].groupby(level=0).sum() - h2_retro = h2_retro["p_nom_opt"].groupby(level=0).sum() - h2_retro = h2_retro.groupby(level=0).sum().reindex(h2_new.index).fillna(0) n.links.rename(index=lambda x: x.split("-2")[0], inplace=True) n.links = n.links.groupby(level=0).first() diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 7ea0f064..46362ad0 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -185,6 +185,8 @@ def add_chp_constraints(n): define_constraints(n, lhs, "<=", 0, 'chplink', 'backpressure') +def basename(x): + return x.split("-2")[0] def add_pipe_retrofit_constraint(n): """Add constraint for retrofitting existing CH4 pipelines to H2 pipelines.""" @@ -201,10 +203,9 @@ def add_pipe_retrofit_constraint(n): fr = "H2 pipeline retrofitted" to = "gas pipeline" - pipe_capacity = n.links.loc[gas_pipes_i, 'p_nom'].rename(index=lambda x: x.split("-2")[0]) + pipe_capacity = n.links.loc[gas_pipes_i, 'p_nom'].rename(basename) already_retrofitted = (n.links.loc[h2_retrofitted_fixed_i, 'p_nom'] - .rename(index= lambda x: x.split("-2")[0] - .replace(fr, to)).groupby(level=0).sum()) + .rename(lambda x: basename(x).replace(fr, to)).groupby(level=0).sum()) remaining_capacity = pipe_capacity - CH4_per_H2 * already_retrofitted.reindex(index=pipe_capacity.index).fillna(0) lhs = linexpr( @@ -212,7 +213,7 @@ def add_pipe_retrofit_constraint(n): (1, link_p_nom.loc[gas_pipes_i]) ) - lhs.rename(index=lambda x: x.split("-2")[0], inplace=True) + lhs.rename(basename, inplace=True) define_constraints(n, lhs, "=", remaining_capacity, 'Link', 'pipe_retrofit') @@ -284,7 +285,7 @@ if __name__ == "__main__": 'solve_network_myopic', simpl='', opts="", - clusters="45", + clusters="37", lv=1.0, sector_opts='168H-T-H-B-I-A-solar+p3-dist1', planning_horizons="2030", From cfb5a797cbfbd3d14f59b23516a318b2e7d4da4d Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 18 Mar 2022 10:18:24 +0100 Subject: [PATCH 17/45] start to extend define_spatial for conventional carriers --- scripts/add_existing_baseyear.py | 7 +++++-- scripts/prepare_sector_network.py | 31 +++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 9fbac0ce..2ccda05b 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -12,9 +12,11 @@ import xarray as xr import pypsa import yaml -from prepare_sector_network import prepare_costs +from prepare_sector_network import prepare_costs, define_spatial from helper import override_component_attrs +from types import SimpleNamespace +spatial = SimpleNamespace() def add_build_year_to_new_assets(n, baseyear): """ @@ -473,7 +475,8 @@ if __name__ == "__main__": overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) - + # define spatial resolution of carriers + define_spatial(n.buses[n.buses.carrier=="AC"].index, options) add_build_year_to_new_assets(n, baseyear) Nyears = n.snapshot_weightings.generators.sum() / 8760. diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index bd1c7038..1e41a954 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -28,7 +28,7 @@ from types import SimpleNamespace spatial = SimpleNamespace() -def define_spatial(nodes): +def define_spatial(nodes, options): """ Namespace for spatial @@ -38,7 +38,6 @@ def define_spatial(nodes): """ global spatial - global options spatial.nodes = nodes @@ -73,7 +72,7 @@ def define_spatial(nodes): spatial.co2.vents = ["co2 vent"] spatial.co2.df = pd.DataFrame(vars(spatial.co2), index=nodes) - + # gas spatial.gas = SimpleNamespace() @@ -95,6 +94,26 @@ def define_spatial(nodes): spatial.gas.df = pd.DataFrame(vars(spatial.gas), index=nodes) + # oil + spatial.oil = SimpleNamespace() + spatial.oil.nodes = ["EU oil"] + spatial.oil.locations = ["EU"] + + # uranium + spatial.uranium = SimpleNamespace() + spatial.uranium.nodes = ["EU uranium"] + spatial.uranium.locations = ["EU"] + + # coal + spatial.coal = SimpleNamespace() + spatial.coal.nodes = ["EU coal"] + spatial.coal.locations = ["EU"] + + # lignite + spatial.lignite = SimpleNamespace() + spatial.lignite.nodes = ["EU lignite"] + spatial.lignite.locations = ["EU"] + from types import SimpleNamespace spatial = SimpleNamespace() @@ -1049,7 +1068,7 @@ def add_storage_and_grids(n, costs): # only use sites with at least 2 TWh potential h2_caverns = h2_caverns[h2_caverns > 2] - + # convert TWh to MWh h2_caverns = h2_caverns * 1e6 @@ -1119,7 +1138,7 @@ def add_storage_and_grids(n, costs): carrier="gas pipeline", lifetime=costs.at['CH4 (g) pipeline', 'lifetime'] ) - + # remove fossil generators where there is neither # production, LNG terminal, nor entry-point beyond system scope @@ -2438,7 +2457,7 @@ if __name__ == "__main__": patch_electricity_network(n) - define_spatial(pop_layout.index) + define_spatial(pop_layout.index, options) if snakemake.config["foresight"] == 'myopic': From 4f288834b25881bded5747285fda25f8dff79a9f Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 18 Mar 2022 13:46:09 +0100 Subject: [PATCH 18/45] update overrides align with PyPSA version>=0.18 --- data/override_component_attrs/generators.csv | 5 +++-- data/override_component_attrs/links.csv | 16 ++++++++-------- data/override_component_attrs/stores.csv | 4 ++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/data/override_component_attrs/generators.csv b/data/override_component_attrs/generators.csv index bd3925fc..4f214396 100644 --- a/data/override_component_attrs/generators.csv +++ b/data/override_component_attrs/generators.csv @@ -1,3 +1,4 @@ attribute,type,unit,default,description,status -build_year,integer,year,n/a,build year,Input (optional) -lifetime,float,years,n/a,lifetime,Input (optional) +carrier,string,n/a,n/a,carrier,Input (optional) +lifetime,float,years,inf,lifetime,Input (optional) +build_year,int,year ,0,build year,Input (optional) diff --git a/data/override_component_attrs/links.csv b/data/override_component_attrs/links.csv index 709a9211..0fc2747a 100644 --- a/data/override_component_attrs/links.csv +++ b/data/override_component_attrs/links.csv @@ -2,12 +2,12 @@ attribute,type,unit,default,description,status bus2,string,n/a,n/a,2nd bus,Input (optional) bus3,string,n/a,n/a,3rd bus,Input (optional) bus4,string,n/a,n/a,4th bus,Input (optional) -efficiency2,static or series,per unit,1.,2nd bus efficiency,Input (optional) -efficiency3,static or series,per unit,1.,3rd bus efficiency,Input (optional) -efficiency4,static or series,per unit,1.,4th bus efficiency,Input (optional) -p2,series,MW,0.,2nd bus output,Output -p3,series,MW,0.,3rd bus output,Output -p4,series,MW,0.,4th bus output,Output -build_year,integer,year,n/a,build year,Input (optional) -lifetime,float,years,n/a,lifetime,Input (optional) +efficiency2,static or series,per unit,1,2nd bus efficiency,Input (optional) +efficiency3,static or series,per unit,1,3rd bus efficiency,Input (optional) +efficiency4,static or series,per unit,1,4th bus efficiency,Input (optional) +p2,series,MW,0,2nd bus output,Output +p3,series,MW,0,3rd bus output,Output +p4,series,MW,0,4th bus output,Output carrier,string,n/a,n/a,carrier,Input (optional) +lifetime,float,years,inf,lifetime,Input (optional) +build_year,int,year ,0,build year,Input (optional) diff --git a/data/override_component_attrs/stores.csv b/data/override_component_attrs/stores.csv index 1228fea9..4f214396 100644 --- a/data/override_component_attrs/stores.csv +++ b/data/override_component_attrs/stores.csv @@ -1,4 +1,4 @@ attribute,type,unit,default,description,status -build_year,integer,year,n/a,build year,Input (optional) -lifetime,float,years,n/a,lifetime,Input (optional) carrier,string,n/a,n/a,carrier,Input (optional) +lifetime,float,years,inf,lifetime,Input (optional) +build_year,int,year ,0,build year,Input (optional) From 71a8bc6c96d59e77e4b55cb46ec86f0b60bb3df1 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 18 Mar 2022 13:46:40 +0100 Subject: [PATCH 19/45] use define spatial for all conventionals, add lifetime for H2 storage --- scripts/add_brownfield.py | 4 +- scripts/add_existing_baseyear.py | 20 ++++----- scripts/prepare_sector_network.py | 73 ++++++++++++++++--------------- 3 files changed, 47 insertions(+), 50 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 937ba409..294c1ccf 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -88,10 +88,10 @@ if __name__ == "__main__": snakemake = mock_snakemake( 'add_brownfield', simpl='', - clusters="45", + clusters="37", opts="", lv=1.0, - sector_opts='168H-T-H-B-I-A-solar+p3-dist1', + sector_opts='168H-T-H-B-I-solar+p3-dist1', planning_horizons=2030, ) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 2ccda05b..a2ad99ec 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -253,10 +253,9 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas ) else: - bus0 = n.buses[(n.buses.carrier==carrier[generator])].index - if any(n.buses.loc[bus0,"location"]!="EU"): - bus0 = n.buses[n.buses.location.isin(capacity.index) & - (n.buses.carrier==carrier[generator])].index + bus0 = vars(spatial)[carrier[generator]].nodes + if "EU" not in vars(spatial)[carrier[generator]].locations: + bus0 = bus0.intersection(capacity.index + " gas") n.madd("Link", capacity.index, @@ -410,10 +409,7 @@ def add_heating_capacities_installed_before_baseyear(n, baseyear, grouping_years lifetime=costs.at[costs_name, 'lifetime'] ) - bus0 = n.buses[(n.buses.carrier=="gas")].index - if any(n.buses.loc[bus0,"location"]!="EU"): - bus0 = n.buses[n.buses.location.isin(nodal_df[f'{heat_type} gas boiler'][nodes[name]].index) & - (n.buses.carrier=="gas")].index + bus0 = vars(spatial)["gas"].nodes n.madd("Link", nodes[name], @@ -452,17 +448,17 @@ def add_heating_capacities_installed_before_baseyear(n, baseyear, grouping_years threshold = snakemake.config['existing_capacities']['threshold_capacity'] n.mremove("Link", [index for index in n.links.index.to_list() if str(grouping_year) in index and n.links.p_nom[index] < threshold]) - +#%% if __name__ == "__main__": if 'snakemake' not in globals(): from helper import mock_snakemake snakemake = mock_snakemake( 'add_existing_baseyear', simpl='', - clusters="45", + clusters="37", lv=1.0, opts='', - sector_opts='168H-T-H-B-I-A-solar+p3-dist1', + sector_opts='Co2L0-168H-T-H-B-I-solar+p3-dist1', planning_horizons=2020, ) @@ -476,7 +472,7 @@ if __name__ == "__main__": overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) # define spatial resolution of carriers - define_spatial(n.buses[n.buses.carrier=="AC"].index, options) + spatial = define_spatial(n.buses[n.buses.carrier=="AC"].index, options) add_build_year_to_new_assets(n, baseyear) Nyears = n.snapshot_weightings.generators.sum() / 8760. diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 1e41a954..15b9eb5f 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -114,6 +114,8 @@ def define_spatial(nodes, options): spatial.lignite.nodes = ["EU lignite"] spatial.lignite.locations = ["EU"] + return spatial + from types import SimpleNamespace spatial = SimpleNamespace() @@ -371,7 +373,8 @@ def add_carrier_buses(n, carrier, nodes=None): """ if nodes is None: - nodes = ["EU " + carrier] + nodes = vars(spatial)[carrier].nodes + location = vars(spatial)[carrier].locations # skip if carrier already exists if carrier in n.carriers.index: @@ -384,7 +387,7 @@ def add_carrier_buses(n, carrier, nodes=None): n.madd("Bus", nodes, - location=nodes.str.replace(" " + carrier, ""), + location=location, carrier=carrier ) @@ -825,10 +828,8 @@ def add_generation(n, costs): for generator, carrier in conventionals.items(): - if carrier == 'gas': - carrier_nodes = spatial.gas.nodes - else: - carrier_nodes = ["EU " + carrier] + + carrier_nodes = vars(spatial)[carrier].nodes add_carrier_buses(n, carrier, carrier_nodes) @@ -1088,7 +1089,8 @@ def add_storage_and_grids(n, costs): e_nom_max=h2_caverns.values, e_cyclic=True, carrier="H2 Store", - capital_cost=h2_capital_cost + capital_cost=h2_capital_cost, + lifetime=costs.at["hydrogen storage underground", "lifetime"] ) # hydrogen stored overground (where not already underground) @@ -1434,10 +1436,10 @@ def add_land_transport(n, costs): if ice_share > 0: - if "EU oil" not in n.buses.index: - n.add("Bus", - "EU oil", - location="EU", + if "oil" not in n.buses.carrier.unique(): + n.madd("Bus", + vars(spatial)["oil"].nodes, + location=vars(spatial)["oil"].locations, carrier="oil" ) @@ -1446,7 +1448,7 @@ def add_land_transport(n, costs): n.madd("Load", nodes, suffix=" land transport oil", - bus="EU oil", + bus=vars(spatial)["oil"].nodes, carrier="land transport oil", p_set=ice_share / ice_efficiency * transport[nodes] ) @@ -2113,7 +2115,7 @@ def add_industry(n, costs): n.madd("Load", nodes, suffix=" shipping oil", - bus="EU oil", + bus=vars(spatial)["oil"].nodes, carrier="shipping oil", p_set=p_set ) @@ -2127,30 +2129,29 @@ def add_industry(n, costs): p_set=-co2 ) - if "EU oil" not in n.buses.index: - - n.add("Bus", - "EU oil", - location="EU", + if "oil" not in n.buses.carrier.unique(): + n.madd("Bus", + vars(spatial)["oil"].nodes, + location=vars(spatial)["oil"].locations, carrier="oil" ) - if "EU oil Store" not in n.stores.index: + if "oil" not in n.stores.carrier.unique(): #could correct to e.g. 0.001 EUR/kWh * annuity and O&M - n.add("Store", - "EU oil Store", - bus="EU oil", + n.madd("Store", + [oil_bus + " Store" for oil_bus in vars(spatial)["oil"].nodes], + bus=vars(spatial)["oil"].nodes, e_nom_extendable=True, e_cyclic=True, carrier="oil", ) - if "EU oil" not in n.generators.index: + if "oil" not in n.generators.carrier.unique(): - n.add("Generator", - "EU oil", - bus="EU oil", + n.madd("Generator", + vars(spatial)["oil"].nodes, + bus=vars(spatial)["oil"].nodes, p_nom_extendable=True, carrier="oil", marginal_cost=costs.at["oil", 'fuel'] @@ -2165,7 +2166,7 @@ def add_industry(n, costs): n.madd("Link", nodes_heat[name] + f" {name} oil boiler", p_nom_extendable=True, - bus0="EU oil", + bus0=vars(spatial)["oil"].nodes, bus1=nodes_heat[name] + f" {name} heat", bus2="co2 atmosphere", carrier=f"{name} oil boiler", @@ -2178,7 +2179,7 @@ def add_industry(n, costs): n.madd("Link", nodes + " Fischer-Tropsch", bus0=nodes + " H2", - bus1="EU oil", + bus1=vars(spatial)["oil"].nodes, bus2=spatial.co2.nodes, carrier="Fischer-Tropsch", efficiency=costs.at["Fischer-Tropsch", 'efficiency'], @@ -2188,9 +2189,9 @@ def add_industry(n, costs): lifetime=costs.at['Fischer-Tropsch', 'lifetime'] ) - n.add("Load", - "naphtha for industry", - bus="EU oil", + n.madd("Load", + ["naphtha for industry"], + bus=vars(spatial)["oil"].nodes, carrier="naphtha for industry", p_set=industrial_demand.loc[nodes, "naphtha"].sum() / 8760 ) @@ -2198,9 +2199,9 @@ def add_industry(n, costs): all_aviation = ["total international aviation", "total domestic aviation"] p_set = nodal_energy_totals.loc[nodes, all_aviation].sum(axis=1).sum() * 1e6 / 8760 - n.add("Load", - "kerosene for aviation", - bus="EU oil", + n.madd("Load", + ["kerosene for aviation"], + bus=vars(spatial)["oil"].nodes, carrier="kerosene for aviation", p_set=p_set ) @@ -2353,7 +2354,7 @@ def add_agriculture(n, costs): n.add("Load", "agriculture machinery oil", - bus="EU oil", + bus=vars(spatial)["oil"].nodes, carrier="agriculture machinery oil", p_set=ice_share * machinery_nodal_energy.sum() * 1e6 / 8760 ) @@ -2457,7 +2458,7 @@ if __name__ == "__main__": patch_electricity_network(n) - define_spatial(pop_layout.index, options) + spatial = define_spatial(pop_layout.index, options) if snakemake.config["foresight"] == 'myopic': From 03fca360ff0debc7e623c1126499c82c264cafad Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 21 Mar 2022 09:14:15 +0100 Subject: [PATCH 20/45] style improvement --- scripts/add_brownfield.py | 10 ++++++++-- scripts/add_existing_baseyear.py | 7 +++---- scripts/plot_network.py | 2 +- scripts/prepare_sector_network.py | 30 +++++++++++++++--------------- scripts/solve_network.py | 2 +- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 294c1ccf..068ca255 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -78,8 +78,14 @@ def add_brownfield(n, n_p, year): # deal with gas network pipe_carrier = ['gas pipeline'] - to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year!=year) - n.mremove("Link", n.links.loc[to_drop].index) + if snakemake.config["sector"]['H2_retrofit']: + to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year!=year) + n.mremove("Link", n.links.loc[to_drop].index) + else: + new_pipes = n.links.carrier.isin(pipe_carrier) & (n.links.build_year==year) + n.links.loc[new_pipes, "p_nom"] = 0. + n.links.loc[new_pipes, "p_nom_min"] = 0. + #%% if __name__ == "__main__": diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index a2ad99ec..a0acc86f 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -409,12 +409,11 @@ def add_heating_capacities_installed_before_baseyear(n, baseyear, grouping_years lifetime=costs.at[costs_name, 'lifetime'] ) - bus0 = vars(spatial)["gas"].nodes n.madd("Link", nodes[name], suffix= f" {name} gas boiler-{grouping_year}", - bus0=bus0, + bus0=spatial.gas.nodes, bus1=nodes[name] + " " + name + " heat", bus2="co2 atmosphere", carrier=name + " gas boiler", @@ -429,7 +428,7 @@ def add_heating_capacities_installed_before_baseyear(n, baseyear, grouping_years n.madd("Link", nodes[name], suffix=f" {name} oil boiler-{grouping_year}", - bus0="EU oil", + bus0=spatial.oil.nodes, bus1=nodes[name] + " " + name + " heat", bus2="co2 atmosphere", carrier=name + " oil boiler", @@ -458,7 +457,7 @@ if __name__ == "__main__": clusters="37", lv=1.0, opts='', - sector_opts='Co2L0-168H-T-H-B-I-solar+p3-dist1', + sector_opts='168H-T-H-B-I-solar+p3-dist1', planning_horizons=2020, ) diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 416e4c4e..1a56cc4b 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -266,7 +266,7 @@ def plot_h2_map(network): # make a fake MultiIndex so that area is correct for legend bus_sizes.rename(index=lambda x: x.replace(" H2", ""), level=0, inplace=True) - # frop all links which are not H2 pipelines + # drop all links which are not H2 pipelines n.links.drop(n.links.index[~n.links.carrier.str.contains("H2 pipeline")], inplace=True) h2_new = n.links.loc[n.links.carrier=="H2 pipeline"] diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 15b9eb5f..b1b010ce 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1438,8 +1438,8 @@ def add_land_transport(n, costs): if "oil" not in n.buses.carrier.unique(): n.madd("Bus", - vars(spatial)["oil"].nodes, - location=vars(spatial)["oil"].locations, + spatial.oil.nodes, + location=spatial.oil.locations, carrier="oil" ) @@ -1448,7 +1448,7 @@ def add_land_transport(n, costs): n.madd("Load", nodes, suffix=" land transport oil", - bus=vars(spatial)["oil"].nodes, + bus=spatial.oil.nodes, carrier="land transport oil", p_set=ice_share / ice_efficiency * transport[nodes] ) @@ -2115,7 +2115,7 @@ def add_industry(n, costs): n.madd("Load", nodes, suffix=" shipping oil", - bus=vars(spatial)["oil"].nodes, + bus=spatial.oil.nodes, carrier="shipping oil", p_set=p_set ) @@ -2131,8 +2131,8 @@ def add_industry(n, costs): if "oil" not in n.buses.carrier.unique(): n.madd("Bus", - vars(spatial)["oil"].nodes, - location=vars(spatial)["oil"].locations, + spatial.oil.nodes, + location=spatial.oil.locations, carrier="oil" ) @@ -2140,8 +2140,8 @@ def add_industry(n, costs): #could correct to e.g. 0.001 EUR/kWh * annuity and O&M n.madd("Store", - [oil_bus + " Store" for oil_bus in vars(spatial)["oil"].nodes], - bus=vars(spatial)["oil"].nodes, + [oil_bus + " Store" for oil_bus in spatial.oil.nodes], + bus=spatial.oil.nodes, e_nom_extendable=True, e_cyclic=True, carrier="oil", @@ -2150,8 +2150,8 @@ def add_industry(n, costs): if "oil" not in n.generators.carrier.unique(): n.madd("Generator", - vars(spatial)["oil"].nodes, - bus=vars(spatial)["oil"].nodes, + spatial.oil.nodes, + bus=spatial.oil.nodes, p_nom_extendable=True, carrier="oil", marginal_cost=costs.at["oil", 'fuel'] @@ -2166,7 +2166,7 @@ def add_industry(n, costs): n.madd("Link", nodes_heat[name] + f" {name} oil boiler", p_nom_extendable=True, - bus0=vars(spatial)["oil"].nodes, + bus0=spatial.oil.nodes, bus1=nodes_heat[name] + f" {name} heat", bus2="co2 atmosphere", carrier=f"{name} oil boiler", @@ -2179,7 +2179,7 @@ def add_industry(n, costs): n.madd("Link", nodes + " Fischer-Tropsch", bus0=nodes + " H2", - bus1=vars(spatial)["oil"].nodes, + bus1=spatial.oil.nodes, bus2=spatial.co2.nodes, carrier="Fischer-Tropsch", efficiency=costs.at["Fischer-Tropsch", 'efficiency'], @@ -2191,7 +2191,7 @@ def add_industry(n, costs): n.madd("Load", ["naphtha for industry"], - bus=vars(spatial)["oil"].nodes, + bus=spatial.oil.nodes, carrier="naphtha for industry", p_set=industrial_demand.loc[nodes, "naphtha"].sum() / 8760 ) @@ -2201,7 +2201,7 @@ def add_industry(n, costs): n.madd("Load", ["kerosene for aviation"], - bus=vars(spatial)["oil"].nodes, + bus=spatial.oil.nodes, carrier="kerosene for aviation", p_set=p_set ) @@ -2354,7 +2354,7 @@ def add_agriculture(n, costs): n.add("Load", "agriculture machinery oil", - bus=vars(spatial)["oil"].nodes, + bus=spatial.oil.nodes, carrier="agriculture machinery oil", p_set=ice_share * machinery_nodal_energy.sum() * 1e6 / 8760 ) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 46362ad0..5e15386f 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -282,7 +282,7 @@ if __name__ == "__main__": if 'snakemake' not in globals(): from helper import mock_snakemake snakemake = mock_snakemake( - 'solve_network_myopic', + 'solve_network', simpl='', opts="", clusters="37", From b112da0565496d58843464845faef688295a487c Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 3 Apr 2022 18:49:35 +0200 Subject: [PATCH 21/45] prepare: separate code for transport demand and nodal energy totals --- Snakefile | 36 +++- ...build_population_weighted_energy_totals.py | 22 ++ scripts/build_transport_demand.py | 201 +++++++++++++++++ scripts/helper.py | 24 ++- scripts/prepare_sector_network.py | 202 +++--------------- 5 files changed, 312 insertions(+), 173 deletions(-) create mode 100644 scripts/build_population_weighted_energy_totals.py create mode 100644 scripts/build_transport_demand.py diff --git a/Snakefile b/Snakefile index d428db45..8d2c27dd 100644 --- a/Snakefile +++ b/Snakefile @@ -427,15 +427,45 @@ else: build_retro_cost_output = {} +rule build_population_weighted_energy_totals: + input: + energy_totals='resources/energy_totals.csv', + clustered_pop_layout="resources/pop_layout_elec_s{simpl}_{clusters}.csv" + output: "resources/pop_weighted_energy_totals_s{simpl}_{clusters}.csv" + threads: 1 + resources: mem_mb=2000 + script: "scripts/build_population_weighted_energy_totals.py" + + +rule build_transport_demand: + input: + clustered_pop_layout="resources/pop_layout_elec_s{simpl}_{clusters}.csv", + pop_weighted_energy_totals="resources/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", + transport_data='resources/transport_data.csv', + traffic_data_KFZ="data/emobility/KFZ__count", + traffic_data_Pkw="data/emobility/Pkw__count", + temp_air_total="resources/temp_air_total_elec_s{simpl}_{clusters}.nc", + output: + transport_demand="resources/transport_demand_s{simpl}_{clusters}.csv", + transport_data="resources/transport_data_s{simpl}_{clusters}.csv", + avail_profile="resources/avail_profile_s{simpl}_{clusters}.csv", + dsm_profile="resources/dsm_profile_s{simpl}_{clusters}.csv" + threads: 1 + resources: mem_mb=2000 + script: "scripts/build_transport_demand.py" + + rule prepare_sector_network: input: overrides="data/override_component_attrs", network=pypsaeur('networks/elec_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc'), energy_totals_name='resources/energy_totals.csv', + pop_weighted_energy_totals="resources/pop_weighted_energy_totals_s{simpl}_{clusters}.csv", + transport_demand="resources/transport_demand_s{simpl}_{clusters}.csv", + transport_data="resources/transport_data_s{simpl}_{clusters}.csv", + avail_profile="resources/avail_profile_s{simpl}_{clusters}.csv", + dsm_profile="resources/dsm_profile_s{simpl}_{clusters}.csv", co2_totals_name='resources/co2_totals.csv', - transport_name='resources/transport_data.csv', - traffic_data_KFZ="data/emobility/KFZ__count", - traffic_data_Pkw="data/emobility/Pkw__count", biomass_potentials='resources/biomass_potentials_s{simpl}_{clusters}.csv', heat_profile="data/heat_load_profile_BDEW.csv", costs=CDIR + "costs_{planning_horizons}.csv", diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py new file mode 100644 index 00000000..ec75f069 --- /dev/null +++ b/scripts/build_population_weighted_energy_totals.py @@ -0,0 +1,22 @@ +"""Build population-weighted energy totals.""" + +import pandas as pd + +if __name__ == '__main__': + if 'snakemake' not in globals(): + from helper import mock_snakemake + snakemake = mock_snakemake( + 'build_transport_demand', + simpl='', + clusters=48, + ) + + pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) + + energy_totals = pd.read_csv(snakemake.input.energy_totals, index_col=0) + + nodal_energy_totals = energy_totals.loc[pop_layout.ct].fillna(0.) + nodal_energy_totals.index = pop_layout.index + nodal_energy_totals = nodal_energy_totals.multiply(pop_layout.fraction, axis=0) + + nodal_energy_totals.to_csv(snakemake.output[0]) \ No newline at end of file diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py new file mode 100644 index 00000000..a5aabc31 --- /dev/null +++ b/scripts/build_transport_demand.py @@ -0,0 +1,201 @@ +"""Build transport demand.""" + +import pandas as pd +import numpy as np +import xarray as xr +from helper import generate_periodic_profiles + + +def build_nodal_transport_data(fn, pop_layout): + + transport_data = pd.read_csv(fn, index_col=0) + + nodal_transport_data = transport_data.loc[pop_layout.ct].fillna(0.0) + nodal_transport_data.index = pop_layout.index + nodal_transport_data["number cars"] = ( + pop_layout["fraction"] * nodal_transport_data["number cars"] + ) + nodal_transport_data.loc[ + nodal_transport_data["average fuel efficiency"] == 0.0, + "average fuel efficiency", + ] = transport_data["average fuel efficiency"].mean() + + return nodal_transport_data + + +def build_transport_demand(traffic_fn, airtemp_fn, nodes, nodal_transport_data): + + ## Get overall demand curve for all vehicles + + traffic = pd.read_csv( + traffic_fn, skiprows=2, usecols=["count"], squeeze=True + ) + + transport_shape = generate_periodic_profiles( + dt_index=snapshots, + nodes=nodes, + weekly_profile=traffic.values, + ) + transport_shape = transport_shape / transport_shape.sum() + + # electric motors are more efficient, so alter transport demand + + plug_to_wheels_eta = options["bev_plug_to_wheel_efficiency"] + battery_to_wheels_eta = plug_to_wheels_eta * options["bev_charge_efficiency"] + + efficiency_gain = ( + nodal_transport_data["average fuel efficiency"] / battery_to_wheels_eta + ) + + # get heating demand for correction to demand time series + temperature = xr.open_dataarray(airtemp_fn).to_pandas() + + # correction factors for vehicle heating + dd_ICE = transport_degree_factor( + temperature, + options["transport_heating_deadband_lower"], + options["transport_heating_deadband_upper"], + options["ICE_lower_degree_factor"], + options["ICE_upper_degree_factor"], + ) + + dd_EV = transport_degree_factor( + temperature, + options["transport_heating_deadband_lower"], + options["transport_heating_deadband_upper"], + options["EV_lower_degree_factor"], + options["EV_upper_degree_factor"], + ) + + # divide out the heating/cooling demand from ICE totals + # and multiply back in the heating/cooling demand for EVs + ice_correction = (transport_shape * (1 + dd_ICE)).sum() / transport_shape.sum() + + energy_totals_transport = ( + pop_weighted_energy_totals["total road"] + + pop_weighted_energy_totals["total rail"] + - pop_weighted_energy_totals["electricity rail"] + ) + + transport = ( + (transport_shape.multiply(energy_totals_transport) * 1e6 * Nyears) + .divide(efficiency_gain * ice_correction) + .multiply(1 + dd_EV) + ) + + return transport + + +def transport_degree_factor( + temperature, + deadband_lower=15, + deadband_upper=20, + lower_degree_factor=0.5, + upper_degree_factor=1.6, +): + """ + Work out how much energy demand in vehicles increases due to heating and cooling. + There is a deadband where there is no increase. + Degree factors are % increase in demand compared to no heating/cooling fuel consumption. + Returns per unit increase in demand for each place and time + """ + + dd = temperature.copy() + + dd[(temperature > deadband_lower) & (temperature < deadband_upper)] = 0.0 + + dT_lower = deadband_lower - temperature[temperature < deadband_lower] + dd[temperature < deadband_lower] = lower_degree_factor / 100 * dT_lower + + dT_upper = temperature[temperature > deadband_upper] - deadband_upper + dd[temperature > deadband_upper] = upper_degree_factor / 100 * dT_upper + + return dd + + +def bev_availability_profile(fn, snapshots, nodes, options): + """ + Derive plugged-in availability for passenger electric vehicles. + """ + + traffic = pd.read_csv(fn, skiprows=2, usecols=["count"], squeeze=True) + + avail_max = options["bev_avail_max"] + avail_mean = options["bev_avail_mean"] + + avail = avail_max - (avail_max - avail_mean) * (traffic - traffic.min()) / ( + traffic.mean() - traffic.min() + ) + + avail_profile = generate_periodic_profiles( + dt_index=snapshots, + nodes=nodes, + weekly_profile=avail.values, + ) + + return avail_profile + + +def bev_dsm_profile(snapshots, nodes, options): + + dsm_week = np.zeros((24 * 7,)) + + dsm_week[(np.arange(0, 7, 1) * 24 + options["bev_dsm_restriction_time"])] = options[ + "bev_dsm_restriction_value" + ] + + dsm_profile = generate_periodic_profiles( + dt_index=snapshots, + nodes=nodes, + weekly_profile=dsm_week, + ) + + return dsm_profile + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from helper import mock_snakemake + + snakemake = mock_snakemake( + "build_transport_demand", + simpl="", + clusters=48, + ) + + pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) + + nodes = pop_layout.index + + pop_weighted_energy_totals = pd.read_csv( + snakemake.input.pop_weighted_energy_totals, index_col=0 + ) + + options = snakemake.config["sector"] + + snapshots = pd.date_range(freq='h', **snakemake.config["snapshots"], tz="UTC") + + Nyears = 1 + + nodal_transport_data = build_nodal_transport_data( + snakemake.input.transport_data, + pop_layout + ) + + transport_demand = build_transport_demand( + snakemake.input.traffic_data_KFZ, + snakemake.input.temp_air_total, + nodes, nodal_transport_data + ) + + avail_profile = bev_availability_profile( + snakemake.input.traffic_data_Pkw, + snapshots, nodes, options + ) + + dsm_profile = bev_dsm_profile(snapshots, nodes, options) + + nodal_transport_data.to_csv(snakemake.output.transport_data) + transport_demand.to_csv(snakemake.output.transport_demand) + avail_profile.to_csv(snakemake.output.avail_profile) + dsm_profile.to_csv(snakemake.output.dsm_profile) \ No newline at end of file diff --git a/scripts/helper.py b/scripts/helper.py index 51411c2e..bd55a829 100644 --- a/scripts/helper.py +++ b/scripts/helper.py @@ -1,4 +1,5 @@ import os +import pytz import pandas as pd from pathlib import Path from pypsa.descriptors import Dict @@ -100,4 +101,25 @@ def progress_retrieve(url, file): def dlProgress(count, blockSize, totalSize): pbar.update( int(count * blockSize * 100 / totalSize) ) - urllib.request.urlretrieve(url, file, reporthook=dlProgress) \ No newline at end of file + urllib.request.urlretrieve(url, file, reporthook=dlProgress) + + +def generate_periodic_profiles(dt_index, nodes, weekly_profile, localize=None): + """ + Give a 24*7 long list of weekly hourly profiles, generate this for each + country for the period dt_index, taking account of time zones and summer time. + """ + + weekly_profile = pd.Series(weekly_profile, range(24*7)) + + week_df = pd.DataFrame(index=dt_index, columns=nodes) + + for node in nodes: + timezone = pytz.timezone(pytz.country_timezones[node[:2]][0]) + tz_dt_index = dt_index.tz_convert(timezone) + week_df[node] = [24 * dt.weekday() + dt.hour for dt in tz_dt_index] + week_df[node] = week_df[node].map(weekly_profile) + + week_df = week_df.tz_localize(localize) + + return week_df \ No newline at end of file diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 9ad969fb..358b8fbb 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3,7 +3,6 @@ import pypsa import re import os -import pytz import pandas as pd import numpy as np @@ -15,7 +14,7 @@ from scipy.stats import beta from vresutils.costdata import annuity from build_energy_totals import build_eea_co2, build_eurostat_co2, build_co2_totals -from helper import override_component_attrs +from helper import override_component_attrs, generate_periodic_profiles from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation from networkx.algorithms import complement @@ -565,27 +564,6 @@ def average_every_nhours(n, offset): return m -def generate_periodic_profiles(dt_index, nodes, weekly_profile, localize=None): - """ - Give a 24*7 long list of weekly hourly profiles, generate this for each - country for the period dt_index, taking account of time zones and summer time. - """ - - weekly_profile = pd.Series(weekly_profile, range(24*7)) - - week_df = pd.DataFrame(index=dt_index, columns=nodes) - - for node in nodes: - timezone = pytz.timezone(pytz.country_timezones[node[:2]][0]) - tz_dt_index = dt_index.tz_convert(timezone) - week_df[node] = [24 * dt.weekday() + dt.hour for dt in tz_dt_index] - week_df[node] = week_df[node].map(weekly_profile) - - week_df = week_df.tz_localize(localize) - - return week_df - - def cycling_shift(df, steps=1): """Cyclic shift on index of pd.Series|pd.DataFrame by number of steps""" df = df.copy() @@ -594,55 +572,10 @@ def cycling_shift(df, steps=1): return df -def transport_degree_factor( - temperature, - deadband_lower=15, - deadband_upper=20, - lower_degree_factor=0.5, - upper_degree_factor=1.6): - """ - Work out how much energy demand in vehicles increases due to heating and cooling. - There is a deadband where there is no increase. - Degree factors are % increase in demand compared to no heating/cooling fuel consumption. - Returns per unit increase in demand for each place and time - """ - dd = temperature.copy() - - dd[(temperature > deadband_lower) & (temperature < deadband_upper)] = 0. - - dT_lower = deadband_lower - temperature[temperature < deadband_lower] - dd[temperature < deadband_lower] = lower_degree_factor / 100 * dT_lower - - dT_upper = temperature[temperature > deadband_upper] - deadband_upper - dd[temperature > deadband_upper] = upper_degree_factor / 100 * dT_upper - - return dd +def build_heat_demand(n): -# TODO separate sectors and move into own rules -def prepare_data(n): - - - ############## - #Heating - ############## - - - ashp_cop = xr.open_dataarray(snakemake.input.cop_air_total).to_pandas().reindex(index=n.snapshots) - gshp_cop = xr.open_dataarray(snakemake.input.cop_soil_total).to_pandas().reindex(index=n.snapshots) - - solar_thermal = xr.open_dataarray(snakemake.input.solar_thermal_total).to_pandas().reindex(index=n.snapshots) - # 1e3 converts from W/m^2 to MW/(1000m^2) = kW/m^2 - solar_thermal = options['solar_cf_correction'] * solar_thermal / 1e3 - - energy_totals = pd.read_csv(snakemake.input.energy_totals_name, index_col=0) - - nodal_energy_totals = energy_totals.loc[pop_layout.ct].fillna(0.) - nodal_energy_totals.index = pop_layout.index - # district heat share not weighted by population - district_heat_share = nodal_energy_totals["district heat share"].round(2) - nodal_energy_totals = nodal_energy_totals.multiply(pop_layout.fraction, axis=0) # copy forward the daily average heat demand into each hour, so it can be multipled by the intraday profile daily_space_heat_demand = xr.open_dataarray(snakemake.input.heat_demand_total).to_pandas().reindex(index=n.snapshots, method="ffill") @@ -669,8 +602,8 @@ def prepare_data(n): else: heat_demand_shape = intraday_year_profile - heat_demand[f"{sector} {use}"] = (heat_demand_shape/heat_demand_shape.sum()).multiply(nodal_energy_totals[f"total {sector} {use}"]) * 1e6 - electric_heat_supply[f"{sector} {use}"] = (heat_demand_shape/heat_demand_shape.sum()).multiply(nodal_energy_totals[f"electricity {sector} {use}"]) * 1e6 + heat_demand[f"{sector} {use}"] = (heat_demand_shape/heat_demand_shape.sum()).multiply(pop_weighted_energy_totals[f"total {sector} {use}"]) * 1e6 + electric_heat_supply[f"{sector} {use}"] = (heat_demand_shape/heat_demand_shape.sum()).multiply(pop_weighted_energy_totals[f"electricity {sector} {use}"]) * 1e6 heat_demand = pd.concat(heat_demand, axis=1) electric_heat_supply = pd.concat(electric_heat_supply, axis=1) @@ -679,92 +612,7 @@ def prepare_data(n): electric_nodes = n.loads.index[n.loads.carrier == "electricity"] n.loads_t.p_set[electric_nodes] = n.loads_t.p_set[electric_nodes] - electric_heat_supply.groupby(level=1, axis=1).sum()[electric_nodes] - ############## - #Transport - ############## - - ## Get overall demand curve for all vehicles - - traffic = pd.read_csv(snakemake.input.traffic_data_KFZ, skiprows=2, usecols=["count"], squeeze=True) - - #Generate profiles - transport_shape = generate_periodic_profiles( - dt_index=n.snapshots.tz_localize("UTC"), - nodes=pop_layout.index, - weekly_profile=traffic.values - ) - transport_shape = transport_shape / transport_shape.sum() - - transport_data = pd.read_csv(snakemake.input.transport_name, index_col=0) - - nodal_transport_data = transport_data.loc[pop_layout.ct].fillna(0.) - nodal_transport_data.index = pop_layout.index - nodal_transport_data["number cars"] = pop_layout["fraction"] * nodal_transport_data["number cars"] - nodal_transport_data.loc[nodal_transport_data["average fuel efficiency"] == 0., "average fuel efficiency"] = transport_data["average fuel efficiency"].mean() - - - # electric motors are more efficient, so alter transport demand - - plug_to_wheels_eta = options.get("bev_plug_to_wheel_efficiency", 0.2) - battery_to_wheels_eta = plug_to_wheels_eta * options.get("bev_charge_efficiency", 0.9) - - efficiency_gain = nodal_transport_data["average fuel efficiency"] / battery_to_wheels_eta - - #get heating demand for correction to demand time series - temperature = xr.open_dataarray(snakemake.input.temp_air_total).to_pandas() - - # correction factors for vehicle heating - dd_ICE = transport_degree_factor( - temperature, - options['transport_heating_deadband_lower'], - options['transport_heating_deadband_upper'], - options['ICE_lower_degree_factor'], - options['ICE_upper_degree_factor'] - ) - - dd_EV = transport_degree_factor( - temperature, - options['transport_heating_deadband_lower'], - options['transport_heating_deadband_upper'], - options['EV_lower_degree_factor'], - options['EV_upper_degree_factor'] - ) - - # divide out the heating/cooling demand from ICE totals - # and multiply back in the heating/cooling demand for EVs - ice_correction = (transport_shape * (1 + dd_ICE)).sum() / transport_shape.sum() - - energy_totals_transport = nodal_energy_totals["total road"] + nodal_energy_totals["total rail"] - nodal_energy_totals["electricity rail"] - - transport = (transport_shape.multiply(energy_totals_transport) * 1e6 * Nyears).divide(efficiency_gain * ice_correction).multiply(1 + dd_EV) - - ## derive plugged-in availability for PKW's (cars) - - traffic = pd.read_csv(snakemake.input.traffic_data_Pkw, skiprows=2, usecols=["count"], squeeze=True) - - avail_max = options.get("bev_avail_max", 0.95) - avail_mean = options.get("bev_avail_mean", 0.8) - - avail = avail_max - (avail_max - avail_mean) * (traffic - traffic.min()) / (traffic.mean() - traffic.min()) - - avail_profile = generate_periodic_profiles( - dt_index=n.snapshots.tz_localize("UTC"), - nodes=pop_layout.index, - weekly_profile=avail.values - ) - - dsm_week = np.zeros((24*7,)) - - dsm_week[(np.arange(0,7,1) * 24 + options['bev_dsm_restriction_time'])] = options['bev_dsm_restriction_value'] - - dsm_profile = generate_periodic_profiles( - dt_index=n.snapshots.tz_localize("UTC"), - nodes=pop_layout.index, - weekly_profile=dsm_week - ) - - - return nodal_energy_totals, heat_demand, ashp_cop, gshp_cop, solar_thermal, transport, avail_profile, dsm_profile, nodal_transport_data, district_heat_share + return heat_demand # TODO checkout PyPSA-Eur script @@ -1324,6 +1172,11 @@ def add_land_transport(n, costs): logger.info("Add land transport") + transport = pd.read_csv(snakemake.input.transport_demand, index_col=0, parse_dates=True) + number_cars = pd.read_csv(snakemake.input.transport_data, index_col=0)["number cars"] + avail_profile = pd.read_csv(snakemake.input.avail_profile, index_col=0, parse_dates=True) + dsm_profile = pd.read_csv(snakemake.input.dsm_profile, index_col=0, parse_dates=True) + fuel_cell_share = get(options["land_transport_fuel_cell_share"], investment_year) electric_share = get(options["land_transport_electric_share"], investment_year) ice_share = 1 - fuel_cell_share - electric_share @@ -1357,8 +1210,7 @@ def add_land_transport(n, costs): p_set=p_set ) - - p_nom = nodal_transport_data["number cars"] * options.get("bev_charge_rate", 0.011) * electric_share + p_nom = number_cars * options.get("bev_charge_rate", 0.011) * electric_share n.madd("Link", nodes, @@ -1390,7 +1242,7 @@ def add_land_transport(n, costs): if electric_share > 0 and options["bev_dsm"]: - e_nom = nodal_transport_data["number cars"] * options.get("bev_energy", 0.05) * options["bev_availability"] * electric_share + e_nom = number_cars * options.get("bev_energy", 0.05) * options["bev_availability"] * electric_share n.madd("Store", nodes, @@ -1468,6 +1320,15 @@ def add_heat(n, costs): "urban central" ] + cop = { + "air": xr.open_dataarray(snakemake.input.cop_air_total).to_pandas().reindex(index=n.snapshots), + "ground": xr.open_dataarray(snakemake.input.cop_soil_total).to_pandas().reindex(index=n.snapshots) + } + + solar_thermal = xr.open_dataarray(snakemake.input.solar_thermal_total).to_pandas().reindex(index=n.snapshots) + # 1e3 converts from W/m^2 to MW/(1000m^2) = kW/m^2 + solar_thermal = options['solar_cf_correction'] * solar_thermal / 1e3 + for name in heat_systems: name_type = "central" if name == "urban central" else "decentral" @@ -1513,7 +1374,6 @@ def add_heat(n, costs): heat_pump_type = "air" if "urban" in name else "ground" costs_name = f"{name_type} {heat_pump_type}-sourced heat pump" - cop = {"air" : ashp_cop, "ground" : gshp_cop} efficiency = cop[heat_pump_type][nodes[name]] if options["time_dep_hp_cop"] else costs.at[costs_name, 'efficiency'] n.madd("Link", @@ -1792,6 +1652,8 @@ def create_nodes_for_heat_sector(): nodes[sector + " rural"] = pop_layout.index nodes[sector + " urban decentral"] = pop_layout.index + district_heat_share = pop_weighted_energy_totals["district heat share"] + # maximum potential of urban demand covered by district heating central_fraction = options['district_heating']["potential"] # district heating share at each node @@ -2075,7 +1937,7 @@ def add_industry(n, costs): all_navigation = ["total international navigation", "total domestic navigation"] efficiency = options['shipping_average_efficiency'] / costs.at["fuel cell", "efficiency"] shipping_hydrogen_share = get(options['shipping_hydrogen_share'], investment_year) - p_set = shipping_hydrogen_share * nodal_energy_totals.loc[nodes, all_navigation].sum(axis=1) * 1e6 * efficiency / 8760 + p_set = shipping_hydrogen_share * pop_weighted_energy_totals.loc[nodes, all_navigation].sum(axis=1) * 1e6 * efficiency / 8760 n.madd("Load", nodes, @@ -2089,7 +1951,7 @@ def add_industry(n, costs): shipping_oil_share = 1 - shipping_hydrogen_share - p_set = shipping_oil_share * nodal_energy_totals.loc[nodes, all_navigation].sum(axis=1) * 1e6 / 8760. + p_set = shipping_oil_share * pop_weighted_energy_totals.loc[nodes, all_navigation].sum(axis=1) * 1e6 / 8760. n.madd("Load", nodes, @@ -2099,7 +1961,7 @@ def add_industry(n, costs): p_set=p_set ) - co2 = shipping_oil_share * nodal_energy_totals.loc[nodes, all_navigation].sum().sum() * 1e6 / 8760 * costs.at["oil", "CO2 intensity"] + co2 = shipping_oil_share * pop_weighted_energy_totals.loc[nodes, all_navigation].sum().sum() * 1e6 / 8760 * costs.at["oil", "CO2 intensity"] n.add("Load", "shipping oil emissions", @@ -2177,7 +2039,7 @@ def add_industry(n, costs): ) all_aviation = ["total international aviation", "total domestic aviation"] - p_set = nodal_energy_totals.loc[nodes, all_aviation].sum(axis=1).sum() * 1e6 / 8760 + p_set = pop_weighted_energy_totals.loc[nodes, all_aviation].sum(axis=1).sum() * 1e6 / 8760 n.add("Load", "kerosene for aviation", @@ -2297,7 +2159,7 @@ def add_agriculture(n, costs): suffix=" agriculture electricity", bus=nodes, carrier='agriculture electricity', - p_set=nodal_energy_totals.loc[nodes, "total agriculture electricity"] * 1e6 / 8760 + p_set=pop_weighted_energy_totals.loc[nodes, "total agriculture electricity"] * 1e6 / 8760 ) # heat @@ -2307,7 +2169,7 @@ def add_agriculture(n, costs): suffix=" agriculture heat", bus=nodes + " services rural heat", carrier="agriculture heat", - p_set=nodal_energy_totals.loc[nodes, "total agriculture heat"] * 1e6 / 8760 + p_set=pop_weighted_energy_totals.loc[nodes, "total agriculture heat"] * 1e6 / 8760 ) # machinery @@ -2316,7 +2178,7 @@ def add_agriculture(n, costs): assert electric_share <= 1. ice_share = 1 - electric_share - machinery_nodal_energy = nodal_energy_totals.loc[nodes, "total agriculture machinery"] + machinery_nodal_energy = pop_weighted_energy_totals.loc[nodes, "total agriculture machinery"] if electric_share > 0: @@ -2436,6 +2298,8 @@ if __name__ == "__main__": Nyears, snakemake.config['costs']['lifetime']) + pop_weighted_energy_totals = pd.read_csv(snakemake.input.pop_weighted_energy_totals, index_col=0) + patch_electricity_network(n) define_spatial(pop_layout.index) @@ -2466,7 +2330,7 @@ if __name__ == "__main__": if o == "biomasstransport": options["biomass_transport"] = True - nodal_energy_totals, heat_demand, ashp_cop, gshp_cop, solar_thermal, transport, avail_profile, dsm_profile, nodal_transport_data, district_heat_share = prepare_data(n) + heat_demand = build_heat_demand(n) if "nodistrict" in opts: options["district_heating"]["progress"] = 0.0 From 95d5d8f889c19858cb99de9ee86a3c2d0de3ac9a Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 3 Apr 2022 18:55:53 +0200 Subject: [PATCH 22/45] prepare: move build_heat_demand closer to add_heat --- scripts/prepare_sector_network.py | 85 +++++++++++++++---------------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 358b8fbb..60b06dd0 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -573,48 +573,6 @@ def cycling_shift(df, steps=1): -def build_heat_demand(n): - - - - # copy forward the daily average heat demand into each hour, so it can be multipled by the intraday profile - daily_space_heat_demand = xr.open_dataarray(snakemake.input.heat_demand_total).to_pandas().reindex(index=n.snapshots, method="ffill") - - intraday_profiles = pd.read_csv(snakemake.input.heat_profile, index_col=0) - - sectors = ["residential", "services"] - uses = ["water", "space"] - - heat_demand = {} - electric_heat_supply = {} - for sector, use in product(sectors, uses): - weekday = list(intraday_profiles[f"{sector} {use} weekday"]) - weekend = list(intraday_profiles[f"{sector} {use} weekend"]) - weekly_profile = weekday * 5 + weekend * 2 - intraday_year_profile = generate_periodic_profiles( - daily_space_heat_demand.index.tz_localize("UTC"), - nodes=daily_space_heat_demand.columns, - weekly_profile=weekly_profile - ) - - if use == "space": - heat_demand_shape = daily_space_heat_demand * intraday_year_profile - else: - heat_demand_shape = intraday_year_profile - - heat_demand[f"{sector} {use}"] = (heat_demand_shape/heat_demand_shape.sum()).multiply(pop_weighted_energy_totals[f"total {sector} {use}"]) * 1e6 - electric_heat_supply[f"{sector} {use}"] = (heat_demand_shape/heat_demand_shape.sum()).multiply(pop_weighted_energy_totals[f"electricity {sector} {use}"]) * 1e6 - - heat_demand = pd.concat(heat_demand, axis=1) - electric_heat_supply = pd.concat(electric_heat_supply, axis=1) - - # subtract from electricity load since heat demand already in heat_demand - electric_nodes = n.loads.index[n.loads.carrier == "electricity"] - n.loads_t.p_set[electric_nodes] = n.loads_t.p_set[electric_nodes] - electric_heat_supply.groupby(level=1, axis=1).sum()[electric_nodes] - - return heat_demand - - # TODO checkout PyPSA-Eur script def prepare_costs(cost_file, USD_to_EUR, discount_rate, Nyears, lifetime): @@ -1294,12 +1252,53 @@ def add_land_transport(n, costs): ) +def build_heat_demand(n): + + # copy forward the daily average heat demand into each hour, so it can be multipled by the intraday profile + daily_space_heat_demand = xr.open_dataarray(snakemake.input.heat_demand_total).to_pandas().reindex(index=n.snapshots, method="ffill") + + intraday_profiles = pd.read_csv(snakemake.input.heat_profile, index_col=0) + + sectors = ["residential", "services"] + uses = ["water", "space"] + + heat_demand = {} + electric_heat_supply = {} + for sector, use in product(sectors, uses): + weekday = list(intraday_profiles[f"{sector} {use} weekday"]) + weekend = list(intraday_profiles[f"{sector} {use} weekend"]) + weekly_profile = weekday * 5 + weekend * 2 + intraday_year_profile = generate_periodic_profiles( + daily_space_heat_demand.index.tz_localize("UTC"), + nodes=daily_space_heat_demand.columns, + weekly_profile=weekly_profile + ) + + if use == "space": + heat_demand_shape = daily_space_heat_demand * intraday_year_profile + else: + heat_demand_shape = intraday_year_profile + + heat_demand[f"{sector} {use}"] = (heat_demand_shape/heat_demand_shape.sum()).multiply(pop_weighted_energy_totals[f"total {sector} {use}"]) * 1e6 + electric_heat_supply[f"{sector} {use}"] = (heat_demand_shape/heat_demand_shape.sum()).multiply(pop_weighted_energy_totals[f"electricity {sector} {use}"]) * 1e6 + + heat_demand = pd.concat(heat_demand, axis=1) + electric_heat_supply = pd.concat(electric_heat_supply, axis=1) + + # subtract from electricity load since heat demand already in heat_demand + electric_nodes = n.loads.index[n.loads.carrier == "electricity"] + n.loads_t.p_set[electric_nodes] = n.loads_t.p_set[electric_nodes] - electric_heat_supply.groupby(level=1, axis=1).sum()[electric_nodes] + + return heat_demand + + def add_heat(n, costs): logger.info("Add heat sector") sectors = ["residential", "services"] + heat_demand = build_heat_demand(n) nodes, dist_fraction, urban_fraction = create_nodes_for_heat_sector() @@ -2330,8 +2329,6 @@ if __name__ == "__main__": if o == "biomasstransport": options["biomass_transport"] = True - heat_demand = build_heat_demand(n) - if "nodistrict" in opts: options["district_heating"]["progress"] = 0.0 From 6ed92475c95252abe1e9ee04ad8ea1bb04290f81 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 11 Apr 2022 17:08:25 +0200 Subject: [PATCH 23/45] Add basic CI for overnight and myopic (#234) * Snakefile: add missing folders that are retrieved * Snakefile: fix syntax error * Add basic CI * fix datafiles list to work with directories * prepare: only add new gas pipes if augmentation not empty * prepare: handle case where salt caverns dataframe is empty * prepare: handle case where underwater faction not in link columns * build_biomass: handle case where Balkan not in country list * refer correct environment path * fix Snakemake call * make subworkflow dependency explicit again * hashing pypsa-eur environment file does not seem to work * change solver to glpk * add data cache * glpk without options * try cbc * alternative snakemake test workflow * test: add exogenous pathways to config --- .github/workflows/ci.yaml | 109 +++++ Snakefile | 26 +- scripts/build_biomass_potentials.py | 4 +- scripts/prepare_sector_network.py | 54 +-- test/config.myopic.yaml | 607 ++++++++++++++++++++++++++++ test/config.overnight.yaml | 605 +++++++++++++++++++++++++++ 6 files changed, 1369 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 test/config.myopic.yaml create mode 100644 test/config.overnight.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..7d54bd18 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: : 2021 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + +name: CI + +# Caching method based on and described by: +# epassaro (2021): https://dev.to/epassaro/caching-anaconda-environments-in-github-actions-5hde +# and code in GitHub repo: https://github.com/epassaro/cache-conda-envs + +on: + push: + branches: + - master + pull_request: + branches: + - master + schedule: + - cron: "0 5 * * TUE" + +env: + CONDA_CACHE_NUMBER: 1 # Change this value to manually reset the environment cache + DATA_CACHE_NUMBER: 1 + +jobs: + build: + + strategy: + matrix: + include: + # Matrix required to handle caching with Mambaforge + - os: ubuntu-latest + label: ubuntu-latest + prefix: /usr/share/miniconda3/envs/pypsa-eur + + # - os: macos-latest + # label: macos-latest + # prefix: /Users/runner/miniconda3/envs/pypsa-eur + + # - os: windows-latest + # label: windows-latest + # prefix: C:\Miniconda3\envs\pypsa-eur + + name: ${{ matrix.label }} + + runs-on: ${{ matrix.os }} + + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v2 + + - name: Clone pypsa-eur and technology-data repositories + run: | + git clone https://github.com/pypsa/pypsa-eur ../pypsa-eur + git clone https://github.com/pypsa/technology-data ../technology-data + cp ../pypsa-eur/test/config.test1.yaml ../pypsa-eur/config.yaml + + - name: Setup secrets + run: | + echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc + + - name: Add solver to environment + run: | + echo -e " - coincbc\n - ipopt<3.13.3" >> ../pypsa-eur/envs/environment.yaml + + - name: Setup Mambaforge + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + activate-environment: pypsa-eur + use-mamba: true + + - name: Set cache dates + run: | + echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV + echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV + + - name: Cache data and cutouts folders + uses: actions/cache@v3 + with: + path: | + data + ../pypsa-eur/cutouts + ../pypsa-eur/data + key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }} + + - name: Create environment cache + uses: actions/cache@v2 + id: cache + with: + path: ${{ matrix.prefix }} + key: ${{ matrix.label }}-conda-${{ env.DATE }}-${{ env.CONDA_CACHE_NUMBER }} + + - name: Update environment due to outdated or unavailable cache + run: mamba env update -n pypsa-eur -f ../pypsa-eur/envs/environment.yaml + if: steps.cache.outputs.cache-hit != 'true' + + - name: Test snakemake workflow + run: | + conda activate pypsa-eur + conda list + cp test/config.overnight.yaml config.yaml + snakemake -call solve_all_networks + cp test/config.myopic.yaml config.yaml + snakemake -call solve_all_networks diff --git a/Snakefile b/Snakefile index d428db45..024c3a4d 100644 --- a/Snakefile +++ b/Snakefile @@ -45,18 +45,22 @@ rule prepare_sector_networks: **config['scenario']) datafiles = [ - "eea/UNFCCC_v23.csv", - "switzerland-sfoe/switzerland-new_format.csv", - "nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", - "myb1-2017-nitro.xls", - "Industrial_Database.csv", - "emobility/KFZ__count", - "emobility/Pkw__count", + "data/eea/UNFCCC_v23.csv", + "data/switzerland-sfoe/switzerland-new_format.csv", + "data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", + "data/myb1-2017-nitro.xls", + "data/Industrial_Database.csv", + "data/emobility/KFZ__count", + "data/emobility/Pkw__count", + "data/h2_salt_caverns_GWh_per_sqkm.geojson", + directory("data/eurostat-energy_balances-june_2016_edition"), + directory("data/eurostat-energy_balances-may_2018_edition"), + directory("data/jrc-idees-2015"), ] if config.get('retrieve_sector_databundle', True): rule retrieve_sector_databundle: - output: expand('data/{file}', file=datafiles) + output: *datafiles log: "logs/retrieve_sector_databundle.log" script: 'scripts/retrieve_sector_databundle.py' @@ -252,9 +256,9 @@ rule build_biomass_potentials: enspreso_biomass=HTTP.remote("https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", keep_local=True), nuts2="data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 regions_onshore=pypsaeur("resources/regions_onshore_elec_s{simpl}_{clusters}.geojson"), - nuts3_population="../pypsa-eur/data/bundle/nama_10r_3popgdp.tsv.gz", - swiss_cantons="../pypsa-eur/data/bundle/ch_cantons.csv", - swiss_population="../pypsa-eur/data/bundle/je-e-21.03.02.xls", + nuts3_population=pypsaeur("data/bundle/nama_10r_3popgdp.tsv.gz"), + swiss_cantons=pypsaeur("data/bundle/ch_cantons.csv"), + swiss_population=pypsaeur("data/bundle/je-e-21.03.02.xls"), country_shapes=pypsaeur('resources/country_shapes.geojson') output: biomass_potentials_all='resources/biomass_potentials_all_s{simpl}_{clusters}.csv', diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 2b6812f3..c9a2594d 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -144,7 +144,9 @@ def build_nuts2_shapes(): nuts2 = gpd.GeoDataFrame(gpd.read_file(snakemake.input.nuts2).set_index('id').geometry) countries = gpd.read_file(snakemake.input.country_shapes).set_index('name') - missing = countries.loc[["AL", "RS", "BA"]] + missing_iso2 = countries.index.intersection(["AL", "RS", "BA"]) + missing = countries.loc[missing_iso2] + nuts2.rename(index={"ME00": "ME", "MK00": "MK"}, inplace=True) return nuts2.append(missing) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 9ad969fb..56f448fa 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -252,6 +252,7 @@ def create_network_topology(n, prefix, carriers=["DC"], connector=" -> ", bidire ln_attrs = ["bus0", "bus1", "length"] lk_attrs = ["bus0", "bus1", "length", "underwater_fraction"] + lk_attrs = n.links.columns.intersection(lk_attrs) candidates = pd.concat([ n.lines[ln_attrs], @@ -1045,18 +1046,20 @@ def add_storage_and_grids(n, costs): ) cavern_types = snakemake.config["sector"]["hydrogen_underground_storage_locations"] - h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0)[cavern_types].sum(axis=1) - - # only use sites with at least 2 TWh potential - h2_caverns = h2_caverns[h2_caverns > 2] + h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0) - # convert TWh to MWh - h2_caverns = h2_caverns * 1e6 + if not h2_caverns.empty and options['hydrogen_underground_storage']: - # clip at 1000 TWh for one location - h2_caverns.clip(upper=1e9, inplace=True) + h2_caverns = h2_caverns[cavern_types].sum(axis=1) - if options['hydrogen_underground_storage']: + # only use sites with at least 2 TWh potential + h2_caverns = h2_caverns[h2_caverns > 2] + + # convert TWh to MWh + h2_caverns = h2_caverns * 1e6 + + # clip at 1000 TWh for one location + h2_caverns.clip(upper=1e9, inplace=True) logger.info("Add hydrogen underground storage") @@ -1155,23 +1158,26 @@ def add_storage_and_grids(n, costs): # apply k_edge_augmentation weighted by length of complement edges k_edge = options.get("gas_network_connectivity_upgrade", 3) augmentation = k_edge_augmentation(G, k_edge, avail=complement_edges.values) - new_gas_pipes = pd.DataFrame(augmentation, columns=["bus0", "bus1"]) - new_gas_pipes["length"] = new_gas_pipes.apply(haversine, axis=1) - new_gas_pipes.index = new_gas_pipes.apply( - lambda x: f"gas pipeline new {x.bus0} <-> {x.bus1}", axis=1) + if list(augmentation): - n.madd("Link", - new_gas_pipes.index, - bus0=new_gas_pipes.bus0 + " gas", - bus1=new_gas_pipes.bus1 + " gas", - p_min_pu=-1, # new gas pipes are bidirectional - p_nom_extendable=True, - length=new_gas_pipes.length, - capital_cost=new_gas_pipes.length * costs.at['CH4 (g) pipeline', 'fixed'], - carrier="gas pipeline new", - lifetime=costs.at['CH4 (g) pipeline', 'lifetime'] - ) + new_gas_pipes = pd.DataFrame(augmentation, columns=["bus0", "bus1"]) + new_gas_pipes["length"] = new_gas_pipes.apply(haversine, axis=1) + + new_gas_pipes.index = new_gas_pipes.apply( + lambda x: f"gas pipeline new {x.bus0} <-> {x.bus1}", axis=1) + + n.madd("Link", + new_gas_pipes.index, + bus0=new_gas_pipes.bus0 + " gas", + bus1=new_gas_pipes.bus1 + " gas", + p_min_pu=-1, # new gas pipes are bidirectional + p_nom_extendable=True, + length=new_gas_pipes.length, + capital_cost=new_gas_pipes.length * costs.at['CH4 (g) pipeline', 'fixed'], + carrier="gas pipeline new", + lifetime=costs.at['CH4 (g) pipeline', 'lifetime'] + ) if options["H2_retrofit"]: diff --git a/test/config.myopic.yaml b/test/config.myopic.yaml new file mode 100644 index 00000000..1a54b937 --- /dev/null +++ b/test/config.myopic.yaml @@ -0,0 +1,607 @@ +version: 0.6.0 + +logging_level: INFO + +retrieve_sector_databundle: true + +results_dir: results/ +summary_dir: results +costs_dir: ../technology-data/outputs/ +run: test-myopic # use this to keep track of runs with different settings +foresight: myopic # options are overnight, myopic, perfect (perfect is not yet implemented) +# if you use myopic or perfect foresight, set the investment years in "planning_horizons" below + +scenario: + simpl: # only relevant for PyPSA-Eur + - '' + lv: # allowed transmission line volume expansion, can be any float >= 1.0 (today) or "opt" + - 1.5 + clusters: # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred + - 5 + opts: # only relevant for PyPSA-Eur + - '' + sector_opts: # this is where the main scenario settings are + - 191H-T-H-B-I-A-solar+p3-dist1 + # to really understand the options here, look in scripts/prepare_sector_network.py + # Co2Lx specifies the CO2 target in x% of the 1990 values; default will give default (5%); + # Co2L0p25 will give 25% CO2 emissions; Co2Lm0p05 will give 5% negative emissions + # xH is the temporal resolution; 3H is 3-hourly, i.e. one snapshot every 3 hours + # single letters are sectors: T for land transport, H for building heating, + # B for biomass supply, I for industry, shipping and aviation, + # A for agriculture, forestry and fishing + # solar+c0.5 reduces the capital cost of solar to 50\% of reference value + # solar+p3 multiplies the available installable potential by factor 3 + # co2 stored+e2 multiplies the potential of CO2 sequestration by a factor 2 + # dist{n} includes distribution grids with investment cost of n times cost in data/costs.csv + # for myopic/perfect foresight cb states the carbon budget in GtCO2 (cumulative + # emissions throughout the transition path in the timeframe determined by the + # planning_horizons), be:beta decay; ex:exponential decay + # cb40ex0 distributes a carbon budget of 40 GtCO2 following an exponential + # decay with initial growth rate 0 + planning_horizons: # investment years for myopic and perfect; or costs year for overnight + - 2030 + - 2040 + - 2050 + # for example, set to [2020, 2030, 2040, 2050] for myopic foresight + +# CO2 budget as a fraction of 1990 emissions +# this is over-ridden if CO2Lx is set in sector_opts +# this is also over-ridden if cb is set in sector_opts +co2_budget: + 2020: 0.7011648746 + 2025: 0.5241935484 + 2030: 0.2970430108 + 2035: 0.1500896057 + 2040: 0.0712365591 + 2045: 0.0322580645 + 2050: 0 + +# snapshots are originally set in PyPSA-Eur/config.yaml but used again by PyPSA-Eur-Sec +snapshots: + # arguments to pd.date_range + start: "2013-03-01" + end: "2013-04-01" + closed: left # end is not inclusive + +atlite: + cutout: ../pypsa-eur/cutouts/be-03-2013-era5.nc + +# this information is NOT used but needed as an argument for +# pypsa-eur/scripts/add_electricity.py/load_costs in make_summary.py +electricity: + max_hours: + battery: 6 + H2: 168 + +# regulate what components with which carriers are kept from PyPSA-Eur; +# some technologies are removed because they are implemented differently +# (e.g. battery or H2 storage) or have different year-dependent costs +# in PyPSA-Eur-Sec +pypsa_eur: + Bus: + - AC + Link: + - DC + Generator: + - onwind + - offwind-ac + - offwind-dc + - solar + - ror + StorageUnit: + - PHS + - hydro + Store: [] + + +energy: + energy_totals_year: 2011 + base_emissions_year: 1990 + eurostat_report_year: 2016 + emissions: CO2 # "CO2" or "All greenhouse gases - (CO2 equivalent)" + +biomass: + year: 2030 + scenario: ENS_Med + classes: + solid biomass: + - Agricultural waste + - Fuelwood residues + - Secondary Forestry residues - woodchips + - Sawdust + - Residues from landscape care + - Municipal waste + not included: + - Sugar from sugar beet + - Rape seed + - "Sunflower, soya seed " + - Bioethanol barley, wheat, grain maize, oats, other cereals and rye + - Miscanthus, switchgrass, RCG + - Willow + - Poplar + - FuelwoodRW + - C&P_RW + biogas: + - Manure solid, liquid + - Sludge + + +solar_thermal: + clearsky_model: simple # should be "simple" or "enhanced"? + orientation: + slope: 45. + azimuth: 180. + +# only relevant for foresight = myopic or perfect +existing_capacities: + grouping_years: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] + threshold_capacity: 10 + conventional_carriers: + - lignite + - coal + - oil + - uranium + + +sector: + district_heating: + potential: 0.6 # maximum fraction of urban demand which can be supplied by district heating + # increase of today's district heating demand to potential maximum district heating share + # progress = 0 means today's district heating share, progress = 1 means maximum fraction of urban demand is supplied by district heating + progress: + 2020: 0.0 + 2030: 0.3 + 2040: 0.6 + 2050: 1.0 + district_heating_loss: 0.15 + bev_dsm_restriction_value: 0.75 #Set to 0 for no restriction on BEV DSM + bev_dsm_restriction_time: 7 #Time at which SOC of BEV has to be dsm_restriction_value + transport_heating_deadband_upper: 20. + transport_heating_deadband_lower: 15. + ICE_lower_degree_factor: 0.375 #in per cent increase in fuel consumption per degree above deadband + ICE_upper_degree_factor: 1.6 + EV_lower_degree_factor: 0.98 + EV_upper_degree_factor: 0.63 + bev_dsm: true #turns on EV battery + bev_availability: 0.5 #How many cars do smart charging + bev_energy: 0.05 #average battery size in MWh + bev_charge_efficiency: 0.9 #BEV (dis-)charging efficiency + bev_plug_to_wheel_efficiency: 0.2 #kWh/km from EPA https://www.fueleconomy.gov/feg/ for Tesla Model S + bev_charge_rate: 0.011 #3-phase charger with 11 kW + bev_avail_max: 0.95 + bev_avail_mean: 0.8 + v2g: true #allows feed-in to grid from EV battery + #what is not EV or FCEV is oil-fuelled ICE + land_transport_fuel_cell_share: + 2020: 0 + 2030: 0.05 + 2040: 0.1 + 2050: 0.15 + land_transport_electric_share: + 2020: 0 + 2030: 0.25 + 2040: 0.6 + 2050: 0.85 + transport_fuel_cell_efficiency: 0.5 + transport_internal_combustion_efficiency: 0.3 + agriculture_machinery_electric_share: 0 + agriculture_machinery_fuel_efficiency: 0.7 # fuel oil per use + agriculture_machinery_electric_efficiency: 0.3 # electricity per use + shipping_average_efficiency: 0.4 #For conversion of fuel oil to propulsion in 2011 + shipping_hydrogen_liquefaction: false # whether to consider liquefaction costs for shipping H2 demands + shipping_hydrogen_share: + 2020: 0 + 2025: 0 + 2030: 0.05 + 2035: 0.15 + 2040: 0.3 + 2045: 0.6 + 2050: 1 + time_dep_hp_cop: true #time dependent heat pump coefficient of performance + heat_pump_sink_T: 55. # Celsius, based on DTU / large area radiators; used in build_cop_profiles.py + # conservatively high to cover hot water and space heating in poorly-insulated buildings + reduce_space_heat_exogenously: true # reduces space heat demand by a given factor (applied before losses in DH) + # this can represent e.g. building renovation, building demolition, or if + # the factor is negative: increasing floor area, increased thermal comfort, population growth + reduce_space_heat_exogenously_factor: # 0.29 # per unit reduction in space heat demand + # the default factors are determined by the LTS scenario from http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221 + 2020: 0.10 # this results in a space heat demand reduction of 10% + 2025: 0.09 # first heat demand increases compared to 2020 because of larger floor area per capita + 2030: 0.09 + 2035: 0.11 + 2040: 0.16 + 2045: 0.21 + 2050: 0.29 + retrofitting : # co-optimises building renovation to reduce space heat demand + retro_endogen: false # co-optimise space heat savings + cost_factor: 1.0 # weight costs for building renovation + interest_rate: 0.04 # for investment in building components + annualise_cost: true # annualise the investment costs + tax_weighting: false # weight costs depending on taxes in countries + construction_index: true # weight costs depending on labour/material costs per country + tes: true + tes_tau: # 180 day time constant for centralised, 3 day for decentralised + decentral: 3 + central: 180 + boilers: true + oil_boilers: false + chp: true + micro_chp: false + solar_thermal: true + solar_cf_correction: 0.788457 # = >>> 1/1.2683 + marginal_cost_storage: 0. #1e-4 + methanation: true + helmeth: true + dac: true + co2_vent: true + SMR: true + co2_sequestration_potential: 200 #MtCO2/a sequestration potential for Europe + co2_sequestration_cost: 10 #EUR/tCO2 for sequestration of CO2 + co2_network: false + cc_fraction: 0.9 # default fraction of CO2 captured with post-combustion capture + hydrogen_underground_storage: true + hydrogen_underground_storage_locations: + # - onshore # more than 50 km from sea + - nearshore # within 50 km of sea + # - offshore + use_fischer_tropsch_waste_heat: true + use_fuel_cell_waste_heat: true + electricity_distribution_grid: true + electricity_distribution_grid_cost_factor: 1.0 #multiplies cost in data/costs.csv + electricity_grid_connection: true # only applies to onshore wind and utility PV + H2_network: true + gas_network: false + H2_retrofit: false # if set to True existing gas pipes can be retrofitted to H2 pipes + # according to hydrogen backbone strategy (April, 2020) p.15 + # https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf + # 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity + H2_retrofit_capacity_per_CH4: 0.6 # ratio for H2 capacity per original CH4 capacity of retrofitted pipelines + gas_network_connectivity_upgrade: 1 # https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation + gas_distribution_grid: true + gas_distribution_grid_cost_factor: 1.0 #multiplies cost in data/costs.csv + biomass_transport: false # biomass transport between nodes + conventional_generation: # generator : carrier + OCGT: gas + + +industry: + St_primary_fraction: # 0.3 # fraction of steel produced via primary route versus secondary route (scrap+EAF); today fraction is 0.6 + 2020: 0.6 + 2025: 0.55 + 2030: 0.5 + 2035: 0.45 + 2040: 0.4 + 2045: 0.35 + 2050: 0.3 + DRI_fraction: # 1 # fraction of the primary route converted to DRI + EAF + 2020: 0 + 2025: 0 + 2030: 0.05 + 2035: 0.2 + 2040: 0.4 + 2045: 0.7 + 2050: 1 + H2_DRI: 1.7 #H2 consumption in Direct Reduced Iron (DRI), MWh_H2,LHV/ton_Steel from 51kgH2/tSt in Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279 + elec_DRI: 0.322 #electricity consumption in Direct Reduced Iron (DRI) shaft, MWh/tSt HYBRIT brochure https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf + Al_primary_fraction: # 0.2 # fraction of aluminium produced via the primary route versus scrap; today fraction is 0.4 + 2020: 0.4 + 2025: 0.375 + 2030: 0.35 + 2035: 0.325 + 2040: 0.3 + 2045: 0.25 + 2050: 0.2 + MWh_CH4_per_tNH3_SMR: 10.8 # 2012's demand from https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf + MWh_elec_per_tNH3_SMR: 0.7 # same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3 + MWh_H2_per_tNH3_electrolysis: 6.5 # from https://doi.org/10.1016/j.joule.2018.04.017, around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy) + MWh_elec_per_tNH3_electrolysis: 1.17 # from https://doi.org/10.1016/j.joule.2018.04.017 Table 13 (air separation and HB) + NH3_process_emissions: 24.5 # in MtCO2/a from SMR for H2 production for NH3 from UNFCCC for 2015 for EU28 + petrochemical_process_emissions: 25.5 # in MtCO2/a for petrochemical and other from UNFCCC for 2015 for EU28 + HVC_primary_fraction: 1. # fraction of today's HVC produced via primary route + HVC_mechanical_recycling_fraction: 0. # fraction of today's HVC produced via mechanical recycling + HVC_chemical_recycling_fraction: 0. # fraction of today's HVC produced via chemical recycling + HVC_production_today: 52. # MtHVC/a from DECHEMA (2017), Figure 16, page 107; includes ethylene, propylene and BTX + MWh_elec_per_tHVC_mechanical_recycling: 0.547 # from SI of https://doi.org/10.1016/j.resconrec.2020.105010, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756. + MWh_elec_per_tHVC_chemical_recycling: 6.9 # Material Economics (2019), page 125; based on pyrolysis and electric steam cracking + chlorine_production_today: 9.58 # MtCl/a from DECHEMA (2017), Table 7, page 43 + MWh_elec_per_tCl: 3.6 # DECHEMA (2017), Table 6, page 43 + MWh_H2_per_tCl: -0.9372 # DECHEMA (2017), page 43; negative since hydrogen produced in chloralkali process + methanol_production_today: 1.5 # MtMeOH/a from DECHEMA (2017), page 62 + MWh_elec_per_tMeOH: 0.167 # DECHEMA (2017), Table 14, page 65 + MWh_CH4_per_tMeOH: 10.25 # DECHEMA (2017), Table 14, page 65 + hotmaps_locate_missing: false + reference_year: 2015 + # references: + # DECHEMA (2017): https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf + # Material Economics (2019): https://materialeconomics.com/latest-updates/industrial-transformation-2050 + +costs: + lifetime: 25 #default lifetime + # From a Lion Hirth paper, also reflects average of Noothout et al 2016 + discountrate: 0.07 + # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html # noqa: E501 + USD2013_to_EUR2013: 0.7532 + + # Marginal and capital costs can be overwritten + # capital_cost: + # onwind: 500 + marginal_cost: + solar: 0.01 + onwind: 0.015 + offwind: 0.015 + hydro: 0. + H2: 0. + battery: 0. + + emission_prices: # only used with the option Ep (emission prices) + co2: 0. + + lines: + length_factor: 1.25 #to estimate offwind connection costs + + +solving: + #tmpdir: "path/to/tmp" + options: + formulation: kirchhoff + clip_p_max_pu: 1.e-2 + load_shedding: false + noisy_costs: true + skip_iterations: true + track_iterations: false + min_iterations: 4 + max_iterations: 6 + keep_shadowprices: + - Bus + - Line + - Link + - Transformer + - GlobalConstraint + - Generator + - Store + - StorageUnit + + solver: + name: cbc + # threads: 4 + # method: 2 # barrier + # crossover: 0 + # BarConvTol: 1.e-6 + # Seed: 123 + # AggFill: 0 + # PreDual: 0 + # GURO_PAR_BARDENSETHRESH: 200 + #FeasibilityTol: 1.e-6 + + #name: cplex + #threads: 4 + #lpmethod: 4 # barrier + #solutiontype: 2 # non basic solution, ie no crossover + #barrier_convergetol: 1.e-5 + #feasopt_tolerance: 1.e-6 + mem: 4000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 + + +plotting: + map: + boundaries: [-11, 30, 34, 71] + color_geomap: + ocean: white + land: whitesmoke + costs_max: 1000 + costs_threshold: 1 + energy_max: 20000 + energy_min: -20000 + energy_threshold: 50 + vre_techs: + - onwind + - offwind-ac + - offwind-dc + - solar + - ror + renewable_storage_techs: + - PHS + - hydro + conv_techs: + - OCGT + - CCGT + - Nuclear + - Coal + storage_techs: + - hydro+PHS + - battery + - H2 + load_carriers: + - AC load + AC_carriers: + - AC line + - AC transformer + link_carriers: + - DC line + - Converter AC-DC + heat_links: + - heat pump + - resistive heater + - CHP heat + - CHP electric + - gas boiler + - central heat pump + - central resistive heater + - central CHP heat + - central CHP electric + - central gas boiler + heat_generators: + - gas boiler + - central gas boiler + - solar thermal collector + - central solar thermal collector + tech_colors: + # wind + onwind: "#235ebc" + onshore wind: "#235ebc" + offwind: "#6895dd" + offshore wind: "#6895dd" + offwind-ac: "#6895dd" + offshore wind (AC): "#6895dd" + offwind-dc: "#74c6f2" + offshore wind (DC): "#74c6f2" + # water + hydro: '#298c81' + hydro reservoir: '#298c81' + ror: '#3dbfb0' + run of river: '#3dbfb0' + hydroelectricity: '#298c81' + PHS: '#51dbcc' + wave: '#a7d4cf' + # solar + solar: "#f9d002" + solar PV: "#f9d002" + solar thermal: '#ffbf2b' + solar rooftop: '#ffea80' + # gas + OCGT: '#e0986c' + OCGT marginal: '#e0986c' + OCGT-heat: '#e0986c' + gas boiler: '#db6a25' + gas boilers: '#db6a25' + gas boiler marginal: '#db6a25' + gas: '#e05b09' + fossil gas: '#e05b09' + natural gas: '#e05b09' + CCGT: '#a85522' + CCGT marginal: '#a85522' + gas for industry co2 to atmosphere: '#692e0a' + gas for industry co2 to stored: '#8a3400' + gas for industry: '#853403' + gas for industry CC: '#692e0a' + gas pipeline: '#ebbca0' + gas pipeline new: '#a87c62' + # oil + oil: '#c9c9c9' + oil boiler: '#adadad' + agriculture machinery oil: '#949494' + shipping oil: "#808080" + land transport oil: '#afafaf' + # nuclear + Nuclear: '#ff8c00' + Nuclear marginal: '#ff8c00' + nuclear: '#ff8c00' + uranium: '#ff8c00' + # coal + Coal: '#545454' + coal: '#545454' + Coal marginal: '#545454' + solid: '#545454' + Lignite: '#826837' + lignite: '#826837' + Lignite marginal: '#826837' + # biomass + biogas: '#e3d37d' + biomass: '#baa741' + solid biomass: '#baa741' + solid biomass transport: '#baa741' + solid biomass for industry: '#7a6d26' + solid biomass for industry CC: '#47411c' + solid biomass for industry co2 from atmosphere: '#736412' + solid biomass for industry co2 to stored: '#47411c' + # power transmission + lines: '#6c9459' + transmission lines: '#6c9459' + electricity distribution grid: '#97ad8c' + # electricity demand + Electric load: '#110d63' + electric demand: '#110d63' + electricity: '#110d63' + industry electricity: '#2d2a66' + industry new electricity: '#2d2a66' + agriculture electricity: '#494778' + # battery + EVs + battery: '#ace37f' + battery storage: '#ace37f' + home battery: '#80c944' + home battery storage: '#80c944' + BEV charger: '#baf238' + V2G: '#e5ffa8' + land transport EV: '#baf238' + Li ion: '#baf238' + # hot water storage + water tanks: '#e69487' + hot water storage: '#e69487' + hot water charging: '#e69487' + hot water discharging: '#e69487' + # heat demand + Heat load: '#cc1f1f' + heat: '#cc1f1f' + heat demand: '#cc1f1f' + rural heat: '#ff5c5c' + central heat: '#cc1f1f' + decentral heat: '#750606' + low-temperature heat for industry: '#8f2727' + process heat: '#ff0000' + agriculture heat: '#d9a5a5' + # heat supply + heat pumps: '#2fb537' + heat pump: '#2fb537' + air heat pump: '#36eb41' + ground heat pump: '#2fb537' + Ambient: '#98eb9d' + CHP: '#8a5751' + CHP CC: '#634643' + CHP heat: '#8a5751' + CHP electric: '#8a5751' + district heating: '#e8beac' + resistive heater: '#d8f9b8' + retrofitting: '#8487e8' + building retrofitting: '#8487e8' + # hydrogen + H2 for industry: "#f073da" + H2 for shipping: "#ebaee0" + H2: '#bf13a0' + hydrogen: '#bf13a0' + SMR: '#870c71' + SMR CC: '#4f1745' + H2 liquefaction: '#d647bd' + hydrogen storage: '#bf13a0' + H2 storage: '#bf13a0' + land transport fuel cell: '#6b3161' + H2 pipeline: '#f081dc' + H2 pipeline retrofitted: '#ba99b5' + H2 Fuel Cell: '#c251ae' + H2 Electrolysis: '#ff29d9' + # syngas + Sabatier: '#9850ad' + methanation: '#c44ce6' + methane: '#c44ce6' + helmeth: '#e899ff' + # synfuels + Fischer-Tropsch: '#25c49a' + liquid: '#25c49a' + kerosene for aviation: '#a1ffe6' + naphtha for industry: '#57ebc4' + # co2 + CC: '#f29dae' + CCS: '#f29dae' + CO2 sequestration: '#f29dae' + DAC: '#ff5270' + co2 stored: '#f2385a' + co2: '#f29dae' + co2 vent: '#ffd4dc' + CO2 pipeline: '#f5627f' + # emissions + process emissions CC: '#000000' + process emissions: '#222222' + process emissions to stored: '#444444' + process emissions to atmosphere: '#888888' + oil emissions: '#aaaaaa' + shipping oil emissions: "#555555" + land transport oil emissions: '#777777' + agriculture machinery oil emissions: '#333333' + # other + shipping: '#03a2ff' + power-to-heat: '#2fb537' + power-to-gas: '#c44ce6' + power-to-H2: '#ff29d9' + power-to-liquid: '#25c49a' + gas-to-power/heat: '#ee8340' + waste: '#e3d37d' + other: '#000000' diff --git a/test/config.overnight.yaml b/test/config.overnight.yaml new file mode 100644 index 00000000..3764c8fa --- /dev/null +++ b/test/config.overnight.yaml @@ -0,0 +1,605 @@ +version: 0.6.0 + +logging_level: INFO + +retrieve_sector_databundle: true + +results_dir: results/ +summary_dir: results +costs_dir: ../technology-data/outputs/ +run: test-overnight # use this to keep track of runs with different settings +foresight: overnight # options are overnight, myopic, perfect (perfect is not yet implemented) +# if you use myopic or perfect foresight, set the investment years in "planning_horizons" below + +scenario: + simpl: # only relevant for PyPSA-Eur + - '' + lv: # allowed transmission line volume expansion, can be any float >= 1.0 (today) or "opt" + - 1.5 + clusters: # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred + - 5 + opts: # only relevant for PyPSA-Eur + - '' + sector_opts: # this is where the main scenario settings are + - CO2L0-191H-T-H-B-I-A-solar+p3-dist1 + # to really understand the options here, look in scripts/prepare_sector_network.py + # Co2Lx specifies the CO2 target in x% of the 1990 values; default will give default (5%); + # Co2L0p25 will give 25% CO2 emissions; Co2Lm0p05 will give 5% negative emissions + # xH is the temporal resolution; 3H is 3-hourly, i.e. one snapshot every 3 hours + # single letters are sectors: T for land transport, H for building heating, + # B for biomass supply, I for industry, shipping and aviation, + # A for agriculture, forestry and fishing + # solar+c0.5 reduces the capital cost of solar to 50\% of reference value + # solar+p3 multiplies the available installable potential by factor 3 + # co2 stored+e2 multiplies the potential of CO2 sequestration by a factor 2 + # dist{n} includes distribution grids with investment cost of n times cost in data/costs.csv + # for myopic/perfect foresight cb states the carbon budget in GtCO2 (cumulative + # emissions throughout the transition path in the timeframe determined by the + # planning_horizons), be:beta decay; ex:exponential decay + # cb40ex0 distributes a carbon budget of 40 GtCO2 following an exponential + # decay with initial growth rate 0 + planning_horizons: # investment years for myopic and perfect; or costs year for overnight + - 2030 + # for example, set to [2020, 2030, 2040, 2050] for myopic foresight + +# CO2 budget as a fraction of 1990 emissions +# this is over-ridden if CO2Lx is set in sector_opts +# this is also over-ridden if cb is set in sector_opts +co2_budget: + 2020: 0.7011648746 + 2025: 0.5241935484 + 2030: 0.2970430108 + 2035: 0.1500896057 + 2040: 0.0712365591 + 2045: 0.0322580645 + 2050: 0 + +# snapshots are originally set in PyPSA-Eur/config.yaml but used again by PyPSA-Eur-Sec +snapshots: + # arguments to pd.date_range + start: "2013-03-01" + end: "2013-04-01" + closed: left # end is not inclusive + +atlite: + cutout: ../pypsa-eur/cutouts/be-03-2013-era5.nc + +# this information is NOT used but needed as an argument for +# pypsa-eur/scripts/add_electricity.py/load_costs in make_summary.py +electricity: + max_hours: + battery: 6 + H2: 168 + +# regulate what components with which carriers are kept from PyPSA-Eur; +# some technologies are removed because they are implemented differently +# (e.g. battery or H2 storage) or have different year-dependent costs +# in PyPSA-Eur-Sec +pypsa_eur: + Bus: + - AC + Link: + - DC + Generator: + - onwind + - offwind-ac + - offwind-dc + - solar + - ror + StorageUnit: + - PHS + - hydro + Store: [] + + +energy: + energy_totals_year: 2011 + base_emissions_year: 1990 + eurostat_report_year: 2016 + emissions: CO2 # "CO2" or "All greenhouse gases - (CO2 equivalent)" + +biomass: + year: 2030 + scenario: ENS_Med + classes: + solid biomass: + - Agricultural waste + - Fuelwood residues + - Secondary Forestry residues - woodchips + - Sawdust + - Residues from landscape care + - Municipal waste + not included: + - Sugar from sugar beet + - Rape seed + - "Sunflower, soya seed " + - Bioethanol barley, wheat, grain maize, oats, other cereals and rye + - Miscanthus, switchgrass, RCG + - Willow + - Poplar + - FuelwoodRW + - C&P_RW + biogas: + - Manure solid, liquid + - Sludge + + +solar_thermal: + clearsky_model: simple # should be "simple" or "enhanced"? + orientation: + slope: 45. + azimuth: 180. + +# only relevant for foresight = myopic or perfect +existing_capacities: + grouping_years: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] + threshold_capacity: 10 + conventional_carriers: + - lignite + - coal + - oil + - uranium + + +sector: + district_heating: + potential: 0.6 # maximum fraction of urban demand which can be supplied by district heating + # increase of today's district heating demand to potential maximum district heating share + # progress = 0 means today's district heating share, progress = 1 means maximum fraction of urban demand is supplied by district heating + progress: 1 + # 2020: 0.0 + # 2030: 0.3 + # 2040: 0.6 + # 2050: 1.0 + district_heating_loss: 0.15 + bev_dsm_restriction_value: 0.75 #Set to 0 for no restriction on BEV DSM + bev_dsm_restriction_time: 7 #Time at which SOC of BEV has to be dsm_restriction_value + transport_heating_deadband_upper: 20. + transport_heating_deadband_lower: 15. + ICE_lower_degree_factor: 0.375 #in per cent increase in fuel consumption per degree above deadband + ICE_upper_degree_factor: 1.6 + EV_lower_degree_factor: 0.98 + EV_upper_degree_factor: 0.63 + bev_dsm: true #turns on EV battery + bev_availability: 0.5 #How many cars do smart charging + bev_energy: 0.05 #average battery size in MWh + bev_charge_efficiency: 0.9 #BEV (dis-)charging efficiency + bev_plug_to_wheel_efficiency: 0.2 #kWh/km from EPA https://www.fueleconomy.gov/feg/ for Tesla Model S + bev_charge_rate: 0.011 #3-phase charger with 11 kW + bev_avail_max: 0.95 + bev_avail_mean: 0.8 + v2g: true #allows feed-in to grid from EV battery + #what is not EV or FCEV is oil-fuelled ICE + land_transport_fuel_cell_share: 0.15 # 1 means all FCEVs + # 2020: 0 + # 2030: 0.05 + # 2040: 0.1 + # 2050: 0.15 + land_transport_electric_share: 0.85 # 1 means all EVs + # 2020: 0 + # 2030: 0.25 + # 2040: 0.6 + # 2050: 0.85 + transport_fuel_cell_efficiency: 0.5 + transport_internal_combustion_efficiency: 0.3 + agriculture_machinery_electric_share: 0 + agriculture_machinery_fuel_efficiency: 0.7 # fuel oil per use + agriculture_machinery_electric_efficiency: 0.3 # electricity per use + shipping_average_efficiency: 0.4 #For conversion of fuel oil to propulsion in 2011 + shipping_hydrogen_liquefaction: false # whether to consider liquefaction costs for shipping H2 demands + shipping_hydrogen_share: 1 # 1 means all hydrogen FC + # 2020: 0 + # 2025: 0 + # 2030: 0.05 + # 2035: 0.15 + # 2040: 0.3 + # 2045: 0.6 + # 2050: 1 + time_dep_hp_cop: true #time dependent heat pump coefficient of performance + heat_pump_sink_T: 55. # Celsius, based on DTU / large area radiators; used in build_cop_profiles.py + # conservatively high to cover hot water and space heating in poorly-insulated buildings + reduce_space_heat_exogenously: true # reduces space heat demand by a given factor (applied before losses in DH) + # this can represent e.g. building renovation, building demolition, or if + # the factor is negative: increasing floor area, increased thermal comfort, population growth + reduce_space_heat_exogenously_factor: 0.29 # per unit reduction in space heat demand + # the default factors are determined by the LTS scenario from http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221 + # 2020: 0.10 # this results in a space heat demand reduction of 10% + # 2025: 0.09 # first heat demand increases compared to 2020 because of larger floor area per capita + # 2030: 0.09 + # 2035: 0.11 + # 2040: 0.16 + # 2045: 0.21 + # 2050: 0.29 + retrofitting : # co-optimises building renovation to reduce space heat demand + retro_endogen: false # co-optimise space heat savings + cost_factor: 1.0 # weight costs for building renovation + interest_rate: 0.04 # for investment in building components + annualise_cost: true # annualise the investment costs + tax_weighting: false # weight costs depending on taxes in countries + construction_index: true # weight costs depending on labour/material costs per country + tes: true + tes_tau: # 180 day time constant for centralised, 3 day for decentralised + decentral: 3 + central: 180 + boilers: true + oil_boilers: false + chp: true + micro_chp: false + solar_thermal: true + solar_cf_correction: 0.788457 # = >>> 1/1.2683 + marginal_cost_storage: 0. #1e-4 + methanation: true + helmeth: true + dac: true + co2_vent: true + SMR: true + co2_sequestration_potential: 200 #MtCO2/a sequestration potential for Europe + co2_sequestration_cost: 10 #EUR/tCO2 for sequestration of CO2 + co2_network: false + cc_fraction: 0.9 # default fraction of CO2 captured with post-combustion capture + hydrogen_underground_storage: true + hydrogen_underground_storage_locations: + # - onshore # more than 50 km from sea + - nearshore # within 50 km of sea + # - offshore + use_fischer_tropsch_waste_heat: true + use_fuel_cell_waste_heat: true + electricity_distribution_grid: true + electricity_distribution_grid_cost_factor: 1.0 #multiplies cost in data/costs.csv + electricity_grid_connection: true # only applies to onshore wind and utility PV + H2_network: true + gas_network: true + H2_retrofit: true # if set to True existing gas pipes can be retrofitted to H2 pipes + # according to hydrogen backbone strategy (April, 2020) p.15 + # https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf + # 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity + H2_retrofit_capacity_per_CH4: 0.6 # ratio for H2 capacity per original CH4 capacity of retrofitted pipelines + gas_network_connectivity_upgrade: 1 # https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation + gas_distribution_grid: true + gas_distribution_grid_cost_factor: 1.0 #multiplies cost in data/costs.csv + biomass_transport: false # biomass transport between nodes + conventional_generation: # generator : carrier + OCGT: gas + + +industry: + St_primary_fraction: 0.3 # fraction of steel produced via primary route versus secondary route (scrap+EAF); today fraction is 0.6 + # 2020: 0.6 + # 2025: 0.55 + # 2030: 0.5 + # 2035: 0.45 + # 2040: 0.4 + # 2045: 0.35 + # 2050: 0.3 + DRI_fraction: 1 # fraction of the primary route converted to DRI + EAF + # 2020: 0 + # 2025: 0 + # 2030: 0.05 + # 2035: 0.2 + # 2040: 0.4 + # 2045: 0.7 + # 2050: 1 + H2_DRI: 1.7 #H2 consumption in Direct Reduced Iron (DRI), MWh_H2,LHV/ton_Steel from 51kgH2/tSt in Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279 + elec_DRI: 0.322 #electricity consumption in Direct Reduced Iron (DRI) shaft, MWh/tSt HYBRIT brochure https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf + Al_primary_fraction: 0.2 # fraction of aluminium produced via the primary route versus scrap; today fraction is 0.4 + # 2020: 0.4 + # 2025: 0.375 + # 2030: 0.35 + # 2035: 0.325 + # 2040: 0.3 + # 2045: 0.25 + # 2050: 0.2 + MWh_CH4_per_tNH3_SMR: 10.8 # 2012's demand from https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf + MWh_elec_per_tNH3_SMR: 0.7 # same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3 + MWh_H2_per_tNH3_electrolysis: 6.5 # from https://doi.org/10.1016/j.joule.2018.04.017, around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy) + MWh_elec_per_tNH3_electrolysis: 1.17 # from https://doi.org/10.1016/j.joule.2018.04.017 Table 13 (air separation and HB) + NH3_process_emissions: 24.5 # in MtCO2/a from SMR for H2 production for NH3 from UNFCCC for 2015 for EU28 + petrochemical_process_emissions: 25.5 # in MtCO2/a for petrochemical and other from UNFCCC for 2015 for EU28 + HVC_primary_fraction: 1. # fraction of today's HVC produced via primary route + HVC_mechanical_recycling_fraction: 0. # fraction of today's HVC produced via mechanical recycling + HVC_chemical_recycling_fraction: 0. # fraction of today's HVC produced via chemical recycling + HVC_production_today: 52. # MtHVC/a from DECHEMA (2017), Figure 16, page 107; includes ethylene, propylene and BTX + MWh_elec_per_tHVC_mechanical_recycling: 0.547 # from SI of https://doi.org/10.1016/j.resconrec.2020.105010, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756. + MWh_elec_per_tHVC_chemical_recycling: 6.9 # Material Economics (2019), page 125; based on pyrolysis and electric steam cracking + chlorine_production_today: 9.58 # MtCl/a from DECHEMA (2017), Table 7, page 43 + MWh_elec_per_tCl: 3.6 # DECHEMA (2017), Table 6, page 43 + MWh_H2_per_tCl: -0.9372 # DECHEMA (2017), page 43; negative since hydrogen produced in chloralkali process + methanol_production_today: 1.5 # MtMeOH/a from DECHEMA (2017), page 62 + MWh_elec_per_tMeOH: 0.167 # DECHEMA (2017), Table 14, page 65 + MWh_CH4_per_tMeOH: 10.25 # DECHEMA (2017), Table 14, page 65 + hotmaps_locate_missing: false + reference_year: 2015 + # references: + # DECHEMA (2017): https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf + # Material Economics (2019): https://materialeconomics.com/latest-updates/industrial-transformation-2050 + +costs: + lifetime: 25 #default lifetime + # From a Lion Hirth paper, also reflects average of Noothout et al 2016 + discountrate: 0.07 + # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html # noqa: E501 + USD2013_to_EUR2013: 0.7532 + + # Marginal and capital costs can be overwritten + # capital_cost: + # onwind: 500 + marginal_cost: + solar: 0.01 + onwind: 0.015 + offwind: 0.015 + hydro: 0. + H2: 0. + battery: 0. + + emission_prices: # only used with the option Ep (emission prices) + co2: 0. + + lines: + length_factor: 1.25 #to estimate offwind connection costs + + +solving: + #tmpdir: "path/to/tmp" + options: + formulation: kirchhoff + clip_p_max_pu: 1.e-2 + load_shedding: false + noisy_costs: true + skip_iterations: true + track_iterations: false + min_iterations: 4 + max_iterations: 6 + keep_shadowprices: + - Bus + - Line + - Link + - Transformer + - GlobalConstraint + - Generator + - Store + - StorageUnit + + solver: + name: cbc + # threads: 4 + # method: 2 # barrier + # crossover: 0 + # BarConvTol: 1.e-6 + # Seed: 123 + # AggFill: 0 + # PreDual: 0 + # GURO_PAR_BARDENSETHRESH: 200 + #FeasibilityTol: 1.e-6 + + #name: cplex + #threads: 4 + #lpmethod: 4 # barrier + #solutiontype: 2 # non basic solution, ie no crossover + #barrier_convergetol: 1.e-5 + #feasopt_tolerance: 1.e-6 + mem: 4000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 + + +plotting: + map: + boundaries: [-11, 30, 34, 71] + color_geomap: + ocean: white + land: whitesmoke + costs_max: 1000 + costs_threshold: 1 + energy_max: 20000 + energy_min: -20000 + energy_threshold: 50 + vre_techs: + - onwind + - offwind-ac + - offwind-dc + - solar + - ror + renewable_storage_techs: + - PHS + - hydro + conv_techs: + - OCGT + - CCGT + - Nuclear + - Coal + storage_techs: + - hydro+PHS + - battery + - H2 + load_carriers: + - AC load + AC_carriers: + - AC line + - AC transformer + link_carriers: + - DC line + - Converter AC-DC + heat_links: + - heat pump + - resistive heater + - CHP heat + - CHP electric + - gas boiler + - central heat pump + - central resistive heater + - central CHP heat + - central CHP electric + - central gas boiler + heat_generators: + - gas boiler + - central gas boiler + - solar thermal collector + - central solar thermal collector + tech_colors: + # wind + onwind: "#235ebc" + onshore wind: "#235ebc" + offwind: "#6895dd" + offshore wind: "#6895dd" + offwind-ac: "#6895dd" + offshore wind (AC): "#6895dd" + offwind-dc: "#74c6f2" + offshore wind (DC): "#74c6f2" + # water + hydro: '#298c81' + hydro reservoir: '#298c81' + ror: '#3dbfb0' + run of river: '#3dbfb0' + hydroelectricity: '#298c81' + PHS: '#51dbcc' + wave: '#a7d4cf' + # solar + solar: "#f9d002" + solar PV: "#f9d002" + solar thermal: '#ffbf2b' + solar rooftop: '#ffea80' + # gas + OCGT: '#e0986c' + OCGT marginal: '#e0986c' + OCGT-heat: '#e0986c' + gas boiler: '#db6a25' + gas boilers: '#db6a25' + gas boiler marginal: '#db6a25' + gas: '#e05b09' + fossil gas: '#e05b09' + natural gas: '#e05b09' + CCGT: '#a85522' + CCGT marginal: '#a85522' + gas for industry co2 to atmosphere: '#692e0a' + gas for industry co2 to stored: '#8a3400' + gas for industry: '#853403' + gas for industry CC: '#692e0a' + gas pipeline: '#ebbca0' + gas pipeline new: '#a87c62' + # oil + oil: '#c9c9c9' + oil boiler: '#adadad' + agriculture machinery oil: '#949494' + shipping oil: "#808080" + land transport oil: '#afafaf' + # nuclear + Nuclear: '#ff8c00' + Nuclear marginal: '#ff8c00' + nuclear: '#ff8c00' + uranium: '#ff8c00' + # coal + Coal: '#545454' + coal: '#545454' + Coal marginal: '#545454' + solid: '#545454' + Lignite: '#826837' + lignite: '#826837' + Lignite marginal: '#826837' + # biomass + biogas: '#e3d37d' + biomass: '#baa741' + solid biomass: '#baa741' + solid biomass transport: '#baa741' + solid biomass for industry: '#7a6d26' + solid biomass for industry CC: '#47411c' + solid biomass for industry co2 from atmosphere: '#736412' + solid biomass for industry co2 to stored: '#47411c' + # power transmission + lines: '#6c9459' + transmission lines: '#6c9459' + electricity distribution grid: '#97ad8c' + # electricity demand + Electric load: '#110d63' + electric demand: '#110d63' + electricity: '#110d63' + industry electricity: '#2d2a66' + industry new electricity: '#2d2a66' + agriculture electricity: '#494778' + # battery + EVs + battery: '#ace37f' + battery storage: '#ace37f' + home battery: '#80c944' + home battery storage: '#80c944' + BEV charger: '#baf238' + V2G: '#e5ffa8' + land transport EV: '#baf238' + Li ion: '#baf238' + # hot water storage + water tanks: '#e69487' + hot water storage: '#e69487' + hot water charging: '#e69487' + hot water discharging: '#e69487' + # heat demand + Heat load: '#cc1f1f' + heat: '#cc1f1f' + heat demand: '#cc1f1f' + rural heat: '#ff5c5c' + central heat: '#cc1f1f' + decentral heat: '#750606' + low-temperature heat for industry: '#8f2727' + process heat: '#ff0000' + agriculture heat: '#d9a5a5' + # heat supply + heat pumps: '#2fb537' + heat pump: '#2fb537' + air heat pump: '#36eb41' + ground heat pump: '#2fb537' + Ambient: '#98eb9d' + CHP: '#8a5751' + CHP CC: '#634643' + CHP heat: '#8a5751' + CHP electric: '#8a5751' + district heating: '#e8beac' + resistive heater: '#d8f9b8' + retrofitting: '#8487e8' + building retrofitting: '#8487e8' + # hydrogen + H2 for industry: "#f073da" + H2 for shipping: "#ebaee0" + H2: '#bf13a0' + hydrogen: '#bf13a0' + SMR: '#870c71' + SMR CC: '#4f1745' + H2 liquefaction: '#d647bd' + hydrogen storage: '#bf13a0' + H2 storage: '#bf13a0' + land transport fuel cell: '#6b3161' + H2 pipeline: '#f081dc' + H2 pipeline retrofitted: '#ba99b5' + H2 Fuel Cell: '#c251ae' + H2 Electrolysis: '#ff29d9' + # syngas + Sabatier: '#9850ad' + methanation: '#c44ce6' + methane: '#c44ce6' + helmeth: '#e899ff' + # synfuels + Fischer-Tropsch: '#25c49a' + liquid: '#25c49a' + kerosene for aviation: '#a1ffe6' + naphtha for industry: '#57ebc4' + # co2 + CC: '#f29dae' + CCS: '#f29dae' + CO2 sequestration: '#f29dae' + DAC: '#ff5270' + co2 stored: '#f2385a' + co2: '#f29dae' + co2 vent: '#ffd4dc' + CO2 pipeline: '#f5627f' + # emissions + process emissions CC: '#000000' + process emissions: '#222222' + process emissions to stored: '#444444' + process emissions to atmosphere: '#888888' + oil emissions: '#aaaaaa' + shipping oil emissions: "#555555" + land transport oil emissions: '#777777' + agriculture machinery oil emissions: '#333333' + # other + shipping: '#03a2ff' + power-to-heat: '#2fb537' + power-to-gas: '#c44ce6' + power-to-H2: '#ff29d9' + power-to-liquid: '#25c49a' + gas-to-power/heat: '#ee8340' + waste: '#e3d37d' + other: '#000000' From cfdec7e56d651a62e7ba01ae7dc3aa51d47873e1 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 12 Apr 2022 09:56:58 +0200 Subject: [PATCH 24/45] simplify pipe retrofitting constraint --- scripts/add_brownfield.py | 16 ++++++++++++++++ scripts/solve_network.py | 6 +----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 068ca255..23ef1352 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -12,6 +12,7 @@ import numpy as np from add_existing_baseyear import add_build_year_to_new_assets from helper import override_component_attrs +from solve_network import basename def add_brownfield(n, n_p, year): @@ -79,8 +80,23 @@ def add_brownfield(n, n_p, year): # deal with gas network pipe_carrier = ['gas pipeline'] if snakemake.config["sector"]['H2_retrofit']: + # drop capacities of previous year to avoid duplicating to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year!=year) n.mremove("Link", n.links.loc[to_drop].index) + + # subtract the already retrofitted from today's gas grid capacity + h2_retrofitted_fixed_i = n.links[(n.links.carrier=='H2 pipeline retrofitted') & (n.links.build_year!=year)].index + gas_pipes_i = n.links[n.links.carrier.isin(pipe_carrier)].index + CH4_per_H2 = 1 / snakemake.config["sector"]["H2_retrofit_capacity_per_CH4"] + fr = "H2 pipeline retrofitted" + to = "gas pipeline" + # today's pipe capacity + pipe_capacity = n.links.loc[gas_pipes_i, 'p_nom'] + # already retrofitted capacity from gas -> H2 + already_retrofitted = (n.links.loc[h2_retrofitted_fixed_i, 'p_nom'] + .rename(lambda x: basename(x).replace(fr, to)).groupby(level=0).sum()) + remaining_capacity = pipe_capacity - CH4_per_H2 * already_retrofitted.reindex(index=pipe_capacity.index).fillna(0) + n.links.loc[gas_pipes_i, "p_nom"] = remaining_capacity else: new_pipes = n.links.carrier.isin(pipe_carrier) & (n.links.build_year==year) n.links.loc[new_pipes, "p_nom"] = 0. diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 7d2045ed..4f6cc2c4 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -193,7 +193,6 @@ def add_pipe_retrofit_constraint(n): gas_pipes_i = n.links.query("carrier == 'gas pipeline' and p_nom_extendable").index h2_retrofitted_i = n.links.query("carrier == 'H2 pipeline retrofitted' and p_nom_extendable").index - h2_retrofitted_fixed_i = n.links.query("carrier == 'H2 pipeline retrofitted' and not p_nom_extendable").index if h2_retrofitted_i.empty or gas_pipes_i.empty: return @@ -204,9 +203,6 @@ def add_pipe_retrofit_constraint(n): to = "gas pipeline" pipe_capacity = n.links.loc[gas_pipes_i, 'p_nom'].rename(basename) - already_retrofitted = (n.links.loc[h2_retrofitted_fixed_i, 'p_nom'] - .rename(lambda x: basename(x).replace(fr, to)).groupby(level=0).sum()) - remaining_capacity = pipe_capacity - CH4_per_H2 * already_retrofitted.reindex(index=pipe_capacity.index).fillna(0) lhs = linexpr( (CH4_per_H2, link_p_nom.loc[h2_retrofitted_i].rename(index=lambda x: x.replace(fr, to))), @@ -214,7 +210,7 @@ def add_pipe_retrofit_constraint(n): ) lhs.rename(basename, inplace=True) - define_constraints(n, lhs, "=", remaining_capacity, 'Link', 'pipe_retrofit') + define_constraints(n, lhs, "=", pipe_capacity, 'Link', 'pipe_retrofit') def add_co2_sequestration_limit(n, sns): From 9322f90318cac0584dad44144a423aa95abd3dcd Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 12 Apr 2022 10:03:04 +0200 Subject: [PATCH 25/45] add minimum capacity for AC and DC lines depending on previous year --- scripts/add_brownfield.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 23ef1352..d7418a79 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -19,6 +19,11 @@ def add_brownfield(n, n_p, year): print("adding brownfield") + # electric transmission grid set optimised capacities of previous as minimum + n.lines.s_nom_min = n_p.lines.s_nom_opt + dc_i = n.links[n.links.carrier=="DC"].index + n.links.loc[dc_i, "p_nom_min"] = n_p.links.loc[dc_i, "p_nom_opt"] + for c in n_p.iterate_components(["Link", "Generator", "Store"]): attr = "e" if c.name == "Store" else "p" @@ -103,6 +108,7 @@ def add_brownfield(n, n_p, year): n.links.loc[new_pipes, "p_nom_min"] = 0. + #%% if __name__ == "__main__": if 'snakemake' not in globals(): From 408b494612216fa9b16e06c1b418a12229960b61 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 12 Apr 2022 10:45:11 +0200 Subject: [PATCH 26/45] bug fix with augmentation object --- scripts/prepare_sector_network.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 7017e0b9..018981c0 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1067,14 +1067,14 @@ def add_storage_and_grids(n, costs): cavern_types = snakemake.config["sector"]["hydrogen_underground_storage_locations"] h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0) - + if not h2_caverns.empty and options['hydrogen_underground_storage']: h2_caverns = h2_caverns[cavern_types].sum(axis=1) # only use sites with at least 2 TWh potential h2_caverns = h2_caverns[h2_caverns > 2] - + # convert TWh to MWh h2_caverns = h2_caverns * 1e6 @@ -1178,9 +1178,9 @@ def add_storage_and_grids(n, costs): # apply k_edge_augmentation weighted by length of complement edges k_edge = options.get("gas_network_connectivity_upgrade", 3) - augmentation = k_edge_augmentation(G, k_edge, avail=complement_edges.values) + augmentation = list(k_edge_augmentation(G, k_edge, avail=complement_edges.values)) - if list(augmentation): + if augmentation: new_gas_pipes = pd.DataFrame(augmentation, columns=["bus0", "bus1"]) new_gas_pipes["length"] = new_gas_pipes.apply(haversine, axis=1) From 1625d9db75244d1951a5cf9cc0c395452ddc2438 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 12 Apr 2022 14:37:05 +0200 Subject: [PATCH 27/45] address recent deprecations (#235) * address recent deprecations * address recent deprecations in pd.read_csv --- scripts/add_existing_baseyear.py | 4 ++-- scripts/build_biomass_potentials.py | 2 +- scripts/build_gas_input_locations.py | 2 +- scripts/build_industrial_production_per_country.py | 6 +++--- scripts/build_industry_sector_ratios.py | 3 +-- scripts/build_population_layouts.py | 2 +- scripts/prepare_sector_network.py | 11 +++++------ 7 files changed, 14 insertions(+), 16 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index bb35e378..13863e10 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -153,8 +153,8 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas df_agg.Fueltype = df_agg.Fueltype.map(rename_fuel) # assign clustered bus - busmap_s = pd.read_csv(snakemake.input.busmap_s, index_col=0, squeeze=True) - busmap = pd.read_csv(snakemake.input.busmap, index_col=0, squeeze=True) + busmap_s = pd.read_csv(snakemake.input.busmap_s, index_col=0).squeeze() + busmap = pd.read_csv(snakemake.input.busmap, index_col=0).squeeze() inv_busmap = {} for k, v in busmap.iteritems(): diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index c9a2594d..c80c6b46 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -149,7 +149,7 @@ def build_nuts2_shapes(): nuts2.rename(index={"ME00": "ME", "MK00": "MK"}, inplace=True) - return nuts2.append(missing) + return pd.concat([nuts2, missing]) def area(gdf): diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index 1b823d6d..c08d92de 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -26,7 +26,7 @@ def build_gas_input_locations(lng_fn, planned_lng_fn, entry_fn, prod_fn, countri planned_lng = pd.read_csv(planned_lng_fn) planned_lng.geometry = planned_lng.geometry.apply(wkt.loads) planned_lng = gpd.GeoDataFrame(planned_lng, crs=4326) - lng = lng.append(planned_lng, ignore_index=True) + lng = pd.concat([lng, planned_lng], ignore_index=True) # Entry points from outside the model scope entry = read_scigrid_gas(entry_fn) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index eadfb224..f21b711a 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -115,14 +115,14 @@ def get_energy_ratio(country): # estimate physical output, energy consumption in the sector and country fn = f"{eurostat_dir}/{eb_names[country]}.XLSX" df = pd.read_excel(fn, sheet_name='2016', index_col=2, - header=0, skiprows=1, squeeze=True) + header=0, skiprows=1).squeeze('columns') e_country = df.loc[eb_sectors.keys( ), 'Total all products'].rename(eb_sectors) fn = f'{jrc_dir}/JRC-IDEES-2015_Industry_EU28.xlsx' df = pd.read_excel(fn, sheet_name='Ind_Summary', - index_col=0, header=0, squeeze=True) + index_col=0, header=0).squeeze('columns') assert df.index[48] == "by sector" year_i = df.columns.get_loc(year) @@ -142,7 +142,7 @@ def industry_production_per_country(country): fn = f'{jrc_dir}/JRC-IDEES-2015_Industry_{jrc_country}.xlsx' sheet = sub_sheet_name_dict[sector] df = pd.read_excel(fn, sheet_name=sheet, - index_col=0, header=0, squeeze=True) + index_col=0, header=0).squeeze('columns') year_i = df.columns.get_loc(year) df = df.iloc[find_physical_output(df), year_i] diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index a6ef9066..16c9de9e 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -78,9 +78,8 @@ def load_idees_data(sector, country="EU28"): sheet_name=list(sheets.values()), index_col=0, header=0, - squeeze=True, usecols=usecols, - ) + ).squeeze('columns') for k, v in sheets.items(): idees[k] = idees.pop(v) diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index 6c229797..fa5c7d28 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -33,7 +33,7 @@ if __name__ == '__main__': urban_fraction = pd.read_csv(snakemake.input.urban_percent, header=None, index_col=0, - names=['fraction'], squeeze=True) / 100. + names=['fraction']).squeeze() / 100. # fill missing Balkans values missing = ["AL", "ME", "MK"] diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 56f448fa..f81516a9 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -279,7 +279,7 @@ def create_network_topology(n, prefix, carriers=["DC"], connector=" -> ", bidire topo_reverse = topo.copy() topo_reverse.rename(columns=swap_buses, inplace=True) topo_reverse.index = topo_reverse.apply(make_index, axis=1) - topo = topo.append(topo_reverse) + topo = pd.concat([topo, topo_reverse]) return topo @@ -686,7 +686,7 @@ def prepare_data(n): ## Get overall demand curve for all vehicles - traffic = pd.read_csv(snakemake.input.traffic_data_KFZ, skiprows=2, usecols=["count"], squeeze=True) + traffic = pd.read_csv(snakemake.input.traffic_data_KFZ, skiprows=2, usecols=["count"]).squeeze() #Generate profiles transport_shape = generate_periodic_profiles( @@ -741,7 +741,7 @@ def prepare_data(n): ## derive plugged-in availability for PKW's (cars) - traffic = pd.read_csv(snakemake.input.traffic_data_Pkw, skiprows=2, usecols=["count"], squeeze=True) + traffic = pd.read_csv(snakemake.input.traffic_data_Pkw, skiprows=2, usecols=["count"]).squeeze() avail_max = options.get("bev_avail_max", 0.95) avail_mean = options.get("bev_avail_mean", 0.8) @@ -1888,8 +1888,7 @@ def add_biomass(n, costs): transport_costs = pd.read_csv( snakemake.input.biomass_transport_costs, index_col=0, - squeeze=True - ) + ).squeeze() # add biomass transport biomass_transport = create_network_topology(n, "biomass transport ", bidirectional=False) @@ -2521,7 +2520,7 @@ if __name__ == "__main__": fn = snakemake.config['results_dir'] + snakemake.config['run'] + '/csvs/carbon_budget_distribution.csv' if not os.path.exists(fn): build_carbon_budget(o, fn) - co2_cap = pd.read_csv(fn, index_col=0, squeeze=True) + co2_cap = pd.read_csv(fn, index_col=0).squeeze() limit = co2_cap[investment_year] break for o in opts: From c54b4f992023dfb7f28edc9cea7930a15a99044e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 12 Apr 2022 15:16:05 +0200 Subject: [PATCH 28/45] copy_config: copy actual snakemake.config --- scripts/copy_config.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/copy_config.py b/scripts/copy_config.py index ee1ca3f5..6eaf6e66 100644 --- a/scripts/copy_config.py +++ b/scripts/copy_config.py @@ -1,5 +1,6 @@ from shutil import copy +import yaml files = { "config.yaml": "config.yaml", @@ -14,5 +15,16 @@ if __name__ == '__main__': from helper import mock_snakemake snakemake = mock_snakemake('copy_config') + basepath = snakemake.config['summary_dir'] + '/' + snakemake.config['run'] + '/configs/' + for f, name in files.items(): - copy(f,snakemake.config['summary_dir'] + '/' + snakemake.config['run'] + '/configs/' + name) + copy(f, basepath + name) + + with open(basepath + 'config.snakemake.yaml', 'w') as yaml_file: + yaml.dump( + snakemake.config, + yaml_file, + default_flow_style=False, + allow_unicode=True, + sort_keys=False + ) \ No newline at end of file From ae28839cca3a1f83499a3798ed9a6ccb0a1853c6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 12 Apr 2022 15:23:48 +0200 Subject: [PATCH 29/45] CI: include summary and plotting rules --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7d54bd18..f528c28d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -104,6 +104,6 @@ jobs: conda activate pypsa-eur conda list cp test/config.overnight.yaml config.yaml - snakemake -call solve_all_networks + snakemake -call cp test/config.myopic.yaml config.yaml - snakemake -call solve_all_networks + snakemake -call From c5ca8f4c20ada5c987f36f4bba247f19069166d7 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 12 Apr 2022 16:11:06 +0200 Subject: [PATCH 30/45] fix: squeeze IDEES dataframe at correct location --- scripts/build_industry_sector_ratios.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index 16c9de9e..c8cac055 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -79,10 +79,10 @@ def load_idees_data(sector, country="EU28"): index_col=0, header=0, usecols=usecols, - ).squeeze('columns') + ) for k, v in sheets.items(): - idees[k] = idees.pop(v) + idees[k] = idees.pop(v).squeeze() return idees From cdd56288ff73eb6626b7377f1170ba585c5ff939 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 12 Apr 2022 16:21:09 +0200 Subject: [PATCH 31/45] fix EU bus location with new config setting --- config.default.yaml | 3 +++ scripts/plot_network.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 5340682b..8e39affe 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -388,6 +388,9 @@ plotting: color_geomap: ocean: white land: whitesmoke + eu_node_location: + x: -5.5 + y: 46. costs_max: 1000 costs_threshold: 1 energy_max: 20000 diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 1a56cc4b..4a1bc6d0 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -115,7 +115,9 @@ def plot_map(network, components=["links", "stores", "storage_units", "generator costs = costs.stack() # .sort_index() # hack because impossible to drop buses... - n.buses.loc["EU gas", ["x", "y"]] = n.buses.loc["DE0 0", ["x", "y"]] + eu_location = snakemake.config["plotting"].get("eu_node_location", dict(x=-5.5, y=46)) + n.buses.loc["EU gas", "x"] = eu_location["x"] + n.buses.loc["EU gas", "y"] = eu_location["y"] n.links.drop(n.links.index[(n.links.carrier != "DC") & ( n.links.carrier != "B2B")], inplace=True) @@ -524,7 +526,9 @@ def plot_map_without(network): # hack because impossible to drop buses... if "EU gas" in n.buses.index: - n.buses.loc["EU gas", ["x", "y"]] = n.buses.loc["DE0 0", ["x", "y"]] + eu_location = snakemake.config["plotting"].get("eu_node_location", dict(x=-5.5, y=46)) + n.buses.loc["EU gas", "x"] = eu_location["x"] + n.buses.loc["EU gas", "y"] = eu_location["y"] to_drop = n.links.index[(n.links.carrier != "DC") & (n.links.carrier != "B2B")] n.links.drop(to_drop, inplace=True) From dc997c6ff2dbacdc551e010e3f419ab83fe7a20a Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Mon, 2 May 2022 18:19:59 +0200 Subject: [PATCH 32/45] Update build_population_weighted_energy_totals.py adjust mocksnakemake --- scripts/build_population_weighted_energy_totals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index ec75f069..938983d5 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -6,7 +6,7 @@ if __name__ == '__main__': if 'snakemake' not in globals(): from helper import mock_snakemake snakemake = mock_snakemake( - 'build_transport_demand', + 'build_population_weighted_energy_totals', simpl='', clusters=48, ) @@ -19,4 +19,4 @@ if __name__ == '__main__': nodal_energy_totals.index = pop_layout.index nodal_energy_totals = nodal_energy_totals.multiply(pop_layout.fraction, axis=0) - nodal_energy_totals.to_csv(snakemake.output[0]) \ No newline at end of file + nodal_energy_totals.to_csv(snakemake.output[0]) From aa2820c9b18e6b832a4b3786d12a2ba2fa707ffe Mon Sep 17 00:00:00 2001 From: Irieo Date: Tue, 31 May 2022 16:16:40 +0200 Subject: [PATCH 33/45] fix grouping years list issue caused by newer powerplant database --- config.default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.default.yaml b/config.default.yaml index 8e39affe..d48879c6 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -134,7 +134,7 @@ solar_thermal: # only relevant for foresight = myopic or perfect existing_capacities: - grouping_years: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] + grouping_years: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030] threshold_capacity: 10 conventional_carriers: - lignite From 292d71381794e991964739a5212346c888b177f2 Mon Sep 17 00:00:00 2001 From: Tom Brown Date: Fri, 3 Jun 2022 12:33:33 +0200 Subject: [PATCH 34/45] README,doc: soften warnings, add limitations section to doc --- README.md | 27 ++++++++++---------- doc/index.rst | 27 ++++++++++---------- doc/limitations.rst | 61 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 26 deletions(-) create mode 100644 doc/limitations.rst diff --git a/README.md b/README.md index b6929873..5af6f3e4 100644 --- a/README.md +++ b/README.md @@ -7,19 +7,8 @@ # PyPSA-Eur-Sec: A Sector-Coupled Open Optimisation Model of the European Energy System - - -**WARNING**: This model is under construction and contains serious problems that -distort the results. See the github repository -[issues](https://github.com/PyPSA/pypsa-eur-sec/issues) for some of the problems -(please feel free to help or make suggestions). There is neither a full -documentation nor a paper yet, but we hope to have a preprint out by the end of 2021. -You can find out more about the model capabilities in [a recent -presentation at EMP-E](https://nworbmot.org/energy/brown-empe.pdf) or the -following [preprint with a description of the industry -sector](https://arxiv.org/abs/2109.09563). We cannot support this model if you -choose to use it. - +PyPSA-Eur-Sec is an open model dataset of the European energy system at the +transmission network level that covers the full ENTSO-E area. PyPSA-Eur-Sec builds on the electricity generation and transmission model [PyPSA-Eur](https://github.com/PyPSA/pypsa-eur) to add demand @@ -28,6 +17,18 @@ heating, biomass, industry and industrial feedstocks, agriculture, forestry and fishing. This completes the energy system and includes all greenhouse gas emitters except waste management and land use. +**WARNING**: PyPSA-Eur-Sec is under active development and has several +[limitations](https://pypsa-eur-sec.readthedocs.io/en/latest/limitations.html) which +you should understand before using the model. The github repository +[issues](https://github.com/PyPSA/pypsa-eur-sec/issues) collects known +topics we are working on (please feel free to help or make suggestions). There is neither a full +documentation nor a paper yet, but we hope to have a preprint out by mid-2022. +You can find out more about the model capabilities in [a recent +presentation at EMP-E](https://nworbmot.org/energy/brown-empe.pdf) or the +following [paper in Joule with a description of the industry +sector](https://arxiv.org/abs/2109.09563). We cannot support this model if you +choose to use it. + Please see the [documentation](https://pypsa-eur-sec.readthedocs.io/) for installation instructions and other useful information about the snakemake workflow. diff --git a/doc/index.rst b/doc/index.rst index c4174b52..ad43c66b 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -29,10 +29,21 @@ heating, biomass, industry and industrial feedstocks. This completes the energy system and includes all greenhouse gas emitters except waste management, agriculture, forestry and land use. + +**WARNING**: PyPSA-Eur-Sec is under active development and has several +`limitations `_ which +you should understand before using the model. The github repository +`issues `_ collects known +topics we are working on (please feel free to help or make suggestions). There is neither a full +documentation nor a paper yet, but we hope to have a preprint out by mid-2022. +We cannot support this model if you +choose to use it. + + .. note:: More about the current model capabilities and preliminary results can be found in `a recent presentation at EMP-E `_ - and the the following `preprint with a description of the industry sector `_. + and the following `paper in Joule with a description of the industry sector `_. This diagram gives an overview of the sectors and the links between them: @@ -131,6 +142,7 @@ Documentation **References** * :doc:`release_notes` +* :doc:`limitations` .. toctree:: :hidden: @@ -138,18 +150,7 @@ Documentation :caption: References release_notes - - -Warnings -======== - -**WARNING**: This model is under construction and contains serious -problems that distort the results. See the github repository -`issues `_ for some of -the problems (please feel free to help or make suggestions). There is -neither documentation nor a paper yet, but we hope to have a preprint -out by summer 2020. We cannot support this model if you choose to use -it. + limitations Licence diff --git a/doc/limitations.rst b/doc/limitations.rst new file mode 100644 index 00000000..9f02d763 --- /dev/null +++ b/doc/limitations.rst @@ -0,0 +1,61 @@ +########################################## +Limitations +########################################## + +While the benefit of an openly available, functional and partially validated +model of the European energy system is high, many approximations have +been made due to missing data. +The limitations of the dataset are listed below, +both as a warning to the user and as an encouragement to assist in +improving the approximations. + +This list of limitations is incomplete and will be added to over time. + +See also the `GitHub repository issues `_. + +- **Electricity transmission network topology:** + The grid data is based on a map of the ENTSO-E area that is known + to contain small distortions to improve readability. Since the exact impedances + of the lines are unknown, approximations based on line lengths and standard + line parameters were made that ignore specific conductoring choices for + particular lines. There is no openly available data on busbar configurations, switch + locations, transformers or reactive power compensation assets. + +- **Assignment of electricity demand to transmission nodes:** + Using Voronoi cells to aggregate load and generator data to transmission + network substations ignores the topology of the underlying distribution network, + meaning that assets may be connected to the wrong substation. + +- **Incomplete information on existing assets:** Approximations have + been made for missing data, including: existing distribution grid + capacities and costs, existing space and water heating supply, + existing industry facilities, existing transport vehicle fleets. + +- **Exogenous pathways for transformation of transport and industry:** + To avoid penny-switching the transformation of transport and + industry away from fossil fuels is determined exogenously. + +- **Energy demand distribution within countries:** + Assumptions + have been made about the distribution of demand in each country proportional to + population and GDP that may not reflect local circumstances. + Openly available + data on load time series may not correspond to the true vertical load and is + not spatially disaggregated; assuming, as we have done, that the load time series + shape is the same at each node within each country ignores local differences. + +- **Hydro-electric power plants:** + The database of hydro-electric power plants does not include plant-specific + energy storage information, so that blanket values based on country storage + totals have been used. Inflow time series are based on country-wide approximations, + ignoring local topography and basin drainage; in principle a full + hydrological model should be used. + +- **International interactions:** + Border connections and power flows to Russia, + Belarus, Ukraine, Turkey and Morocco have not been taken into account; + islands which are not connected to the main European system, such as Malta, + Crete and Cyprus, are also excluded from the model. + +- **Demand sufficiency:** Further measures of demand reduction may be + possible beyond the assumptions made here. From b025622edfe80ca504509e96f6126a0bb5a53583 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 3 Jun 2022 14:29:23 +0200 Subject: [PATCH 35/45] fix agriculture machinery oil load with spatial.oil.nodes --- scripts/prepare_sector_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 418863e7..f0f60934 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -2217,8 +2217,8 @@ def add_agriculture(n, costs): if ice_share > 0: - n.add("Load", - "agriculture machinery oil", + n.madd("Load", + ["agriculture machinery oil"], bus=spatial.oil.nodes, carrier="agriculture machinery oil", p_set=ice_share * machinery_nodal_energy.sum() * 1e6 / 8760 From 2a3b576cbd9723880d39116b949fe5bcd02dc99d Mon Sep 17 00:00:00 2001 From: Ebbe Kyhl Date: Tue, 14 Jun 2022 15:00:53 +0200 Subject: [PATCH 36/45] Add coal with carbon capture --- config.default.yaml | 1 + scripts/prepare_sector_network.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/config.default.yaml b/config.default.yaml index d48879c6..b56553ec 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -232,6 +232,7 @@ sector: marginal_cost_storage: 0. #1e-4 methanation: true helmeth: true + coal_cc: true dac: true co2_vent: true SMR: true diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index f0f60934..66be98de 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1118,6 +1118,23 @@ def add_storage_and_grids(n, costs): lifetime=costs.at['helmeth', 'lifetime'] ) + if options['coal_cc']: + + for generator,carrier in [("coal CC","coal")]: + n.madd("Link", + nodes + " " + generator, + bus0=["EU " + carrier]*len(nodes), + bus1=nodes, + bus2="co2 atmosphere", + bus3="co2 stored", + marginal_cost=costs.at['coal','efficiency']*costs.at['coal','VOM'], #NB: VOM is per MWel + capital_cost=costs.at['coal','efficiency']*costs.at['coal','fixed']+ costs.at['biomass CHP capture','fixed']*costs.at[carrier,'CO2 intensity'], #NB: fixed cost is per MWel + p_nom_extendable=True, + carrier=generator, + efficiency=costs.at['coal','efficiency'], + efficiency2=costs.at[carrier,'CO2 intensity']*(1-costs.at['biomass CHP capture','capture_rate']), + efficiency3=costs.at[carrier,'CO2 intensity']*costs.at['biomass CHP capture','capture_rate'], + lifetime=costs.at['coal','lifetime']) if options['SMR']: From 5ae56cca7c056646a42cb16161ac56fcf3dce1b7 Mon Sep 17 00:00:00 2001 From: Ebbe Kyhl <69363603+ebbekyhl@users.noreply.github.com> Date: Mon, 27 Jun 2022 18:06:59 +0200 Subject: [PATCH 37/45] Apply suggestions from code review Co-authored-by: Fabian Neumann --- config.default.yaml | 2 +- scripts/prepare_sector_network.py | 33 ++++++++++++++++--------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index b56553ec..45e431ac 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -232,7 +232,7 @@ sector: marginal_cost_storage: 0. #1e-4 methanation: true helmeth: true - coal_cc: true + coal_cc: false dac: true co2_vent: true SMR: true diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 66be98de..3e076886 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1118,23 +1118,24 @@ def add_storage_and_grids(n, costs): lifetime=costs.at['helmeth', 'lifetime'] ) - if options['coal_cc']: + if options.get('coal_cc'): - for generator,carrier in [("coal CC","coal")]: - n.madd("Link", - nodes + " " + generator, - bus0=["EU " + carrier]*len(nodes), - bus1=nodes, - bus2="co2 atmosphere", - bus3="co2 stored", - marginal_cost=costs.at['coal','efficiency']*costs.at['coal','VOM'], #NB: VOM is per MWel - capital_cost=costs.at['coal','efficiency']*costs.at['coal','fixed']+ costs.at['biomass CHP capture','fixed']*costs.at[carrier,'CO2 intensity'], #NB: fixed cost is per MWel - p_nom_extendable=True, - carrier=generator, - efficiency=costs.at['coal','efficiency'], - efficiency2=costs.at[carrier,'CO2 intensity']*(1-costs.at['biomass CHP capture','capture_rate']), - efficiency3=costs.at[carrier,'CO2 intensity']*costs.at['biomass CHP capture','capture_rate'], - lifetime=costs.at['coal','lifetime']) +n.madd("Link", + spatial.nodes, + suffix=" coal CC", + bus0=spatial.coal.nodes, + bus1=spatial.nodes, + bus2="co2 atmosphere", + bus3="co2 stored", + marginal_cost=costs.at['coal', 'efficiency'] * costs.at['coal', 'VOM'], #NB: VOM is per MWel + capital_cost=costs.at['coal', 'efficiency'] * costs.at['coal', 'fixed'] + costs.at['biomass CHP capture', 'fixed'] * costs.at['coal', 'CO2 intensity'], #NB: fixed cost is per MWel + p_nom_extendable=True, + carrier="coal", + efficiency=costs.at['coal', 'efficiency'], + efficiency2=costs.at['coal', 'CO2 intensity'] * (1 - costs.at['biomass CHP capture','capture_rate']), + efficiency3=costs.at['coal', 'CO2 intensity'] * costs.at['biomass CHP capture','capture_rate'], + lifetime=costs.at['coal','lifetime'] +) if options['SMR']: From e9f2d65cda8332b8217115003340bd4450f83ab0 Mon Sep 17 00:00:00 2001 From: Ebbe Kyhl <69363603+ebbekyhl@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:35:44 +0200 Subject: [PATCH 38/45] Indent by two layers --- scripts/prepare_sector_network.py | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 3e076886..5f75b1db 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1120,22 +1120,22 @@ def add_storage_and_grids(n, costs): if options.get('coal_cc'): -n.madd("Link", - spatial.nodes, - suffix=" coal CC", - bus0=spatial.coal.nodes, - bus1=spatial.nodes, - bus2="co2 atmosphere", - bus3="co2 stored", - marginal_cost=costs.at['coal', 'efficiency'] * costs.at['coal', 'VOM'], #NB: VOM is per MWel - capital_cost=costs.at['coal', 'efficiency'] * costs.at['coal', 'fixed'] + costs.at['biomass CHP capture', 'fixed'] * costs.at['coal', 'CO2 intensity'], #NB: fixed cost is per MWel - p_nom_extendable=True, - carrier="coal", - efficiency=costs.at['coal', 'efficiency'], - efficiency2=costs.at['coal', 'CO2 intensity'] * (1 - costs.at['biomass CHP capture','capture_rate']), - efficiency3=costs.at['coal', 'CO2 intensity'] * costs.at['biomass CHP capture','capture_rate'], - lifetime=costs.at['coal','lifetime'] -) + n.madd("Link", + spatial.nodes, + suffix=" coal CC", + bus0=spatial.coal.nodes, + bus1=spatial.nodes, + bus2="co2 atmosphere", + bus3="co2 stored", + marginal_cost=costs.at['coal', 'efficiency'] * costs.at['coal', 'VOM'], #NB: VOM is per MWel + capital_cost=costs.at['coal', 'efficiency'] * costs.at['coal', 'fixed'] + costs.at['biomass CHP capture', 'fixed'] * costs.at['coal', 'CO2 intensity'], #NB: fixed cost is per MWel + p_nom_extendable=True, + carrier="coal", + efficiency=costs.at['coal', 'efficiency'], + efficiency2=costs.at['coal', 'CO2 intensity'] * (1 - costs.at['biomass CHP capture','capture_rate']), + efficiency3=costs.at['coal', 'CO2 intensity'] * costs.at['biomass CHP capture','capture_rate'], + lifetime=costs.at['coal','lifetime'] + ) if options['SMR']: From 20e1fb40fbe6c7bf06daaed44313e59d26b4a4b3 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 28 Jun 2022 17:18:09 +0200 Subject: [PATCH 39/45] update mock_snakemake for new snakemake versions --- scripts/helper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/helper.py b/scripts/helper.py index bd55a829..7144171c 100644 --- a/scripts/helper.py +++ b/scripts/helper.py @@ -58,6 +58,7 @@ def mock_snakemake(rulename, **wildcards): import os from pypsa.descriptors import Dict from snakemake.script import Snakemake + from packaging.version import Version, parse script_dir = Path(__file__).parent.resolve() assert Path.cwd().resolve() == script_dir, \ @@ -67,7 +68,8 @@ def mock_snakemake(rulename, **wildcards): if os.path.exists(p): snakefile = p break - workflow = sm.Workflow(snakefile, overwrite_configfiles=[]) + kwargs = dict(rerun_triggers=[]) if parse(sm.__version__) > Version("7.7.0") else {} + workflow = sm.Workflow(snakefile, overwrite_configfiles=[], **kwargs) workflow.include(snakefile) workflow.global_resources = {} rule = workflow.get_rule(rulename) From 7604bb1654917adab91539b4acc4aaeeeca9b754 Mon Sep 17 00:00:00 2001 From: Irieo Date: Tue, 28 Jun 2022 18:31:45 +0200 Subject: [PATCH 40/45] Document units of bus carriers --- scripts/prepare_sector_network.py | 59 +++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 5f75b1db..6cfaa833 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -388,7 +388,8 @@ def add_carrier_buses(n, carrier, nodes=None): n.madd("Bus", nodes, location=location, - carrier=carrier + carrier=carrier, + unit='MWh_th' ) #capital cost could be corrected to e.g. 0.2 EUR/kWh * annuity and O&M @@ -439,6 +440,7 @@ def patch_electricity_network(n): update_wind_solar_costs(n, costs) n.loads["carrier"] = "electricity" n.buses["location"] = n.buses.index + n.buses["unit"] = "MWh_el" # remove trailing white space of load index until new PyPSA version after v0.18. n.loads.rename(lambda x: x.strip(), inplace=True) n.loads_t.p_set.rename(lambda x: x.strip(), axis=1, inplace=True) @@ -455,7 +457,8 @@ def add_co2_tracking(n, options): n.add("Bus", "co2 atmosphere", location="EU", - carrier="co2" + carrier="co2", + unit="t_co2" ) # can also be negative @@ -471,7 +474,8 @@ def add_co2_tracking(n, options): n.madd("Bus", spatial.co2.nodes, location=spatial.co2.locations, - carrier="co2 stored" + carrier="co2 stored", + unit="t_co2" ) n.madd("Store", @@ -703,7 +707,8 @@ def insert_electricity_distribution_grid(n, costs): n.madd("Bus", nodes + " low voltage", location=nodes, - carrier="low voltage" + carrier="low voltage", + unit="MWh_el" ) n.madd("Link", @@ -770,7 +775,8 @@ def insert_electricity_distribution_grid(n, costs): n.madd("Bus", nodes + " home battery", location=nodes, - carrier="home battery" + carrier="home battery", + unit="MWh_el" ) n.madd("Store", @@ -845,7 +851,8 @@ def add_storage_and_grids(n, costs): n.madd("Bus", nodes + " H2", location=nodes, - carrier="H2" + carrier="H2", + unit="MWh_LHV" ) n.madd("Link", @@ -1051,7 +1058,8 @@ def add_storage_and_grids(n, costs): n.madd("Bus", nodes + " battery", location=nodes, - carrier="battery" + carrier="battery", + unit="MWh_el" ) n.madd("Store", @@ -1199,7 +1207,8 @@ def add_land_transport(n, costs): nodes, location=nodes, suffix=" EV battery", - carrier="Li ion" + carrier="Li ion", + unit="MWh_el" ) p_set = electric_share * (transport[nodes] + cycling_shift(transport[nodes], 1) + cycling_shift(transport[nodes], 2)) / 3 @@ -1273,7 +1282,8 @@ def add_land_transport(n, costs): n.madd("Bus", spatial.oil.nodes, location=spatial.oil.locations, - carrier="oil" + carrier="oil", + unit="MWh_th" ) ice_efficiency = options['transport_internal_combustion_efficiency'] @@ -1381,7 +1391,8 @@ def add_heat(n, costs): n.madd("Bus", nodes[name] + f" {name} heat", location=nodes[name], - carrier=name + " heat" + carrier=name + " heat", + unit="MWh_th" ) ## Add heat load @@ -1438,7 +1449,8 @@ def add_heat(n, costs): n.madd("Bus", nodes[name] + f" {name} water tanks", location=nodes[name], - carrier=name + " water tanks" + carrier=name + " water tanks", + unit="MWh_th" ) n.madd("Link", @@ -1743,13 +1755,15 @@ def add_biomass(n, costs): n.madd("Bus", spatial.gas.biogas, location=spatial.gas.locations, - carrier="biogas" + carrier="biogas", + unit="MWh_th" ) n.madd("Bus", spatial.biomass.nodes, location=spatial.biomass.locations, - carrier="solid biomass" + carrier="solid biomass", + unit="MWh_th" ) n.madd("Store", @@ -1860,7 +1874,8 @@ def add_industry(n, costs): n.madd("Bus", spatial.biomass.industry, location=spatial.biomass.locations, - carrier="solid biomass for industry" + carrier="solid biomass for industry", + unit="MWh_th" ) if options["biomass_transport"]: @@ -1902,7 +1917,8 @@ def add_industry(n, costs): n.madd("Bus", spatial.gas.industry, location=spatial.gas.locations, - carrier="gas for industry") + carrier="gas for industry", + unit="MWh_LHV") gas_demand = industrial_demand.loc[nodes, "methane"] / 8760. @@ -1958,7 +1974,8 @@ def add_industry(n, costs): nodes, suffix=" H2 liquid", carrier="H2 liquid", - location=nodes + location=nodes, + unit="MWh_LHV" ) n.madd("Link", @@ -2016,7 +2033,8 @@ def add_industry(n, costs): n.madd("Bus", spatial.oil.nodes, location=spatial.oil.locations, - carrier="oil" + carrier="oil", + unit="MWh_th" ) if "oil" not in n.stores.carrier.unique(): @@ -2130,7 +2148,8 @@ def add_industry(n, costs): n.add("Bus", "process emissions", location="EU", - carrier="process emissions" + carrier="process emissions", + unit="t_co2" ) # this should be process emissions fossil+feedstock @@ -2314,7 +2333,7 @@ if __name__ == "__main__": simpl='', opts="", clusters="37", - lv=1.0, + lv=1.5, sector_opts='Co2L0-168H-T-H-B-I-solar3-dist1', planning_horizons="2020", ) @@ -2448,3 +2467,5 @@ if __name__ == "__main__": add_electricity_grid_connection(n, costs) n.export_to_netcdf(snakemake.output[0]) + +# %% From 187c9cbaa3e0ffb3f691b882f17aa1fcec852649 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 29 Jun 2022 08:49:29 +0200 Subject: [PATCH 41/45] add release notes --- doc/release_notes.rst | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 7808d2ba..f1fb12fe 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -24,7 +24,7 @@ incorporates retrofitting options to hydrogen. * New rule ``build_gas_input_locations`` compiles the LNG import capacities (including planned projects from gem.wiki), pipeline entry capacities and local production capacities for each region of the model. These are the - regions where fossil gas can eventually enter the model. + regions where fossil gas can eventually enter the model. * New rule ``cluster_gas_network`` that clusters the gas transmission network data to the model resolution. Cross-regional pipeline capacities are aggregated @@ -47,8 +47,8 @@ incorporates retrofitting options to hydrogen. H2_retrofit_capacity_per_CH4`` units are made available as hydrogen pipeline capacity in the corresponding corridor. These repurposed hydrogen pipelines have lower costs than new hydrogen pipelines. Both new and repurposed pipelines - can be built simultaneously. The retrofitting option ``sector: H2_retrofit:`` also works - with a copperplated methane infrastructure, i.e. when ``sector: gas_network: false``. + can be built simultaneously. The retrofitting option ``sector: H2_retrofit:`` also works + with a copperplated methane infrastructure, i.e. when ``sector: gas_network: false``. * New hydrogen pipelines can now be built where there are already power or gas transmission routes. Previously, only the electricity transmission routes were @@ -56,6 +56,8 @@ incorporates retrofitting options to hydrogen. **New features and functionality** +* Units are assigned to the buses. These only provide a better understanding. The specifications of the units are not taken into account in the optimisation, which means that no automatic conversion of units takes place. + * Option ``retrieve_sector_databundle`` to automatically retrieve and extract data bundle. * Add regionalised hydrogen salt cavern storage potentials from `Technical Potential of Salt Caverns for Hydrogen Storage in Europe `_. @@ -84,7 +86,7 @@ besides many performance improvements. This release is known to work with `PyPSA-Eur `_ Version 0.4.0, `Technology Data -`_ Version 0.3.0 and +`_ Version 0.3.0 and `PyPSA `_ Version 0.18.0. Please note that the data bundle has also been updated. @@ -202,19 +204,19 @@ Please note that the data bundle has also been updated. A function ``helper.override_component_attrs`` was added that loads this data and can pass the overridden component attributes into ``pypsa.Network()``. -* Add various parameters to ``config.default.yaml`` which were previously hardcoded inside the scripts +* Add various parameters to ``config.default.yaml`` which were previously hardcoded inside the scripts (e.g. energy reference years, BEV settings, solar thermal collector models, geomap colours). * Removed stale industry demand rules ``build_industrial_energy_demand_per_country`` and ``build_industrial_demand``. These are superseded with more regionally resolved rules. * Use simpler and shorter ``gdf.sjoin()`` function to allocate industrial sites - from the Hotmaps database to onshore regions. + from the Hotmaps database to onshore regions. This change also fixes a bug: The previous version allocated sites to the closest bus, but at country borders (where Voronoi cells are distorted by the borders), this had resulted in e.g. a Spanish site close to the French border - being wrongly allocated to the French bus if the bus center was closer. + being wrongly allocated to the French bus if the bus center was closer. * Retrofitting rule is now only triggered if endogeneously optimised. @@ -225,7 +227,7 @@ Please note that the data bundle has also been updated. * Improve legibility of ``config.default.yaml`` and remove unused options. * Use the country-specific time zone mappings from ``pytz`` rather than a manual mapping. - + * A function ``add_carrier_buses()`` was added to the ``prepare_network`` rule to reduce code duplication. * In the ``prepare_network`` rule the cost and potential adjustment was moved into an From b4d0a8404cdfe6fc5be5bcc9857a92e5e2463632 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 29 Jun 2022 08:57:08 +0200 Subject: [PATCH 42/45] change unit from carrier gas from MWh_th -> MWh_LHV --- scripts/prepare_sector_network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 6cfaa833..7baffc83 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -385,11 +385,13 @@ def add_carrier_buses(n, carrier, nodes=None): n.add("Carrier", carrier) + unit="MWh_LHV" if carrier=="gas" else "MWH_th" + n.madd("Bus", nodes, location=location, carrier=carrier, - unit='MWh_th' + unit=unit ) #capital cost could be corrected to e.g. 0.2 EUR/kWh * annuity and O&M From eddeb227a7e86a45285ac444ca6750ac00ccad60 Mon Sep 17 00:00:00 2001 From: Iegor Riepin Date: Thu, 30 Jun 2022 17:15:41 +0200 Subject: [PATCH 43/45] Update scripts/prepare_sector_network.py code style update Co-authored-by: Fabian Neumann --- scripts/prepare_sector_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 7baffc83..7903921c 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -385,7 +385,7 @@ def add_carrier_buses(n, carrier, nodes=None): n.add("Carrier", carrier) - unit="MWh_LHV" if carrier=="gas" else "MWH_th" + unit = "MWh_LHV" if carrier == "gas" else "MWh_th" n.madd("Bus", nodes, From 53ad90095ab1f5d62d2ad61d57a7c9cd94cafe31 Mon Sep 17 00:00:00 2001 From: Iegor Riepin Date: Thu, 30 Jun 2022 17:17:12 +0200 Subject: [PATCH 44/45] Update scripts/prepare_sector_network.py sync it up to Fabian's feeling of useful things Co-authored-by: Fabian Neumann --- scripts/prepare_sector_network.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 7903921c..03ffa4fa 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -2469,5 +2469,3 @@ if __name__ == "__main__": add_electricity_grid_connection(n, costs) n.export_to_netcdf(snakemake.output[0]) - -# %% From 5d60e8c6e6790a3ed6ce492b03bab3a5792ef54b Mon Sep 17 00:00:00 2001 From: Irieo Date: Thu, 30 Jun 2022 17:37:40 +0200 Subject: [PATCH 45/45] Label oil and biomass buses also as MWh_LHV --- scripts/prepare_sector_network.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 03ffa4fa..bcf9cf6b 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1285,7 +1285,7 @@ def add_land_transport(n, costs): spatial.oil.nodes, location=spatial.oil.locations, carrier="oil", - unit="MWh_th" + unit="MWh_LHV" ) ice_efficiency = options['transport_internal_combustion_efficiency'] @@ -1758,14 +1758,14 @@ def add_biomass(n, costs): spatial.gas.biogas, location=spatial.gas.locations, carrier="biogas", - unit="MWh_th" + unit="MWh_LHV" ) n.madd("Bus", spatial.biomass.nodes, location=spatial.biomass.locations, carrier="solid biomass", - unit="MWh_th" + unit="MWh_LHV" ) n.madd("Store", @@ -1877,7 +1877,7 @@ def add_industry(n, costs): spatial.biomass.industry, location=spatial.biomass.locations, carrier="solid biomass for industry", - unit="MWh_th" + unit="MWh_LHV" ) if options["biomass_transport"]: @@ -2036,7 +2036,7 @@ def add_industry(n, costs): spatial.oil.nodes, location=spatial.oil.locations, carrier="oil", - unit="MWh_th" + unit="MWh_LHV" ) if "oil" not in n.stores.carrier.unique():