diff --git a/.gitignore b/.gitignore index e126f9c5..b79cd7ab 100644 --- a/.gitignore +++ b/.gitignore @@ -29,18 +29,18 @@ dconf /data/links_p_nom.csv /data/*totals.csv /data/biomass* -/data/emobility/ -/data/eea* -/data/jrc* +/data/bundle-sector/emobility/ +/data/bundle-sector/eea* +/data/bundle-sector/jrc* /data/heating/ -/data/eurostat* +/data/bundle-sector/eurostat* /data/odyssee/ /data/transport_data.csv -/data/switzerland* +/data/bundle-sector/switzerland* /data/.nfs* -/data/Industrial_Database.csv +/data/bundle-sector/Industrial_Database.csv /data/retro/tabula-calculator-calcsetbuilding.csv -/data/nuts* +/data/bundle-sector/nuts* data/gas_network/scigrid-gas/ data/costs_*.csv diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 57bbd1f4..54b6dad4 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -14,6 +14,12 @@ Upcoming Release * For industry distribution, use EPRTR as fallback if ETS data is not available. +* The minimum capacity for renewable generators when using the myopic option has been fixed. + +* Files downloaded from zenodo are now write-protected to prevent accidental re-download. + +* Files extracted from sector-coupled data bundle have been moved from ``data/`` to ``data/sector-bundle``. + * New feature multi-decade optimisation with perfect foresight. PyPSA-Eur 0.8.1 (27th July 2023) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 383951bd..f9fdc3ac 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -350,7 +350,9 @@ rule add_electricity: hydro_capacities=ancient("data/bundle/hydro_capacities.csv"), geth_hydro_capacities="data/geth2015_hydro_capacities.csv", unit_commitment="data/unit_commitment.csv", - fuel_price=RESOURCES + "monthly_fuel_price.csv", + fuel_price=RESOURCES + "monthly_fuel_price.csv" + if config["conventional"]["dynamic_fuel_price"] + else [], load=RESOURCES + "load.csv", nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", output: @@ -478,7 +480,7 @@ rule prepare_network: input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", tech_costs=COSTS, - co2_price=RESOURCES + "co2_price.csv", + co2_price=lambda w: RESOURCES + "co2_price.csv" if "Ept" in w.opts else [], output: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", log: diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 356abdc5..10a5f821 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -242,9 +242,9 @@ rule build_energy_totals: energy=config["energy"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", - co2="data/eea/UNFCCC_v23.csv", - swiss="data/switzerland-sfoe/switzerland-new_format.csv", - idees="data/jrc-idees-2015", + co2="data/bundle-sector/eea/UNFCCC_v23.csv", + swiss="data/bundle-sector/switzerland-sfoe/switzerland-new_format.csv", + idees="data/bundle-sector/jrc-idees-2015", district_heat_share="data/district_heat_share.csv", eurostat=input_eurostat, output: @@ -272,7 +272,7 @@ rule build_biomass_potentials: "https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", keep_local=True, ), - nuts2="data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 + nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"), swiss_cantons=ancient("data/bundle/ch_cantons.csv"), @@ -366,7 +366,7 @@ if not config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_salt_cavern_potentials: input: - salt_caverns="data/h2_salt_caverns_GWh_per_sqkm.geojson", + salt_caverns="data/bundle-sector/h2_salt_caverns_GWh_per_sqkm.geojson", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}_{clusters}.geojson", output: @@ -388,7 +388,7 @@ rule build_ammonia_production: params: countries=config["countries"], input: - usgs="data/myb1-2017-nitro.xls", + usgs="data/bundle-sector/myb1-2017-nitro.xls", output: ammonia_production=RESOURCES + "ammonia_production.csv", threads: 1 @@ -410,7 +410,7 @@ rule build_industry_sector_ratios: ammonia=config["sector"].get("ammonia", False), input: ammonia_production=RESOURCES + "ammonia_production.csv", - idees="data/jrc-idees-2015", + idees="data/bundle-sector/jrc-idees-2015", output: industry_sector_ratios=RESOURCES + "industry_sector_ratios.csv", threads: 1 @@ -432,8 +432,8 @@ rule build_industrial_production_per_country: countries=config["countries"], input: ammonia_production=RESOURCES + "ammonia_production.csv", - jrc="data/jrc-idees-2015", - eurostat="data/eurostat-energy_balances-may_2018_edition", + jrc="data/bundle-sector/jrc-idees-2015", + eurostat="data/bundle-sector/eurostat-energy_balances-may_2018_edition", output: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -483,7 +483,7 @@ rule build_industrial_distribution_key: input: regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", - hotmaps_industrial_database="data/Industrial_Database.csv", + hotmaps_industrial_database="data/bundle-sector/Industrial_Database.csv", output: industrial_distribution_key=RESOURCES + "industrial_distribution_key_elec_s{simpl}_{clusters}.csv", @@ -558,7 +558,7 @@ rule build_industrial_energy_demand_per_country_today: countries=config["countries"], industry=config["industry"], input: - jrc="data/jrc-idees-2015", + jrc="data/bundle-sector/jrc-idees-2015", ammonia_production=RESOURCES + "ammonia_production.csv", industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -684,8 +684,8 @@ rule build_transport_demand: pop_weighted_energy_totals=RESOURCES + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv", transport_data=RESOURCES + "transport_data.csv", - traffic_data_KFZ="data/emobility/KFZ__count", - traffic_data_Pkw="data/emobility/Pkw__count", + traffic_data_KFZ="data/bundle-sector/emobility/KFZ__count", + traffic_data_Pkw="data/bundle-sector/emobility/Pkw__count", temp_air_total=RESOURCES + "temp_air_total_elec_s{simpl}_{clusters}.nc", output: transport_demand=RESOURCES + "transport_demand_s{simpl}_{clusters}.csv", @@ -734,7 +734,7 @@ rule prepare_sector_network: avail_profile=RESOURCES + "avail_profile_s{simpl}_{clusters}.csv", dsm_profile=RESOURCES + "dsm_profile_s{simpl}_{clusters}.csv", co2_totals_name=RESOURCES + "co2_totals.csv", - co2="data/eea/UNFCCC_v23.csv", + co2="data/bundle-sector/eea/UNFCCC_v23.csv", biomass_potentials=RESOURCES + "biomass_potentials_s{simpl}_{clusters}.csv", heat_profile="data/heat_load_profile_BDEW.csv", costs="data/costs_{}.csv".format(config["costs"]["year"]) diff --git a/rules/common.smk b/rules/common.smk index ec5be355..d3416050 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -42,7 +42,7 @@ def has_internet_access(url="www.zenodo.org") -> bool: def input_eurostat(w): # 2016 includes BA, 2017 does not report_year = config["energy"]["eurostat_report_year"] - return f"data/eurostat-energy_balances-june_{report_year}_edition" + return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition" def solved_previous_horizon(wildcards): diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 32775220..89e8e96b 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -135,6 +135,8 @@ rule plot_summary: countries=config["countries"], planning_horizons=config["scenario"]["planning_horizons"], sector_opts=config["scenario"]["sector_opts"], + emissions_scope=config["energy"]["emissions"], + eurostat_report_year=config["energy"]["eurostat_report_year"], plotting=config["plotting"], RDIR=RDIR, input: @@ -142,7 +144,7 @@ rule plot_summary: energy=RESULTS + "csvs/energy.csv", balances=RESULTS + "csvs/supply_energy.csv", eurostat=input_eurostat, - co2="data/eea/UNFCCC_v23.csv", + co2="data/bundle-sector/eea/UNFCCC_v23.csv", output: costs=RESULTS + "graphs/costs.pdf", energy=RESULTS + "graphs/energy.pdf", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 0b60ee2e..66ce76df 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -27,7 +27,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", rule retrieve_databundle: output: - expand("data/bundle/{file}", file=datafiles), + protected(expand("data/bundle/{file}", file=datafiles)), log: LOGS + "retrieve_databundle.log", resources: @@ -92,7 +92,7 @@ if config["enable"]["retrieve"] and config["enable"].get( static=True, ), output: - RESOURCES + "natura.tiff", + protected(RESOURCES + "natura.tiff"), log: LOGS + "retrieve_natura_raster.log", resources: @@ -106,22 +106,30 @@ if config["enable"]["retrieve"] and config["enable"].get( "retrieve_sector_databundle", True ): datafiles = [ - "data/eea/UNFCCC_v23.csv", - "data/switzerland-sfoe/switzerland-new_format.csv", - "data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", - "data/myb1-2017-nitro.xls", - "data/Industrial_Database.csv", - "data/emobility/KFZ__count", - "data/emobility/Pkw__count", - "data/h2_salt_caverns_GWh_per_sqkm.geojson", - directory("data/eurostat-energy_balances-june_2016_edition"), - directory("data/eurostat-energy_balances-may_2018_edition"), - directory("data/jrc-idees-2015"), + "eea/UNFCCC_v23.csv", + "switzerland-sfoe/switzerland-new_format.csv", + "nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", + "myb1-2017-nitro.xls", + "Industrial_Database.csv", + "emobility/KFZ__count", + "emobility/Pkw__count", + "h2_salt_caverns_GWh_per_sqkm.geojson", + ] + + datafolders = [ + protected( + directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition") + ), + protected( + directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition") + ), + protected(directory("data/bundle-sector/jrc-idees-2015")), ] rule retrieve_sector_databundle: output: - *datafiles, + protected(expand("data/bundle-sector/{files}", files=datafiles)), + *datafolders, log: LOGS + "retrieve_sector_databundle.log", retries: 2 @@ -143,7 +151,9 @@ if config["enable"]["retrieve"] and ( rule retrieve_gas_infrastructure_data: output: - expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles), + protected( + expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles) + ), log: LOGS + "retrieve_gas_infrastructure_data.log", retries: 2 @@ -187,7 +197,7 @@ if config["enable"]["retrieve"]: static=True, ), output: - "data/shipdensity_global.zip", + protected("data/shipdensity_global.zip"), log: LOGS + "retrieve_ship_raster.log", resources: diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index c485aa99..782b873c 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -165,7 +165,7 @@ def sanitize_carriers(n, config): nice_names = ( pd.Series(config["plotting"]["nice_names"]) .reindex(carrier_i) - .fillna(carrier_i.to_series().str.title()) + .fillna(carrier_i.to_series()) ) n.carriers["nice_name"] = n.carriers.nice_name.where( n.carriers.nice_name != "", nice_names diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 782fde78..cb66eb15 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -446,15 +446,23 @@ def add_heating_capacities_installed_before_baseyear( # split existing capacities between residential and services # proportional to energy demand + p_set_sum = n.loads_t.p_set.sum() ratio_residential = pd.Series( [ ( - n.loads_t.p_set.sum()[f"{node} residential rural heat"] + p_set_sum[f"{node} residential rural heat"] / ( - n.loads_t.p_set.sum()[f"{node} residential rural heat"] - + n.loads_t.p_set.sum()[f"{node} services rural heat"] + p_set_sum[f"{node} residential rural heat"] + + p_set_sum[f"{node} services rural heat"] ) ) + # if rural heating demand for one of the nodes doesn't exist, + # then columns were dropped before and heating demand share should be 0.0 + if all( + f"{node} {service} rural heat" in p_set_sum.index + for service in ["residential", "services"] + ) + else 0.0 for node in nodal_df.index ], index=nodal_df.index, diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 335c6a36..a0dc195b 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -13,10 +13,13 @@ logger = logging.getLogger(__name__) import uuid from itertools import product +import country_converter as coco import geopandas as gpd import pandas as pd from packaging.version import Version, parse +cc = coco.CountryConverter() + def locate_missing_industrial_sites(df): """ @@ -107,6 +110,17 @@ def prepare_hotmaps_database(regions): # concat not duplicated and filtered gdf gdf = pd.concat([gdf.drop(duplicated_i), gdf_filtered]).sort_index() + # the .sjoin can lead to duplicates if a geom is in two overlapping regions + if gdf.index.duplicated().any(): + # get all duplicated entries + duplicated_i = gdf.index[gdf.index.duplicated()] + # convert from raw data country name to iso-2-code + code = cc.convert(gdf.loc[duplicated_i, "Country"], to="iso2") + # screen out malformed country allocation + gdf_filtered = gdf.loc[duplicated_i].query("country == @code") + # concat not duplicated and filtered gdf + gdf = pd.concat([gdf.drop(duplicated_i), gdf_filtered]) + return gdf diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 56ee98c9..98a6a6d7 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -711,5 +711,5 @@ if __name__ == "__main__": if snakemake.params.foresight == "myopic": cumulative_cost = calculate_cumulative_cost() cumulative_cost.to_csv( - "results/" + snakemake.params.RDIR + "/csvs/cumulative_cost.csv" + "results/" + snakemake.params.RDIR + "csvs/cumulative_cost.csv" ) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 8405bd8f..8f939a36 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -457,7 +457,6 @@ def plot_carbon_budget_distribution(input_eurostat): """ Plot historical carbon emissions in the EU and decarbonization path. """ - import seaborn as sns sns.set() @@ -502,6 +501,14 @@ def plot_carbon_budget_distribution(input_eurostat): # plot committed and under-discussion targets # (notice that historical emissions include all countries in the # network, but targets refer to EU) + ax1.plot( + [2020], + [0.8 * emissions[1990]], + marker="*", + markersize=12, + markerfacecolor="black", + markeredgecolor="black", + ) ax1.plot( [2030], @@ -512,7 +519,23 @@ def plot_carbon_budget_distribution(input_eurostat): markeredgecolor="black", ) + ax1.plot( + [2030], + [0.6 * emissions[1990]], + marker="*", + markersize=12, + markerfacecolor="black", + markeredgecolor="black", + ) + ax1.plot( + [2050, 2050], + [x * emissions[1990] for x in [0.2, 0.05]], + color="gray", + linewidth=2, + marker="_", + alpha=0.5, + ) ax1.plot( [2050], diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 6b16a4e9..cea18fdf 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -46,7 +46,6 @@ def define_spatial(nodes, options): ---------- nodes : list-like """ - global spatial spatial.nodes = nodes @@ -192,17 +191,17 @@ def get(item, investment_year=None): def co2_emissions_year( - countries, input_eurostat, opts, snakemake, year + countries, input_eurostat, opts, emissions_scope, report_year, year ): """ Calculate CO2 emissions in one specific year (e.g. 1990 or 2018). """ - emissions_scope = snakemake.config["energy"]["emissions"] + emissions_scope = snakemake.params.energy["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope) # TODO: read Eurostat data from year > 2014 # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK - report_year = snakemake.config["energy"]["eurostat_report_year"] + report_year = snakemake.params.energy["eurostat_report_year"] if year > 2014: eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, year=2014 @@ -241,12 +240,24 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries = snakemake.params.countries e_1990 = co2_emissions_year( - countries, input_eurostat, opts, snakemake, year=1990 + countries, + input_eurostat, + opts, + emissions_scope, + report_year, + input_co2, + year=1990, ) # emissions at the beginning of the path (last year available 2018) e_0 = co2_emissions_year( - countries, input_eurostat, opts, snakemake, year=2018, + countries, + input_eurostat, + opts, + emissions_scope, + report_year, + input_co2, + year=2018, ) planning_horizons = snakemake.params.planning_horizons @@ -357,7 +368,6 @@ def update_wind_solar_costs(n, costs): Update costs for wind and solar generators added with pypsa-eur to those cost in the planning year. """ - # NB: solar costs are also manipulated for rooftop # when distribution grid is inserted n.generators.loc[n.generators.carrier == "solar", "capital_cost"] = costs.at[ @@ -435,7 +445,6 @@ def add_carrier_buses(n, carrier, nodes=None): """ Add buses to connect e.g. coal, nuclear and oil plants. """ - if nodes is None: nodes = vars(spatial)[carrier].nodes location = vars(spatial)[carrier].locations @@ -716,6 +725,7 @@ def average_every_nhours(n, offset): return m + def cycling_shift(df, steps=1): """ Cyclic shift on index of pd.Series|pd.DataFrame by number of steps. @@ -1150,7 +1160,6 @@ def add_storage_and_grids(n, costs): e_cyclic=True, carrier="H2 Store", capital_cost=h2_capital_cost, - lifetime=costs.at["hydrogen storage tank type 1 including compressor", "lifetime"], ) if options["gas_network"] or options["H2_retrofit"]: @@ -3076,7 +3085,6 @@ def maybe_adjust_costs_and_potentials(n, opts): logger.info(f"changing {attr} for {carrier} by factor {factor}") -# TODO this should rather be a config no wildcard def limit_individual_line_extension(n, maxext): logger.info(f"Limiting new HVAC and HVDC extensions to {maxext} MW") n.lines["s_nom_max"] = n.lines["s_nom"] + maxext @@ -3211,7 +3219,7 @@ def apply_time_segmentation( df = pnl.copy() df.columns = pd.MultiIndex.from_product([[c.name], [attr], df.columns]) raw = pd.concat([raw, df], axis=1) - raw = raw.dropna(axis=1) + # normalise all time-dependent data annual_max = raw.max().replace(0, 1) raw = raw.div(annual_max, level=0) @@ -3268,28 +3276,26 @@ def set_temporal_aggregation(n, opts, solver_name): # segments with package tsam m = re.match(r"^(\d+)seg$", o, re.IGNORECASE) if m is not None: - if snakemake.params.foresight!="perfect": - segments = int(m[1]) - logger.info(f"Use temporal segmentation with {segments} segments") - n = apply_time_segmentation(n, segments, solver_name=solver_name) - break - else: - logger.info("Apply temporal segmentation at prepare_perfect_foresight.") + segments = int(m[1]) + logger.info(f"Use temporal segmentation with {segments} segments") + n = apply_time_segmentation(n, segments, solver_name=solver_name) + break return n -#%% + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake snakemake = mock_snakemake( "prepare_sector_network", + configfiles="test/config.overnight.yaml", simpl="", opts="", - clusters="37", - ll="v1.0", - sector_opts="60SEG-T-H-B-I-A-solar+p3-dist1", - planning_horizons="2050", + clusters="5", + ll="v1.5", + sector_opts="CO2L0-24H-T-H-B-I-A-solar+p3-dist1", + planning_horizons="2030", ) logging.basicConfig(level=snakemake.config["logging"]["level"]) @@ -3391,7 +3397,6 @@ if __name__ == "__main__": add_allam(n, costs) solver_name = snakemake.config["solving"]["solver"]["name"] - n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" @@ -3404,8 +3409,14 @@ if __name__ == "__main__": if not os.path.exists(fn): emissions_scope = snakemake.params.emissions_scope report_year = snakemake.params.eurostat_report_year + input_co2 = snakemake.input.co2 build_carbon_budget( - o, snakemake.input.eurostat, fn, emissions_scope, report_year + o, + snakemake.input.eurostat, + fn, + emissions_scope, + report_year, + input_co2, ) co2_cap = pd.read_csv(fn, index_col=0).squeeze() limit = co2_cap.loc[investment_year] diff --git a/scripts/solve_network.py b/scripts/solve_network.py index d0a30f74..fa03037f 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -110,6 +110,9 @@ def _add_land_use_constraint(n): # warning: this will miss existing offwind which is not classed AC-DC and has carrier 'offwind' for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]: + extendable_i = (n.generators.carrier == carrier) & n.generators.p_nom_extendable + n.generators.loc[extendable_i, "p_nom_min"] = 0 + ext_i = (n.generators.carrier == carrier) & ~n.generators.p_nom_extendable existing = ( n.generators.loc[ext_i, "p_nom"] @@ -126,7 +129,7 @@ def _add_land_use_constraint(n): if len(existing_large): logger.warning( f"Existing capacities larger than technical potential for {existing_large},\ - adjust technical potential to existing capacities" + adjust technical potential to existing capacities" ) n.generators.loc[existing_large, "p_nom_max"] = n.generators.loc[ existing_large, "p_nom_min"