diff --git a/.gitignore b/.gitignore index 4677557b..10243def 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ gurobi.log /data/Industrial_Database.csv /data/retro/tabula-calculator-calcsetbuilding.csv /data/nuts* + *.org *.nc diff --git a/Snakefile b/Snakefile index a16e60f0..f4c795f3 100644 --- a/Snakefile +++ b/Snakefile @@ -24,7 +24,6 @@ subworkflow pypsaeur: snakefile: "../pypsa-eur/Snakefile" configfile: "../pypsa-eur/config.yaml" - rule all: input: SDIR + '/graphs/costs.pdf' @@ -174,13 +173,19 @@ rule build_energy_totals: rule build_biomass_potentials: input: - jrc_potentials="data/biomass/JRC Biomass Potentials.xlsx" + enspreso_biomass=HTTP.remote("https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", keep_local=True), + nuts2="data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 + regions_onshore=pypsaeur("resources/regions_onshore_elec_s{simpl}_{clusters}.geojson"), + nuts3_population="../pypsa-eur/data/bundle/nama_10r_3popgdp.tsv.gz", + swiss_cantons="../pypsa-eur/data/bundle/ch_cantons.csv", + swiss_population="../pypsa-eur/data/bundle/je-e-21.03.02.xls", + country_shapes=pypsaeur('resources/country_shapes.geojson') output: - biomass_potentials_all='resources/biomass_potentials_all.csv', - biomass_potentials='resources/biomass_potentials.csv' + biomass_potentials_all='resources/biomass_potentials_all_s{simpl}_{clusters}.csv', + biomass_potentials='resources/biomass_potentials_s{simpl}_{clusters}.csv' threads: 1 resources: mem_mb=1000 - benchmark: "benchmarks/build_biomass_potentials" + benchmark: "benchmarks/build_biomass_potentials_s{simpl}_{clusters}" script: 'scripts/build_biomass_potentials.py' @@ -340,9 +345,9 @@ rule prepare_sector_network: energy_totals_name='resources/energy_totals.csv', co2_totals_name='resources/co2_totals.csv', transport_name='resources/transport_data.csv', - traffic_data_KFZ="data/emobility/KFZ__count", - traffic_data_Pkw="data/emobility/Pkw__count", - biomass_potentials='resources/biomass_potentials.csv', + traffic_data_KFZ = "data/emobility/KFZ__count", + traffic_data_Pkw = "data/emobility/Pkw__count", + biomass_potentials='resources/biomass_potentials_s{simpl}_{clusters}.csv', heat_profile="data/heat_load_profile_BDEW.csv", costs=CDIR + "costs_{planning_horizons}.csv", profile_offwind_ac=pypsaeur("resources/profile_offwind-ac.nc"), diff --git a/config.default.yaml b/config.default.yaml index e1ddd60d..d6d4052c 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -99,28 +99,28 @@ energy: biomass: year: 2030 - scenario: Med + scenario: ENS_Med classes: solid biomass: - - Primary agricultural residues - - Forestry energy residue - - Secondary forestry residues - - Secondary Forestry residues sawdust - - Forestry residues from landscape care biomass + - Agricultural waste + - Fuelwood residues + - Secondary Forestry residues - woodchips + - Sawdust + - Residues from landscape care - Municipal waste not included: - - Bioethanol sugar beet biomass - - Rapeseeds for biodiesel - - sunflower and soya for Biodiesel - - Starchy crops biomass - - Grassy crops biomass - - Willow biomass - - Poplar biomass potential - - Roundwood fuelwood - - Roundwood Chips & Pellets + - Sugar from sugar beet + - Rape seed + - "Sunflower, soya seed " + - Bioethanol barley, wheat, grain maize, oats, other cereals and rye + - Miscanthus, switchgrass, RCG + - Willow + - Poplar + - FuelwoodRW + - C&P_RW biogas: - - Manure biomass potential - - Sludge biomass + - Manure solid, liquid + - Sludge solar_thermal: diff --git a/doc/data.csv b/doc/data.csv index 75bdc10e..01fa04b3 100644 --- a/doc/data.csv +++ b/doc/data.csv @@ -1,28 +1,30 @@ -description,file/folder,licence,source,, -JRC IDEES database,jrc-idees-2015/,CC BY 4.0,https://ec.europa.eu/jrc/en/potencia/jrc-idees,, -urban/rural fraction,urban_percent.csv,unknown,unknown,, -JRC biomass potentials,biomass/,unknown,https://doi.org/10.2790/39014,, -EEA emission statistics,eea/UNFCCC_v23.csv,EEA standard re-use policy,https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16,, -Eurostat Energy Balances,eurostat-energy_balances-*/,Eurostat,https://ec.europa.eu/eurostat/web/energy/data/energy-balances,, -Swiss energy statistics from Swiss Federal Office of Energy,switzerland-sfoe/,unknown,http://www.bfe.admin.ch/themen/00526/00541/00542/02167/index.html?dossier_id=02169,, -BASt emobility statistics,emobility/,unknown,http://www.bast.de/DE/Verkehrstechnik/Fachthemen/v2-verkehrszaehlung/Stundenwerte.html?nn=626916,, -timezone mappings,timezone_mappings.csv,CC BY 4.0,Tom Brown,, -BDEW heating profile,heat_load_profile_BDEW.csv,unknown,https://github.com/oemof/demandlib,, -heating profiles for Aarhus,heat_load_profile_DK_AdamJensen.csv,unknown,Adam Jensen MA thesis at Aarhus University,, -George Lavidas wind/wave costs,WindWaveWEC_GLTB.xlsx,unknown,George Lavidas,, -country codes,Country_codes.csv,CC BY 4.0,Marta Victoria,, -co2 budgets,co2_budget.csv,CC BY 4.0,https://arxiv.org/abs/2004.11009,, -existing heating potentials,existing_infrastructure/existing_heating_raw.csv,unknown,https://ec.europa.eu/energy/studies/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment_en?redir=1,, -IRENA existing VRE capacities,existing_infrastructure/{solar,onwind,offwind}_capcity_IRENA.csv,unknown,https://www.irena.org/Statistics/Download-Data -USGS ammonia production,myb1-2017-nitro.xls,unknown,https://www.usgs.gov/centers/nmic/nitrogen-statistics-and-information,, -hydrogen salt cavern potentials,hydrogen_salt_cavern_potentials.csv,CC BY 4.0,https://doi.org/10.1016/j.ijhydene.2019.12.161,, -hotmaps industrial site database,Industrial_Database.csv,CC BY 4.0,https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database,, -Hotmaps building stock data,data_building_stock.csv,CC BY 4.0,https://gitlab.com/hotmaps/building-stock,, -U-values Poland,u_values_poland.csv,unknown,https://data.europa.eu/euodp/de/data/dataset/building-stock-observatory,, -Floor area missing in hotmaps building stock data,floor_area_missing.csv,unknown,https://data.europa.eu/euodp/de/data/dataset/building-stock-observatory,, -Comparative level investment,comparative_level_investment.csv,Eurostat,https://ec.europa.eu/eurostat/statistics-explained/index.php?title=Comparative_price_levels_for_investment,, -Electricity taxes,electricity_taxes_eu.csv,Eurostat,https://appsso.eurostat.ec.europa.eu/nui/show.do?dataset=nrg_pc_204&lang=en,, -Building topologies and corresponding standard values,tabula-calculator-calcsetbuilding.csv,unknown,https://episcope.eu/fileadmin/tabula/public/calc/tabula-calculator.xlsx,, -Retrofitting thermal envelope costs for Germany,retro_cost_germany.csv,unkown,https://www.iwu.de/forschung/handlungslogiken/kosten-energierelevanter-bau-und-anlagenteile-bei-modernisierung/,, +description,file/folder,licence,source +JRC IDEES database,jrc-idees-2015/,CC BY 4.0,https://ec.europa.eu/jrc/en/potencia/jrc-idees +urban/rural fraction,urban_percent.csv,unknown,unknown +JRC biomass potentials,biomass/,unknown,https://doi.org/10.2790/39014 +JRC ENSPRESO biomass potentials,remote,CC BY 4.0,https://data.jrc.ec.europa.eu/dataset/74ed5a04-7d74-4807-9eab-b94774309d9f +EEA emission statistics,eea/UNFCCC_v23.csv,EEA standard re-use policy,https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16 +Eurostat Energy Balances,eurostat-energy_balances-*/,Eurostat,https://ec.europa.eu/eurostat/web/energy/data/energy-balances +Swiss energy statistics from Swiss Federal Office of Energy,switzerland-sfoe/,unknown,http://www.bfe.admin.ch/themen/00526/00541/00542/02167/index.html?dossier_id=02169 +BASt emobility statistics,emobility/,unknown,http://www.bast.de/DE/Verkehrstechnik/Fachthemen/v2-verkehrszaehlung/Stundenwerte.html?nn=626916 +timezone mappings,timezone_mappings.csv,CC BY 4.0,Tom Brown +BDEW heating profile,heat_load_profile_BDEW.csv,unknown,https://github.com/oemof/demandlib +heating profiles for Aarhus,heat_load_profile_DK_AdamJensen.csv,unknown,Adam Jensen MA thesis at Aarhus University +George Lavidas wind/wave costs,WindWaveWEC_GLTB.xlsx,unknown,George Lavidas +country codes,Country_codes.csv,CC BY 4.0,Marta Victoria +co2 budgets,co2_budget.csv,CC BY 4.0,https://arxiv.org/abs/2004.11009 +existing heating potentials,existing_infrastructure/existing_heating_raw.csv,unknown,https://ec.europa.eu/energy/studies/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment_en?redir=1 +IRENA existing VRE capacities,existing_infrastructure/{solar|onwind|offwind}_capcity_IRENA.csv,unknown,https://www.irena.org/Statistics/Download-Data +USGS ammonia production,myb1-2017-nitro.xls,unknown,https://www.usgs.gov/centers/nmic/nitrogen-statistics-and-information +hydrogen salt cavern potentials,hydrogen_salt_cavern_potentials.csv,CC BY 4.0,https://doi.org/10.1016/j.ijhydene.2019.12.161 +hotmaps industrial site database,Industrial_Database.csv,CC BY 4.0,https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database +Hotmaps building stock data,data_building_stock.csv,CC BY 4.0,https://gitlab.com/hotmaps/building-stock +U-values Poland,u_values_poland.csv,unknown,https://data.europa.eu/euodp/de/data/dataset/building-stock-observatory +Floor area missing in hotmaps building stock data,floor_area_missing.csv,unknown,https://data.europa.eu/euodp/de/data/dataset/building-stock-observatory +Comparative level investment,comparative_level_investment.csv,Eurostat,https://ec.europa.eu/eurostat/statistics-explained/index.php?title=Comparative_price_levels_for_investment +Electricity taxes,electricity_taxes_eu.csv,Eurostat,https://appsso.eurostat.ec.europa.eu/nui/show.do?dataset=nrg_pc_204&lang=en +Building topologies and corresponding standard values,tabula-calculator-calcsetbuilding.csv,unknown,https://episcope.eu/fileadmin/tabula/public/calc/tabula-calculator.xlsx +Retrofitting thermal envelope costs for Germany,retro_cost_germany.csv,unkown,https://www.iwu.de/forschung/handlungslogiken/kosten-energierelevanter-bau-und-anlagenteile-bei-modernisierung/ District heating most countries,jrc-idees-2015/,CC BY 4.0,https://ec.europa.eu/jrc/en/potencia/jrc-idees,, District heating missing countries,district_heat_share.csv,unkown,https://www.euroheat.org/knowledge-hub/country-profiles,, + diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 4d0b37bf..16889268 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -82,6 +82,9 @@ Future release The transport costs are determined based on the `JRC-EU-Times Bioenergy report `_ in the new optional rule ``build_biomass_transport_costs``. Biomass transport can be activated with the setting ``sector: biomass_transport: true``. +* Use `JRC ENSPRESO database `_ to + spatially disaggregate biomass potentials to PyPSA-Eur regions based on overlaps with NUTS2 regions from ENSPRESO + (proportional to area) (`#151 `_). * Compatibility with ``xarray`` version 0.19. * Separate basic chemicals into HVC, chlorine, methanol and ammonia [`#166 `_]. * Add option to specify reuse, primary production, and mechanical and chemical recycling fraction of platics [`#166 `_]. diff --git a/doc/supply_demand.rst b/doc/supply_demand.rst index 77317094..8e4f2130 100644 --- a/doc/supply_demand.rst +++ b/doc/supply_demand.rst @@ -183,7 +183,7 @@ Solid biomass provides process heat up to 500 Celsius in industry, as well as fe Solid biomass supply ===================== -Only wastes and residues from the JRC biomass dataset. +Only wastes and residues from the JRC ENSPRESO biomass dataset. Oil product demand diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index f02c9093..68d87808 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -1,55 +1,194 @@ import pandas as pd - -rename = {"UK" : "GB", "BH" : "BA"} +import geopandas as gpd -def build_biomass_potentials(): +def build_nuts_population_data(year=2013): - config = snakemake.config['biomass'] - year = config["year"] - scenario = config["scenario"] + pop = pd.read_csv( + snakemake.input.nuts3_population, + sep=r'\,| \t|\t', + engine='python', + na_values=[":"], + index_col=1 + )[str(year)] + + # only countries + pop.drop("EU28", inplace=True) - df = pd.read_excel(snakemake.input.jrc_potentials, - "Potentials (PJ)", - index_col=[0,1]) + # mapping from Cantons to NUTS3 + cantons = pd.read_csv(snakemake.input.swiss_cantons) + cantons = cantons.set_index(cantons.HASC.str[3:]).NUTS + cantons = cantons.str.pad(5, side='right', fillchar='0') - df.rename(columns={"Unnamed: 18": "Municipal waste"}, inplace=True) - df.drop(columns="Total", inplace=True) - df.replace("-", 0., inplace=True) + # get population by NUTS3 + swiss = pd.read_excel(snakemake.input.swiss_population, skiprows=3, index_col=0).loc["Residents in 1000"] + swiss = swiss.rename(cantons).filter(like="CH") - column = df.iloc[:,0] - countries = column.where(column.str.isalpha()).pad() - countries = [rename.get(ct, ct) for ct in countries] - countries_i = pd.Index(countries, name='country') - df.set_index(countries_i, append=True, inplace=True) + # aggregate also to higher order NUTS levels + swiss = [swiss.groupby(swiss.index.str[:i]).sum() for i in range(2, 6)] - df.drop(index='MS', level=0, inplace=True) + # merge Europe + Switzerland + pop = pd.DataFrame(pop.append(swiss), columns=["total"]) + + # add missing manually + pop["AL"] = 2893 + pop["BA"] = 3871 + pop["RS"] = 7210 + + pop["ct"] = pop.index.str[:2] + + return pop - # convert from PJ to MWh - df = df / 3.6 * 1e6 - df.to_csv(snakemake.output.biomass_potentials_all) +def enspreso_biomass_potentials(year=2020, scenario="ENS_Low"): + """ + Loads the JRC ENSPRESO biomass potentials. + + Parameters + ---------- + year : int + The year for which potentials are to be taken. + Can be {2010, 2020, 2030, 2040, 2050}. + scenario : str + The scenario. Can be {"ENS_Low", "ENS_Med", "ENS_High"}. + + Returns + ------- + pd.DataFrame + Biomass potentials for given year and scenario + in TWh/a by commodity and NUTS2 region. + """ - # solid biomass includes: - # Primary agricultural residues (MINBIOAGRW1), - # Forestry energy residue (MINBIOFRSF1), - # Secondary forestry residues (MINBIOWOOW1), - # Secondary Forestry residues – sawdust (MINBIOWOO1a)', - # Forestry residues from landscape care biomass (MINBIOFRSF1a), - # Municipal waste (MINBIOMUN1)', + glossary = pd.read_excel( + str(snakemake.input.enspreso_biomass), + sheet_name="Glossary", + usecols="B:D", + skiprows=1, + index_col=0 + ) + + df = pd.read_excel( + str(snakemake.input.enspreso_biomass), + sheet_name="ENER - NUTS2 BioCom E", + usecols="A:H" + ) - # biogas includes: - # Manure biomass potential (MINBIOGAS1), - # Sludge biomass (MINBIOSLU1), + df["group"] = df["E-Comm"].map(glossary.group) + df["commodity"] = df["E-Comm"].map(glossary.description) - df = df.loc[year, scenario, :] + to_rename = { + "NUTS2 Potential available by Bio Commodity": "potential", + "NUST2": "NUTS2", + } + df.rename(columns=to_rename, inplace=True) + + # fill up with NUTS0 if NUTS2 is not given + df.NUTS2 = df.apply(lambda x: x.NUTS0 if x.NUTS2 == '-' else x.NUTS2, axis=1) - grouper = {v: k for k, vv in config["classes"].items() for v in vv} - df = df.groupby(grouper, axis=1).sum() + # convert PJ to TWh + df.potential /= 3.6 + df.Unit = "TWh/a" - df.index.name = "MWh/a" + dff = df.query("Year == @year and Scenario == @scenario") - df.to_csv(snakemake.output.biomass_potentials) + bio = dff.groupby(["NUTS2", "commodity"]).potential.sum().unstack() + + # currently Serbia and Kosovo not split, so aggregate + bio.loc["RS"] += bio.loc["XK"] + bio.drop("XK", inplace=True) + + return bio + + +def disaggregate_nuts0(bio): + """ + Some commodities are only given on NUTS0 level. + These are disaggregated here using the NUTS2 + population as distribution key. + + Parameters + ---------- + bio : pd.DataFrame + from enspreso_biomass_potentials() + + Returns + ------- + pd.DataFrame + """ + + pop = build_nuts_population_data() + + # get population in nuts2 + pop_nuts2 = pop.loc[pop.index.str.len() == 4] + by_country = pop_nuts2.total.groupby(pop_nuts2.ct).sum() + pop_nuts2["fraction"] = pop_nuts2.total / pop_nuts2.ct.map(by_country) + + # distribute nuts0 data to nuts2 by population + bio_nodal = bio.loc[pop_nuts2.ct] + bio_nodal.index = pop_nuts2.index + bio_nodal = bio_nodal.mul(pop_nuts2.fraction, axis=0) + + # update inplace + bio.update(bio_nodal) + + return bio + + +def build_nuts2_shapes(): + """ + - load NUTS2 geometries + - add RS, AL, BA country shapes (not covered in NUTS 2013) + - consistently name ME, MK + """ + + nuts2 = gpd.GeoDataFrame(gpd.read_file(snakemake.input.nuts2).set_index('id').geometry) + + countries = gpd.read_file(snakemake.input.country_shapes).set_index('name') + missing = countries.loc[["AL", "RS", "BA"]] + nuts2.rename(index={"ME00": "ME", "MK00": "MK"}, inplace=True) + + return nuts2.append(missing) + + +def area(gdf): + """Returns area of GeoDataFrame geometries in square kilometers.""" + return gdf.to_crs(epsg=3035).area.div(1e6) + + +def convert_nuts2_to_regions(bio_nuts2, regions): + """ + Converts biomass potentials given in NUTS2 to PyPSA-Eur regions based on the + overlay of both GeoDataFrames in proportion to the area. + + Parameters + ---------- + bio_nuts2 : gpd.GeoDataFrame + JRC ENSPRESO biomass potentials indexed by NUTS2 shapes. + regions : gpd.GeoDataFrame + PyPSA-Eur clustered onshore regions + + Returns + ------- + gpd.GeoDataFrame + """ + + # calculate area of nuts2 regions + bio_nuts2["area_nuts2"] = area(bio_nuts2) + + overlay = gpd.overlay(regions, bio_nuts2) + + # calculate share of nuts2 area inside region + overlay["share"] = area(overlay) / overlay["area_nuts2"] + + # multiply all nuts2-level values with share of nuts2 inside region + adjust_cols = overlay.columns.difference({"name", "area_nuts2", "geometry", "share"}) + overlay[adjust_cols] = overlay[adjust_cols].multiply(overlay["share"], axis=0) + + bio_regions = overlay.groupby("name").sum() + + bio_regions.drop(["area_nuts2", "share"], axis=1, inplace=True) + + return bio_regions if __name__ == "__main__": @@ -57,12 +196,28 @@ if __name__ == "__main__": from helper import mock_snakemake snakemake = mock_snakemake('build_biomass_potentials') + config = snakemake.config['biomass'] + year = config["year"] + scenario = config["scenario"] - # This is a hack, to be replaced once snakemake is unicode-conform + enspreso = enspreso_biomass_potentials(year, scenario) - solid_biomass = snakemake.config['biomass']['classes']['solid biomass'] - if 'Secondary Forestry residues sawdust' in solid_biomass: - solid_biomass.remove('Secondary Forestry residues sawdust') - solid_biomass.append('Secondary Forestry residues – sawdust') + enspreso = disaggregate_nuts0(enspreso) - build_biomass_potentials() + nuts2 = build_nuts2_shapes() + + df_nuts2 = gpd.GeoDataFrame(nuts2.geometry).join(enspreso) + + regions = gpd.read_file(snakemake.input.regions_onshore) + + df = convert_nuts2_to_regions(df_nuts2, regions) + + df.to_csv(snakemake.output.biomass_potentials_all) + + grouper = {v: k for k, vv in config["classes"].items() for v in vv} + df = df.groupby(grouper, axis=1).sum() + + df *= 1e6 # TWh/a to MWh/a + df.index.name = "MWh/a" + + df.to_csv(snakemake.output.biomass_potentials) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 89ccbebb..320c4726 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1711,17 +1711,10 @@ def add_biomass(n, costs): print("adding biomass") - # biomass distributed at country level - i.e. transport within country allowed - countries = n.buses.country.dropna().unique() - biomass_potentials = pd.read_csv(snakemake.input.biomass_potentials, index_col=0) if options["biomass_transport"]: - # potential per node distributed within country by population - biomass_potentials_spatial = (biomass_potentials.loc[pop_layout.ct] - .set_index(pop_layout.index) - .mul(pop_layout.fraction, axis="index") - .rename(index=lambda x: x + " solid biomass")) + biomass_potentials_spatial = biomass_potentials.rename(index=lambda x: x + " solid biomass") else: biomass_potentials_spatial = biomass_potentials.sum() @@ -1744,9 +1737,9 @@ def add_biomass(n, costs): "EU biogas", bus="EU biogas", carrier="biogas", - e_nom=biomass_potentials.loc[countries, "biogas"].sum(), + e_nom=biomass_potentials["biogas"].sum(), marginal_cost=costs.at['biogas', 'fuel'], - e_initial=biomass_potentials.loc[countries, "biogas"].sum() + e_initial=biomass_potentials["biogas"].sum() ) n.madd("Store", @@ -2237,6 +2230,7 @@ if __name__ == "__main__": lv=1.0, sector_opts='Co2L0-168H-T-H-B-I-solar3-dist1', planning_horizons="2020", + planning_horizons="2030", ) logging.basicConfig(level=snakemake.config['logging_level'])