From 0bad2097b051c2bd34a00893ec30c04cd06b3372 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 25 Jul 2022 14:35:54 +0200 Subject: [PATCH] interpolate heat demands for years outside 2007-2015 --- Snakefile | 19 ++++-- data/era5-annual-HDD-per-country.csv | 34 ++++++++++ scripts/build_energy_totals.py | 3 - scripts/build_heat_totals.py | 63 +++++++++++++++++++ ...build_population_weighted_energy_totals.py | 21 ++++--- scripts/prepare_sector_network.py | 2 + 6 files changed, 128 insertions(+), 14 deletions(-) create mode 100644 data/era5-annual-HDD-per-country.csv create mode 100644 scripts/build_heat_totals.py diff --git a/Snakefile b/Snakefile index 04356842..617d30f2 100644 --- a/Snakefile +++ b/Snakefile @@ -243,7 +243,6 @@ rule build_energy_totals: district_heat_share='data/district_heat_share.csv', eurostat=directory("data/eurostat-energy_balances-june_2021_edition"), output: - energy_name_full='resources/energy_totals_full.csv', energy_name='resources/energy_totals.csv', co2_name='resources/co2_totals.csv', transport_name='resources/transport_data.csv' @@ -253,6 +252,18 @@ rule build_energy_totals: script: 'scripts/build_energy_totals.py' +rule build_heat_totals: + input: + hdd="data/era5-annual-HDD-per-country.csv", + energy_totals="resources/energy_totals.csv", + output: + heat_totals="resources/heat_totals.csv" + threads: 1 + resources: mem_mb=2000 + benchmark: "benchmarks/build_heat_totals" + script: "scripts/build_heat_totals.py" + + rule build_biomass_potentials: input: enspreso_biomass=HTTP.remote("https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", keep_local=True), @@ -435,9 +446,9 @@ else: rule build_population_weighted_energy_totals: input: - energy_totals='resources/energy_totals.csv', + totals='resources/{kind}_totals.csv', clustered_pop_layout="resources/pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv" - output: "resources/pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv" + output: "resources/pop_weighted_{kind}_totals{weather_year}_s{simpl}_{clusters}.csv" threads: 1 resources: mem_mb=2000 script: "scripts/build_population_weighted_energy_totals.py" @@ -465,8 +476,8 @@ rule prepare_sector_network: input: overrides="data/override_component_attrs", network=pypsaeur('networks/elec{weather_year}_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc'), - energy_totals_name='resources/energy_totals.csv', pop_weighted_energy_totals="resources/pop_weighted_energy_totals{weather_year}_s{simpl}_{clusters}.csv", + pop_weighted_heat_totals="resources/pop_weighted_heat_totals{weather_year}_s{simpl}_{clusters}.csv", transport_demand="resources/transport_demand{weather_year}_s{simpl}_{clusters}.csv", transport_data="resources/transport_data{weather_year}_s{simpl}_{clusters}.csv", avail_profile="resources/avail_profile{weather_year}_s{simpl}_{clusters}.csv", diff --git a/data/era5-annual-HDD-per-country.csv b/data/era5-annual-HDD-per-country.csv new file mode 100644 index 00000000..472257c6 --- /dev/null +++ b/data/era5-annual-HDD-per-country.csv @@ -0,0 +1,34 @@ +name,1951,1952,1953,1954,1955,1956,1957,1958,1959,1960,1961,1962,1963,1964,1965,1966,1967,1968,1969,1970,1971,1972,1973,1974,1975,1976,1977,1978,1979,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021 +AL,56,58,65,67,55,70,58,59,61,54,54,62,62,62,68,59,60,57,57,58,61,57,64,60,61,61,54,63,57,65,63,58,61,57,57,58,61,62,56,53,67,62,62,53,61,61,60,60,56,58,57,53,61,58,65,64,55,55,54,53,58,58,50,46,54,50,54,47,49,50,54 +AT,374,416,379,422,419,449,387,393,378,389,367,437,425,406,424,376,385,400,409,413,394,407,408,379,384,400,380,414,405,428,397,384,377,411,420,406,409,378,369,372,415,373,383,334,380,416,371,373,368,343,371,337,370,370,384,362,333,340,351,388,337,353,358,302,324,336,347,315,320,322,358 +BE,95,105,94,102,109,113,91,97,88,89,84,111,114,100,100,91,94,101,102,101,96,102,101,90,97,97,92,103,108,105,99,94,96,99,112,107,108,85,84,83,101,92,94,82,89,110,90,89,84,79,89,78,89,90,87,86,76,87,87,107,75,90,99,69,81,88,82,81,80,71,89 +BG,283,294,334,367,295,363,306,283,320,268,277,303,324,313,323,259,297,292,320,286,300,306,323,303,296,314,287,310,293,326,301,312,295,306,326,313,333,320,276,272,332,303,318,266,307,335,325,303,273,275,284,272,322,277,298,296,259,266,266,273,313,295,258,250,261,265,281,261,232,250,277 +BA,144,164,167,183,158,194,157,157,152,143,141,173,180,176,176,155,162,157,165,165,163,154,170,156,153,168,145,170,153,176,163,154,157,159,167,160,166,158,146,139,174,152,162,143,157,171,156,158,147,132,145,130,156,146,164,148,134,132,139,147,144,147,133,113,137,134,144,128,125,127,141 +CH,214,225,210,226,221,240,211,215,203,214,200,234,231,214,229,213,217,219,225,227,217,221,224,213,216,220,208,219,220,229,216,208,208,227,224,217,217,200,202,200,217,204,208,183,206,217,195,205,203,192,200,189,199,204,209,193,187,194,193,213,179,196,204,178,181,189,191,174,184,176,197 +CZ,297,344,307,351,350,377,313,322,301,314,294,358,361,345,351,297,296,320,342,336,319,326,322,282,296,315,299,330,328,347,316,308,298,323,350,326,346,297,280,283,333,299,316,279,312,364,319,296,288,265,309,285,308,302,311,300,269,275,289,338,281,298,306,242,261,283,287,262,255,259,302 +DE,1254,1428,1237,1416,1484,1567,1282,1342,1229,1266,1183,1486,1528,1390,1399,1269,1211,1341,1447,1429,1300,1388,1356,1184,1258,1339,1221,1377,1433,1433,1348,1288,1273,1384,1492,1425,1491,1201,1156,1135,1367,1241,1312,1155,1295,1548,1288,1220,1172,1085,1250,1168,1269,1236,1239,1198,1072,1159,1202,1459,1112,1232,1305,992,1095,1188,1148,1092,1070,1019,1230 +DK,160,176,152,175,186,190,163,178,154,169,154,184,194,176,182,181,153,170,182,186,159,166,159,147,147,172,161,171,190,177,175,164,155,162,191,181,190,151,137,132,160,147,166,154,162,191,162,154,147,134,161,148,158,151,150,144,134,137,150,193,144,159,160,125,138,145,139,142,134,125,150 +ES,962,911,927,948,785,1092,934,848,821,890,742,952,981,967,949,878,890,839,933,899,971,961,943,931,928,928,797,906,887,911,793,787,807,920,860,891,781,783,684,767,911,852,889,724,627,759,605,735,796,750,757,677,746,813,856,714,776,785,736,856,673,810,830,649,684,722,688,768,708,672,728 +EE,297,310,283,297,321,336,278,310,281,305,253,298,323,292,306,324,283,310,333,317,285,285,289,256,245,327,302,324,299,313,289,278,260,271,328,294,332,281,232,240,257,257,285,288,265,306,281,278,262,235,273,271,278,270,270,260,250,230,268,315,253,286,259,254,225,261,255,254,235,209,270 +FI,3285,3442,3085,3179,3880,3819,3296,3626,3180,3386,3065,3513,3467,3329,3526,3873,3284,3693,3640,3494,3572,3178,3458,2963,3074,3691,3490,3743,3419,3607,3572,3331,3296,3192,3894,3516,3793,3402,2888,3075,3209,3163,3319,3341,3213,3343,3302,3468,3259,2915,3303,3307,3231,3159,2998,3101,3018,3009,3219,3604,2961,3351,2985,2983,2795,3030,3127,3061,3116,2726,3260 +FR,1430,1480,1422,1489,1396,1633,1338,1378,1240,1332,1196,1584,1630,1499,1471,1331,1408,1412,1473,1469,1475,1453,1545,1356,1462,1445,1312,1454,1502,1545,1396,1315,1424,1484,1599,1518,1520,1269,1237,1227,1501,1375,1389,1168,1280,1454,1222,1337,1263,1190,1299,1132,1294,1346,1376,1263,1199,1301,1277,1522,1060,1300,1408,1034,1143,1266,1236,1150,1165,1043,1279 +GB,906,934,809,901,925,935,813,890,787,849,825,1000,1046,912,970,917,839,874,894,869,805,880,862,842,838,851,863,873,972,880,896,823,824,838,923,944,903,801,764,746,859,824,857,793,794,886,738,752,745,771,816,726,751,743,748,734,706,790,777,930,707,832,841,683,759,769,720,764,750,725,764 +GR,166,162,210,211,160,215,184,169,196,164,172,182,184,192,199,162,185,176,171,164,184,185,195,186,188,190,160,186,172,191,184,192,190,179,174,180,194,193,176,165,206,197,195,161,179,191,192,180,167,177,174,164,191,172,184,189,161,158,155,143,188,179,148,136,165,150,164,139,146,148,156 +HR,116,140,140,155,137,166,131,132,122,119,114,149,158,152,144,124,133,129,143,138,137,131,144,124,124,139,121,140,130,149,133,132,130,133,148,141,144,128,118,114,143,123,138,115,126,145,132,129,124,106,120,109,135,127,140,123,109,110,116,130,123,123,118,92,114,116,119,111,102,105,119 +HU,236,291,282,321,289,333,272,273,246,248,235,288,321,309,298,244,272,268,293,283,276,264,283,244,255,281,267,287,271,311,272,282,265,274,313,294,305,274,243,238,299,262,288,240,271,310,291,272,267,232,269,243,300,267,289,265,229,230,243,275,267,259,250,198,235,251,255,234,211,228,259 +IE,216,215,191,208,213,211,190,204,184,205,195,234,250,208,232,210,216,213,222,214,190,226,204,213,198,211,210,205,239,212,209,204,201,211,229,241,217,200,192,192,209,209,213,201,195,219,180,186,191,204,206,189,192,193,187,188,173,202,204,241,192,205,210,186,205,199,184,200,191,194,189 +IT,652,683,660,700,615,778,640,653,616,631,587,715,726,674,710,665,655,656,682,672,678,639,695,659,655,670,604,681,675,729,684,645,657,695,688,671,676,630,616,610,712,626,650,555,623,639,578,618,619,563,585,546,629,612,675,597,548,568,590,637,562,602,587,481,541,528,570,529,540,520,577 +LT,344,371,342,368,368,405,318,353,329,346,303,355,386,362,371,354,322,365,401,372,330,345,339,312,289,388,348,370,370,382,342,326,301,326,393,352,397,335,275,272,313,311,336,329,325,377,335,325,306,274,323,313,328,324,325,316,299,277,316,366,305,332,314,298,273,308,296,302,267,252,327 +LU,8,9,8,9,10,10,8,9,8,8,7,10,10,9,9,8,8,9,9,9,9,9,9,8,9,9,8,9,10,10,9,9,9,9,10,10,10,8,8,8,9,8,9,8,8,10,8,8,8,7,8,7,8,8,8,8,7,8,8,9,7,8,9,6,7,8,8,7,7,7,8 +LV,383,402,372,394,407,439,351,391,361,384,326,386,420,385,402,401,360,401,434,407,361,372,371,337,316,424,385,411,392,411,372,357,330,354,425,379,428,362,300,301,334,337,366,366,348,402,365,358,334,302,353,346,357,352,354,337,324,300,346,402,330,365,342,329,297,337,330,328,297,272,351 +MK,64,66,74,79,66,79,67,66,71,63,64,72,73,70,77,66,69,66,67,67,70,67,75,69,70,69,62,72,66,73,71,69,70,67,69,67,70,71,65,63,76,68,69,59,69,70,71,68,64,65,65,62,69,64,70,68,60,59,61,59,67,66,56,53,61,58,62,54,55,57,63 +ME,40,43,45,48,41,50,42,42,42,39,39,44,45,45,48,43,43,42,42,43,43,41,44,43,42,45,39,45,42,46,45,41,43,42,42,41,44,44,42,39,47,43,43,38,43,44,42,43,39,39,40,37,42,41,45,43,38,38,38,38,39,40,36,32,37,36,39,34,35,35,39 +NL,113,126,110,121,130,137,108,118,107,107,104,132,141,121,120,114,105,119,124,122,111,120,116,104,111,116,107,120,133,122,118,112,110,118,134,127,130,99,99,94,119,106,113,102,109,138,111,102,97,93,106,98,109,103,100,100,87,102,103,131,93,108,117,82,95,104,96,98,93,85,105 +NO,3339,3541,3020,3307,3639,3731,3315,3547,3122,3353,3400,3807,3641,3496,3783,3951,3259,3533,3425,3431,3339,3128,3312,3028,3094,3405,3369,3436,3457,3380,3502,3181,3141,3117,3541,3359,3449,3200,2890,2848,3057,3032,3196,3207,3170,3301,3060,3203,3062,2878,3166,3028,2983,2943,2874,2828,2918,2954,3018,3442,2796,3157,2946,2754,2793,2859,2963,2930,2981,2711,3035 +PL,1249,1438,1294,1478,1446,1586,1279,1349,1269,1305,1215,1435,1527,1450,1467,1280,1193,1321,1504,1431,1293,1347,1314,1200,1184,1419,1270,1393,1431,1491,1336,1265,1178,1304,1494,1377,1507,1255,1077,1078,1312,1252,1310,1197,1302,1512,1326,1249,1171,1052,1268,1192,1296,1230,1262,1244,1117,1092,1220,1426,1177,1274,1251,1068,1063,1163,1158,1120,1019,1017,1252 +PT,107,103,99,109,80,131,106,93,93,97,78,106,104,115,103,98,103,97,112,110,115,116,106,109,109,109,99,139,107,102,89,90,95,104,99,108,85,89,75,87,104,96,104,85,66,88,62,82,96,90,90,78,87,97,103,90,91,91,84,96,80,101,100,81,79,87,79,96,81,72,78 +RO,736,811,883,948,822,968,810,789,826,731,744,826,890,867,873,706,791,776,858,792,787,798,838,782,752,855,772,836,780,880,801,811,763,807,917,815,882,827,710,700,846,801,848,701,806,869,842,805,740,710,760,721,844,754,799,780,684,695,695,757,793,781,709,662,681,721,720,690,624,648,740 +RS,222,252,273,300,249,313,248,249,254,231,226,270,283,271,274,233,253,243,267,254,251,246,272,242,243,265,233,265,241,273,251,255,244,253,275,256,268,254,231,220,275,243,263,223,250,270,259,250,237,217,234,217,262,238,265,247,217,214,222,229,251,244,214,192,222,223,232,211,197,212,234 +SK,178,210,195,217,208,228,192,193,185,185,178,208,213,211,219,179,187,192,203,200,192,188,195,178,178,197,188,204,193,215,191,190,180,193,212,198,207,187,172,173,202,186,191,171,186,204,193,184,177,162,184,174,190,184,191,182,163,161,170,188,175,179,174,142,161,170,176,157,152,158,180 +SI,61,72,67,73,69,78,65,66,62,61,59,75,75,73,73,62,65,67,71,69,67,68,71,62,61,69,62,71,68,74,67,65,65,69,73,71,71,63,59,58,70,61,63,55,61,71,63,62,62,54,59,55,65,63,67,61,54,57,57,65,58,59,59,46,55,57,58,53,52,53,61 +SE,3891,4219,3560,3919,4426,4488,3950,4223,3662,3988,3814,4451,4260,4021,4358,4613,3929,4280,4255,4254,4043,3806,3975,3634,3625,4238,4132,4314,4246,4287,4301,3913,3840,3819,4588,4139,4376,3931,3476,3446,3785,3695,3893,3991,3916,4073,3757,3950,3781,3446,3898,3778,3755,3769,3632,3561,3606,3590,3806,4397,3474,3935,3675,3452,3421,3635,3693,3705,3689,3247,3807 diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 04147bb8..d9763779 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -705,9 +705,6 @@ if __name__ == "__main__": idees = build_idees(idees_countries) energy = build_energy_totals(countries, eurostat, swiss, idees) - energy.to_csv(snakemake.output.energy_name_full) - - energy = energy.xs(data_year, level='year') energy.to_csv(snakemake.output.energy_name) base_year_emissions = config["base_emissions_year"] diff --git a/scripts/build_heat_totals.py b/scripts/build_heat_totals.py new file mode 100644 index 00000000..34d5dcdf --- /dev/null +++ b/scripts/build_heat_totals.py @@ -0,0 +1,63 @@ +"""Approximate heat demand for all weather years.""" + +import pandas as pd + +from itertools import product +from numpy.polynomial import Polynomial + +idx = pd.IndexSlice + + +def approximate_heat_demand(energy_totals, hdd): + + if isinstance(hdd, str): + hdd = pd.read_csv(hdd, index_col=0).T + hdd.index = hdd.index.astype(int) + + demands = {} + + for kind, sector in product(["total", "electricity"], ["services", "residential"]): + + row = idx[:, 2007:2015] + col = f"{kind} {sector} space" + demand = energy_totals.loc[row, col].unstack(0) + + demand_approx = {} + + for c in countries: + + Y = demand[c].dropna() + X = hdd.loc[Y.index, c] + + to_predict = hdd.index.difference(Y.index) + X_pred = hdd.loc[to_predict, c] + + p = Polynomial.fit(X, Y, 1) + Y_pred = p(X_pred) + + demand_approx[c] = pd.Series(Y_pred, index=to_predict) + + demand_approx = pd.DataFrame(demand_approx) + demand_approx = pd.concat([demand, demand_approx]).sort_index() + demands[f"{kind} {sector} space"] = demand_approx.groupby(demand_approx.index).sum() + + demands = pd.concat(demands).unstack().T.clip(lower=0) + demands.index.names = ["country", "year"] + + return demands + + +if __name__ == "__main__": + if 'snakemake' not in globals(): + from helper import mock_snakemake + snakemake = mock_snakemake('build_energy_totals') + + hdd = pd.read_csv(snakemake.input.hdd, index_col=0).T + + energy_totals = pd.read_csv(snakemake.input.energy_totals, index_col=[0,1]) + + countries = hdd.columns + + heat_demand = approximate_heat_demand(energy_totals, hdd) + + heat_demand.to_csv(snakemake.output.heat_totals) diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py index 938983d5..933a0417 100644 --- a/scripts/build_population_weighted_energy_totals.py +++ b/scripts/build_population_weighted_energy_totals.py @@ -1,4 +1,4 @@ -"""Build population-weighted energy totals.""" +"""Build population-weighted energy and heat totals.""" import pandas as pd @@ -7,16 +7,23 @@ if __name__ == '__main__': from helper import mock_snakemake snakemake = mock_snakemake( 'build_population_weighted_energy_totals', + weather_year='', simpl='', - clusters=48, + clusters=37, ) + config = snakemake.config["energy"] + data_year = int(config["energy_totals_year"]) + if snakemake.wildcards.weather_year and snakemake.wildcards.kind == 'heat': + data_year = int(snakemake.wildcards.weather_year) + pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) - energy_totals = pd.read_csv(snakemake.input.energy_totals, index_col=0) + totals = pd.read_csv(snakemake.input.totals, index_col=[0,1]) + totals = totals.xs(data_year, level='year') - nodal_energy_totals = energy_totals.loc[pop_layout.ct].fillna(0.) - nodal_energy_totals.index = pop_layout.index - nodal_energy_totals = nodal_energy_totals.multiply(pop_layout.fraction, axis=0) + nodal_totals = totals.loc[pop_layout.ct].fillna(0.) + nodal_totals.index = pop_layout.index + nodal_totals = nodal_totals.multiply(pop_layout.fraction, axis=0) - nodal_energy_totals.to_csv(snakemake.output[0]) + nodal_totals.to_csv(snakemake.output[0]) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 14e824c0..3f25251f 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -2359,6 +2359,8 @@ if __name__ == "__main__": snakemake.config['costs']['lifetime']) pop_weighted_energy_totals = pd.read_csv(snakemake.input.pop_weighted_energy_totals, index_col=0) + pop_weighted_heat_totals = pd.read_csv(snakemake.input.pop_weighted_heat_totals, index_col=0) + pop_weighted_energy_totals.update(pop_weighted_heat_totals) patch_electricity_network(n)