Fix all data preparation steps up to prepare_network

Snakefile

@@ -1,22 +1,35 @@
 
 configfile: "config.yaml"
 
+wildcard_constraints:
+    lv="[a-z0-9\.]+",
+    simpl="[a-zA-Z0-9]*",
+    clusters="[0-9]+m?",
+    sectors="[+a-zA-Z0-9]+",
+    opts="[-+a-zA-Z0-9]*"
+
+
+
 subworkflow pypsaeur:
     workdir: "../pypsa-eur"
     snakefile: "../pypsa-eur/Snakefile"
     configfile: "../pypsa-eur/config.yaml"
 
 
+rule test_script:
+    input:
+        expand("resources/heat_demand_urban_elec_s_{clusters}.nc",
+                 **config['scenario'])
+
 rule prepare_sector_networks:
     input:
-        pypsaeur(expand(config['results_dir'] + config['run'] + "/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}.nc",
+        expand(config['results_dir'] + config['run'] + "/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}.nc",
-                 **config['scenario']))
+                 **config['scenario'])
-
 
 
 rule build_population_layouts:
     input:
-        nuts3_shapes='resources/nuts3_shapes.geojson',
+        nuts3_shapes=pypsaeur('resources/nuts3_shapes.geojson'),
         urban_percent="data/urban_percent.csv"
     output:
         pop_layout_total="resources/pop_layout_total.nc",
@@ -30,18 +43,18 @@ rule build_clustered_population_layouts:
         pop_layout_total="resources/pop_layout_total.nc",
         pop_layout_urban="resources/pop_layout_urban.nc",
         pop_layout_rural="resources/pop_layout_rural.nc",
-        regions_onshore="resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson"
+        regions_onshore=pypsaeur('resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson')
     output:
         clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv"
     script: "scripts/build_clustered_population_layouts.py"
 
 
-rule build_heat_demand:
+rule build_heat_demands:
     input:
         pop_layout_total="resources/pop_layout_total.nc",
         pop_layout_urban="resources/pop_layout_urban.nc",
         pop_layout_rural="resources/pop_layout_rural.nc",
-        regions_onshore="resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson"
+        regions_onshore=pypsaeur("resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson")
     output:
         heat_demand_urban="resources/heat_demand_urban_{network}_s{simpl}_{clusters}.nc",
         heat_demand_rural="resources/heat_demand_rural_{network}_s{simpl}_{clusters}.nc",
@@ -53,7 +66,7 @@ rule build_temperature_profiles:
         pop_layout_total="resources/pop_layout_total.nc",
         pop_layout_urban="resources/pop_layout_urban.nc",
         pop_layout_rural="resources/pop_layout_rural.nc",
-        regions_onshore="resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson"
+        regions_onshore=pypsaeur("resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson")
     output:
         temp_soil_total="resources/temp_soil_total_{network}_s{simpl}_{clusters}.nc",
         temp_soil_rural="resources/temp_soil_rural_{network}_s{simpl}_{clusters}.nc",
@@ -87,7 +100,7 @@ rule build_solar_thermal_profiles:
         pop_layout_total="resources/pop_layout_total.nc",
         pop_layout_urban="resources/pop_layout_urban.nc",
         pop_layout_rural="resources/pop_layout_rural.nc",
-        regions_onshore="resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson"
+        regions_onshore=pypsaeur("resources/regions_onshore_{network}_s{simpl}_{clusters}.geojson")
     output:
         solar_thermal_total="resources/solar_thermal_total_{network}_s{simpl}_{clusters}.nc",
         solar_thermal_urban="resources/solar_thermal_urban_{network}_s{simpl}_{clusters}.nc",
@@ -98,7 +111,7 @@ rule build_solar_thermal_profiles:
 
 rule build_energy_totals:
     input:
-        nuts3_shapes='resources/nuts3_shapes.geojson'
+        nuts3_shapes=pypsaeur('resources/nuts3_shapes.geojson')
     output:
         energy_name='data/energy_totals.csv',
 	co2_name='data/co2_totals.csv',
@@ -124,3 +137,39 @@ rule build_industrial_demand:
     threads: 1
     resources: mem_mb=1000
     script: 'scripts/build_industrial_demand.py'
+
+
+
+
+rule prepare_network:
+    input:
+        network=config['results_dir']  +  config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}.nc',
+        energy_totals_name='data/energy_totals.csv',
+        co2_totals_name='data/co2_totals.csv',
+        transport_name='data/transport_data.csv',
+        biomass_potentials='data/biomass_potentials.csv',
+        clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv",
+        industrial_demand="resources/industrial_demand_{network}_s{simpl}_{clusters}.csv",
+        heat_demand_urban="resources/heat_demand_urban_{network}_s{simpl}_{clusters}.nc",
+        heat_demand_rural="resources/heat_demand_rural_{network}_s{simpl}_{clusters}.nc",
+        heat_demand_total="resources/heat_demand_total_{network}_s{simpl}_{clusters}.nc",
+        temp_soil_total="resources/temp_soil_total_{network}_s{simpl}_{clusters}.nc",
+        temp_soil_rural="resources/temp_soil_rural_{network}_s{simpl}_{clusters}.nc",
+        temp_soil_urban="resources/temp_soil_urban_{network}_s{simpl}_{clusters}.nc",
+        temp_air_total="resources/temp_air_total_{network}_s{simpl}_{clusters}.nc",
+        temp_air_rural="resources/temp_air_rural_{network}_s{simpl}_{clusters}.nc",
+        temp_air_urban="resources/temp_air_urban_{network}_s{simpl}_{clusters}.nc",
+        cop_soil_total="resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc",
+        cop_soil_rural="resources/cop_soil_rural_{network}_s{simpl}_{clusters}.nc",
+        cop_soil_urban="resources/cop_soil_urban_{network}_s{simpl}_{clusters}.nc",
+        cop_air_total="resources/cop_air_total_{network}_s{simpl}_{clusters}.nc",
+        cop_air_rural="resources/cop_air_rural_{network}_s{simpl}_{clusters}.nc",
+        cop_air_urban="resources/cop_air_urban_{network}_s{simpl}_{clusters}.nc",
+        solar_thermal_total="resources/solar_thermal_total_{network}_s{simpl}_{clusters}.nc",
+        solar_thermal_urban="resources/solar_thermal_urban_{network}_s{simpl}_{clusters}.nc",
+        solar_thermal_rural="resources/solar_thermal_rural_{network}_s{simpl}_{clusters}.nc"
+    output: config['results_dir']  +  config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}.nc'
+    threads: 1
+    resources: mem=1000
+    benchmark: "benchmarks/prepare_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}"
+    script: "scripts/prepare_network.py"

config.yaml

@@ -10,3 +10,83 @@ scenario:
   clusters: [128] #[90, 128, 181] #[45, 64, 90, 128, 181, 256] #, 362] # (2**np.r_[5.5:9:.5]).astype(int) minimum is 37
   opts: [Co2L0-3H-T-H-B-I,Co2L0-3H-T-H-B-I-onwind0,Co2L0-3H-T-H-B-I-onwind0p1]#,Co2L0p05-3H-T-H-B-I,Co2L0p10-3H-T-H-B-I,Co2L0p20-3H-T-H-B-I,Co2L0p30-3H-T-H-B-I,Co2L0p50-3H-T-H-B-I]#[Co2L-3H-T-H,Co2L0p10-3H-T-H,Co2L0-3H-T-H,Co2L0p20-3H-T-H]   #Co2L-3H-T-H,Co2L0p10-3H-T-H,Co2L0p20-3H-T-HCo2L-3H-T-H,Co2L0p10-3H-T-H,Co2L0p30-3H-T-H,Co2L0p50-3H-T-H] #Co2L-3H,Co2L-3H-T,, LC-FL, LC-T, Ep-T, Co2L-T]
   # Co2L will give default (5%); Co2L0p25 will give 25% CO2 emissions; Co2Lm0p05 will give 5% negative emissions
+
+
+snapshots:
+  # arguments to pd.date_range
+  start: "2013-01-01"
+  end: "2014-01-01"
+  closed: 'left' # end is not inclusive
+
+atlite:
+  nprocesses: 4
+  cutout_dir: '../pypsa-eur/cutouts'
+  cutouts:
+    europe-2013-era5:
+      module: era5
+      xs: [-12., 35.]
+      ys: [72., 33.]
+      years: [2013, 2013]
+    europe-2013-sarah:
+      module: sarah
+      resolution: 0.2
+      xs: [-12., 42.]
+      ys: [65., 33.]
+      years: [2013, 2013]
+
+renewable:
+  onwind:
+    cutout: europe-2013-era5
+    resource:
+      method: wind
+      turbine: Vestas_V112_3MW
+    # ScholzPhd Tab 4.3.1: 10MW/km^2
+    capacity_per_sqm: 3
+    # correction_factor: 0.93
+    corine:
+      #The selection of CORINE Land Cover [1] types that are allowed for wind and solar are based on [2] p.42 / p.28
+      #
+      #[1] https://www.eea.europa.eu/ds_resolveuid/C9RK15EA06
+      #
+      #[2] Scholz, Y. (2012). Renewable energy based electricity supply at low costs: development of the REMix model and application for Europe.
+      grid_codes: [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+                   24, 25, 26, 27, 28, 29, 31, 32]
+      distance: 1000
+      distance_grid_codes: [1, 2, 3, 4, 5, 6]
+      natura: true
+  offwind:
+    cutout: europe-2013-era5
+    resource:
+      method: wind
+      turbine: NREL_ReferenceTurbine_5MW_offshore
+    # ScholzPhd Tab 4.3.1: 10MW/km^2
+    capacity_per_sqm: 3
+    height_cutoff: 50
+    # correction_factor: 0.93
+    corine:
+      grid_codes: [44, 255]
+      natura: true
+  solar:
+    cutout: europe-2013-sarah
+    resource:
+      method: pv
+      panel: CSi
+      orientation:
+        slope: 35.
+        azimuth: 180.
+    # ScholzPhd Tab 4.3.1: 170 MW/km^2
+    capacity_per_sqm: 1.7
+    correction_factor: 0.877
+    corine:
+      grid_codes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
+      18, 19, 20, 26, 31, 32]
+      natura: true
+  hydro:
+    cutout: europe-2013-era5
+    carriers: [ror, PHS, hydro]
+    PHS_max_hours: 6
+
+
+biomass:
+  year: 2030
+  scenario: "Med"

scripts/build_biomass_potentials.py

@@ -7,9 +7,10 @@ def build_biomass_potentials():
 
     #delete empty column C from this sheet first before reading it in
     df = pd.read_excel(snakemake.input.jrc_potentials,
-                       "Potentials (PJ)")
+                       "Potentials (PJ)",
+                       index_col=[0,1])
 
-    df.rename(columns={"Unnamed: 16":"Municipal waste"},inplace=True)
+    df.rename(columns={"Unnamed: 18":"Municipal waste"},inplace=True)
     df.drop(columns="Total",inplace=True)
     df.replace("-",0.,inplace=True)