Merge pull request #2 from PyPSA/master

update with master

.gitignore

@@ -25,6 +25,7 @@ gurobi.log
 /data/transport_data.csv
 /data/switzerland*
 /data/.nfs*
+/data/Industrial_Database.csv
 
 *.org
 
@@ -43,3 +44,5 @@ gurobi.log
 config.yaml
 
 doc/_build
+
+*.xls

Snakefile

@@ -3,6 +3,7 @@ configfile: "config.yaml"
 
 wildcard_constraints:
     lv="[a-z0-9\.]+",
+    network="[a-zA-Z0-9]*",
     simpl="[a-zA-Z0-9]*",
     clusters="[0-9]+m?",
     sectors="[+a-zA-Z0-9]+",
@@ -24,17 +25,12 @@ rule all:
 
 rule solve_all_networks:
     input:
-        expand(config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc",
+        expand(config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
                **config['scenario'])
 
-rule test_script:
-    input:
-        expand("resources/heat_demand_urban_elec_s_{clusters}.nc",
-                 **config['scenario'])
-
 rule prepare_sector_networks:
     input:
-        expand(config['results_dir'] + config['run'] + "/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc",
+        expand(config['results_dir'] + config['run'] + "/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
                  **config['scenario'])
 
 
@@ -62,6 +58,18 @@ rule build_clustered_population_layouts:
     script: "scripts/build_clustered_population_layouts.py"
 
 
+rule build_simplified_population_layouts:
+    input:
+        pop_layout_total="resources/pop_layout_total.nc",
+        pop_layout_urban="resources/pop_layout_urban.nc",
+        pop_layout_rural="resources/pop_layout_rural.nc",
+        regions_onshore=pypsaeur('resources/regions_onshore_{network}_s{simpl}.geojson')
+    output:
+        clustered_pop_layout="resources/pop_layout_{network}_s{simpl}.csv"
+    resources: mem_mb=10000
+    script: "scripts/build_clustered_population_layouts.py"
+
+
 rule build_heat_demands:
     input:
         pop_layout_total="resources/pop_layout_total.nc",
@@ -130,9 +138,9 @@ rule build_energy_totals:
     input:
         nuts3_shapes=pypsaeur('resources/nuts3_shapes.geojson')
     output:
-        energy_name='data/energy_totals.csv',
-	co2_name='data/co2_totals.csv',
-	transport_name='data/transport_data.csv'
+        energy_name='resources/energy_totals.csv',
+	co2_name='resources/co2_totals.csv',
+	transport_name='resources/transport_data.csv'
     threads: 1
     resources: mem_mb=10000
     script: 'scripts/build_energy_totals.py'
@@ -141,13 +149,25 @@ rule build_biomass_potentials:
     input:
         jrc_potentials="data/biomass/JRC Biomass Potentials.xlsx"
     output:
-        biomass_potentials='data/biomass_potentials.csv'
+        biomass_potentials_all='resources/biomass_potentials_all.csv',
+        biomass_potentials='resources/biomass_potentials.csv'
     threads: 1
     resources: mem_mb=1000
     script: 'scripts/build_biomass_potentials.py'
 
+rule build_ammonia_production:
+    input:
+        usgs="data/myb1-2017-nitro.xls"
+    output:
+        ammonia_production="resources/ammonia_production.csv"
+    threads: 1
+    resources: mem_mb=1000
+    script: 'scripts/build_ammonia_production.py'
+
 
 rule build_industry_sector_ratios:
+    input:
+        ammonia_production="resources/ammonia_production.csv"
     output:
         industry_sector_ratios="resources/industry_sector_ratios.csv"
     threads: 1
@@ -155,43 +175,145 @@ rule build_industry_sector_ratios:
     script: 'scripts/build_industry_sector_ratios.py'
 
 
-rule build_industrial_demand_per_country:
+rule build_industrial_production_per_country:
     input:
-        industry_sector_ratios="resources/industry_sector_ratios.csv"
+        ammonia_production="resources/ammonia_production.csv"
     output:
-        industrial_demand_per_country="resources/industrial_demand_per_country.csv"
+        industrial_production_per_country="resources/industrial_production_per_country.csv"
     threads: 1
     resources: mem_mb=1000
-    script: 'scripts/build_industrial_demand_per_country.py'
+    script: 'scripts/build_industrial_production_per_country.py'
+
+
+rule build_industrial_production_per_country_tomorrow:
+    input:
+        industrial_production_per_country="resources/industrial_production_per_country.csv"
+    output:
+        industrial_production_per_country_tomorrow="resources/industrial_production_per_country_tomorrow.csv"
+    threads: 1
+    resources: mem_mb=1000
+    script: 'scripts/build_industrial_production_per_country_tomorrow.py'
+
+
+
+
+rule build_industrial_distribution_key:
+    input:
+        clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv",
+        europe_shape=pypsaeur('resources/europe_shape.geojson'),
+        hotmaps_industrial_database="data/Industrial_Database.csv",
+        network=pypsaeur('networks/{network}_s{simpl}_{clusters}.nc')
+    output:
+        industrial_distribution_key="resources/industrial_distribution_key_{network}_s{simpl}_{clusters}.csv"
+    threads: 1
+    resources: mem_mb=1000
+    script: 'scripts/build_industrial_distribution_key.py'
+
+
+
+rule build_industrial_production_per_node:
+    input:
+        industrial_distribution_key="resources/industrial_distribution_key_{network}_s{simpl}_{clusters}.csv",
+        industrial_production_per_country_tomorrow="resources/industrial_production_per_country_tomorrow.csv"
+    output:
+        industrial_production_per_node="resources/industrial_production_{network}_s{simpl}_{clusters}.csv"
+    threads: 1
+    resources: mem_mb=1000
+    script: 'scripts/build_industrial_production_per_node.py'
+
+
+rule build_industrial_energy_demand_per_node:
+    input:
+        industry_sector_ratios="resources/industry_sector_ratios.csv",
+        industrial_production_per_node="resources/industrial_production_{network}_s{simpl}_{clusters}.csv",
+        industrial_energy_demand_per_node_today="resources/industrial_energy_demand_today_{network}_s{simpl}_{clusters}.csv"
+    output:
+        industrial_energy_demand_per_node="resources/industrial_energy_demand_{network}_s{simpl}_{clusters}.csv"
+    threads: 1
+    resources: mem_mb=1000
+    script: 'scripts/build_industrial_energy_demand_per_node.py'
+
+
+rule build_industrial_energy_demand_per_country_today:
+    input:
+        ammonia_production="resources/ammonia_production.csv",
+        industrial_production_per_country="resources/industrial_production_per_country.csv"
+    output:
+        industrial_energy_demand_per_country_today="resources/industrial_energy_demand_per_country_today.csv"
+    threads: 1
+    resources: mem_mb=1000
+    script: 'scripts/build_industrial_energy_demand_per_country_today.py'
+
+
+rule build_industrial_energy_demand_per_node_today:
+    input:
+        industrial_distribution_key="resources/industrial_distribution_key_{network}_s{simpl}_{clusters}.csv",
+        industrial_energy_demand_per_country_today="resources/industrial_energy_demand_per_country_today.csv"
+    output:
+        industrial_energy_demand_per_node_today="resources/industrial_energy_demand_today_{network}_s{simpl}_{clusters}.csv"
+    threads: 1
+    resources: mem_mb=1000
+    script: 'scripts/build_industrial_energy_demand_per_node_today.py'
+
+
+
+rule build_industrial_energy_demand_per_country:
+    input:
+        industry_sector_ratios="resources/industry_sector_ratios.csv",
+        industrial_production_per_country="resources/industrial_production_per_country_tomorrow.csv"
+    output:
+        industrial_energy_demand_per_country="resources/industrial_energy_demand_per_country.csv"
+    threads: 1
+    resources: mem_mb=1000
+    script: 'scripts/build_industrial_energy_demand_per_country.py'
 
 
 rule build_industrial_demand:
     input:
         clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv",
-        industrial_demand_per_country="resources/industrial_demand_per_country.csv"
+        industrial_demand_per_country="resources/industrial_energy_demand_per_country.csv"
     output:
         industrial_demand="resources/industrial_demand_{network}_s{simpl}_{clusters}.csv"
     threads: 1
     resources: mem_mb=1000
     script: 'scripts/build_industrial_demand.py'
 
+rule build_retro_cost:
+    input:
+        building_stock="data/retro/data_building_stock.csv",
+        u_values_PL="data/retro/u_values_poland.csv",
+        tax_w="data/retro/electricity_taxes_eu.csv",
+        construction_index="data/retro/comparative_level_investment.csv",
+        average_surface="data/retro/average_surface_components.csv",
+        floor_area_missing="data/retro/floor_area_missing.csv",
+        clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv",
+        cost_germany="data/retro/retro_cost_germany.csv",
+        window_assumptions="data/retro/window_assumptions.csv"
+    output:
+        retro_cost="resources/retro_cost_{network}_s{simpl}_{clusters}.csv",
+        floor_area="resources/floor_area_{network}_s{simpl}_{clusters}.csv"
+    script: "scripts/build_retro_cost.py"
+
 
 rule prepare_sector_network:
     input:
         network=pypsaeur('networks/{network}_s{simpl}_{clusters}_ec_lv{lv}_{opts}.nc'),
-        energy_totals_name='data/energy_totals.csv',
-        co2_totals_name='data/co2_totals.csv',
-        transport_name='data/transport_data.csv',
-        biomass_potentials='data/biomass_potentials.csv',
+        energy_totals_name='resources/energy_totals.csv',
+        co2_totals_name='resources/co2_totals.csv',
+        transport_name='resources/transport_data.csv',
+	traffic_data = "data/emobility/",
+        biomass_potentials='resources/biomass_potentials.csv',
         timezone_mappings='data/timezone_mappings.csv',
         heat_profile="data/heat_load_profile_BDEW.csv",
         costs=config['costs_dir'] + "costs_{planning_horizons}.csv",
-        co2_budget="data/co2_budget.csv",
+	h2_cavern = "data/hydrogen_salt_cavern_potentials.csv",
         profile_offwind_ac=pypsaeur("resources/profile_offwind-ac.nc"),
         profile_offwind_dc=pypsaeur("resources/profile_offwind-dc.nc"),
-        clustermaps=pypsaeur('resources/clustermaps_{network}_s{simpl}_{clusters}.h5'),
+        busmap_s=pypsaeur("resources/busmap_{network}_s{simpl}.csv"),
+        busmap=pypsaeur("resources/busmap_{network}_s{simpl}_{clusters}.csv"),
         clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv",
-        industrial_demand="resources/industrial_demand_{network}_s{simpl}_{clusters}.csv",
+        simplified_pop_layout="resources/pop_layout_{network}_s{simpl}.csv",
+        industrial_demand="resources/industrial_energy_demand_{network}_s{simpl}_{clusters}.csv",
         heat_demand_urban="resources/heat_demand_urban_{network}_s{simpl}_{clusters}.nc",
         heat_demand_rural="resources/heat_demand_rural_{network}_s{simpl}_{clusters}.nc",
         heat_demand_total="resources/heat_demand_total_{network}_s{simpl}_{clusters}.nc",
@@ -209,21 +331,23 @@ rule prepare_sector_network:
         cop_air_urban="resources/cop_air_urban_{network}_s{simpl}_{clusters}.nc",
         solar_thermal_total="resources/solar_thermal_total_{network}_s{simpl}_{clusters}.nc",
         solar_thermal_urban="resources/solar_thermal_urban_{network}_s{simpl}_{clusters}.nc",
-        solar_thermal_rural="resources/solar_thermal_rural_{network}_s{simpl}_{clusters}.nc"
-    output: config['results_dir']  +  config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc'
+        solar_thermal_rural="resources/solar_thermal_rural_{network}_s{simpl}_{clusters}.nc",
+	retro_cost_energy = "resources/retro_cost_{network}_s{simpl}_{clusters}.csv",
+        floor_area = "resources/floor_area_{network}_s{simpl}_{clusters}.csv"
+    output: config['results_dir']  +  config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc'
     threads: 1
     resources: mem_mb=2000
-    benchmark: "benchmarks/prepare_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}"
+    benchmark: config['results_dir'] + config['run'] + "/benchmarks/prepare_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}"
     script: "scripts/prepare_sector_network.py"
 
 
 
 rule plot_network:
     input:
-        network=config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc"
+        network=config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc"
     output:
-        map=config['results_dir'] + config['run'] + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{co2_budget_name}_{planning_horizons}.pdf",
-        today=config['results_dir'] + config['run'] + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}-today.pdf"
+        map=config['results_dir'] + config['run'] + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
+        today=config['results_dir'] + config['run'] + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}-today.pdf"
     threads: 2
     resources: mem_mb=10000
     script: "scripts/plot_network.py"
@@ -240,11 +364,11 @@ rule copy_config:
 
 rule make_summary:
     input:
-        networks=expand(config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc",
+        networks=expand(config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
                  **config['scenario']),
         costs=config['costs_dir'] + "costs_{}.csv".format(config['scenario']['planning_horizons'][0]),
-        #plots=expand(config['results_dir'] + config['run'] + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{co2_budget_name}_{planning_horizons}.pdf",
-        #       **config['scenario'])
+        plots=expand(config['results_dir'] + config['run'] + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
+              **config['scenario'])
         #heat_demand_name='data/heating/daily_heat_demand.h5'
     output:
         nodal_costs=config['summary_dir'] + '/' + config['run'] + '/csvs/nodal_costs.csv',
@@ -276,7 +400,7 @@ rule plot_summary:
     output:
         costs=config['summary_dir'] + '/' + config['run'] + '/graphs/costs.pdf',
         energy=config['summary_dir'] + '/' + config['run'] + '/graphs/energy.pdf',
-        #balances=config['summary_dir'] + '/' + config['run'] + '/graphs/balances-energy.pdf'
+        balances=config['summary_dir'] + '/' + config['run'] + '/graphs/balances-energy.pdf'
     threads: 2
     resources: mem_mb=10000
     script:
@@ -286,16 +410,16 @@ if config["foresight"] == "overnight":
 
     rule solve_network:
         input:
-            network=config['results_dir'] + config['run'] + "/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc",
+            network=config['results_dir'] + config['run'] + "/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
             costs=config['costs_dir'] + "costs_{planning_horizons}.csv",
             config=config['summary_dir'] + '/' + config['run'] + '/configs/config.yaml'
-        output: config['results_dir'] + config['run'] + "/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc"
+        output: config['results_dir'] + config['run'] + "/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc"
         shadow: "shallow"
         log:
-            solver="logs/" + config['run'] + "/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_solver.log",
-            python="logs/" + config['run'] + "/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python.log",
-            memory="logs/" + config['run'] + "/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_memory.log"
-        benchmark: "benchmarks/solve_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}"
+            solver=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
+            python=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_python.log",
+            memory=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_memory.log"
+        benchmark: config['results_dir'] + config['run'] + "/benchmarks/solve_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}"
         threads: 4
         resources: mem_mb=config['solving']['mem']
         # group: "solve" # with group, threads is ignored https://bitbucket.org/snakemake/snakemake/issues/971/group-job-description-does-not-contain
@@ -306,14 +430,15 @@ if config["foresight"] == "myopic":
 
     rule add_existing_baseyear:
         input:
-            network=config['results_dir']  +  config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc',
+            network=config['results_dir']  +  config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc',
             powerplants=pypsaeur('resources/powerplants.csv'),
-            clustermaps=pypsaeur('resources/clustermaps_{network}_s{simpl}_{clusters}.h5'),
+            busmap_s=pypsaeur("resources/busmap_{network}_s{simpl}.csv"),
+            busmap=pypsaeur("resources/busmap_{network}_s{simpl}_{clusters}.csv"),
             clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv",
             costs=config['costs_dir'] + "costs_{}.csv".format(config['scenario']['planning_horizons'][0]),
             cop_soil_total="resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc",
             cop_air_total="resources/cop_air_total_{network}_s{simpl}_{clusters}.nc"
-        output: config['results_dir']  +  config['run'] + '/prenetworks-brownfield/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc'
+        output: config['results_dir']  +  config['run'] + '/prenetworks-brownfield/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc'
         wildcard_constraints:
             planning_horizons=config['scenario']['planning_horizons'][0] #only applies to baseyear
         threads: 1
@@ -322,36 +447,36 @@ if config["foresight"] == "myopic":
 
     def process_input(wildcards):
         i = config["scenario"]["planning_horizons"].index(int(wildcards.planning_horizons))
-        return config['results_dir'] + config['run'] + "/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_" + str(config["scenario"]["planning_horizons"][i-1]) + ".nc"
+        return config['results_dir'] + config['run'] + "/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_" + str(config["scenario"]["planning_horizons"][i-1]) + ".nc"
 
 
     rule add_brownfield:
         input:
-            network=config['results_dir']  +  config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc',
+            network=config['results_dir']  +  config['run'] + '/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc',
             network_p=process_input, #solved network at previous time step
             costs=config['costs_dir'] + "costs_{planning_horizons}.csv",
             cop_soil_total="resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc",
             cop_air_total="resources/cop_air_total_{network}_s{simpl}_{clusters}.nc"
 
-        output: config['results_dir'] + config['run'] + "/prenetworks-brownfield/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc"
+        output: config['results_dir'] + config['run'] + "/prenetworks-brownfield/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc"
         threads: 4
-        resources: mem_mb=2000
+        resources: mem_mb=10000
         script: "scripts/add_brownfield.py"
 
     ruleorder: add_existing_baseyear > add_brownfield
 
     rule solve_network_myopic:
         input:
-            network=config['results_dir'] + config['run'] + "/prenetworks-brownfield/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc",
+            network=config['results_dir'] + config['run'] + "/prenetworks-brownfield/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
             costs=config['costs_dir'] + "costs_{planning_horizons}.csv",
             config=config['summary_dir'] + '/' + config['run'] + '/configs/config.yaml'
-        output: config['results_dir'] + config['run'] + "/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}.nc"
+        output: config['results_dir'] + config['run'] + "/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc"
         shadow: "shallow"
         log:
-            solver="logs/" + config['run'] + "/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_solver.log",
-            python="logs/" + config['run'] + "/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python.log",
-            memory="logs/" + config['run'] + "/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}_memory.log"
-        benchmark: "benchmarks/solve_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{co2_budget_name}_{planning_horizons}"
+            solver=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
+            python=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_python.log",
+            memory=config['results_dir'] + config['run'] + "/logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_memory.log"
+        benchmark: config['results_dir'] + config['run'] + "/benchmarks/solve_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}"
         threads: 4
         resources: mem_mb=config['solving']['mem']
         script: "scripts/solve_network.py"

config.default.yaml

@@ -1,11 +1,13 @@
+version: 0.4.0
+
 logging_level: INFO
 
 results_dir: 'results/'
 summary_dir: results
 costs_dir: '../technology-data/outputs/'
 run: 'your-run-name'  # use this to keep track of runs with different settings
-foresight: 'overnight' #options are overnight, myopic, perfect (perfect is not yet implemented)
-
+foresight: 'overnight' # options are overnight, myopic, perfect (perfect is not yet implemented)
+# if you use myopic or perfect foresight, set the investment years in "planning_horizons" below
 
 scenario:
   sectors: [E]  # ignore this legacy setting
@@ -22,8 +24,19 @@ scenario:
   # B for biomass supply, I for industry, shipping and aviation
   # solarx or onwindx changes the available installable potential by factor x
   # dist{n} includes distribution grids with investment cost of n times cost in data/costs.csv
-  planning_horizons : [2030] #investment years for myopic and perfect; or costs year for overnight
-  co2_budget_name: ['go'] #gives shape of CO2 budgets over planning horizon
+  planning_horizons : [2030] # investment years for myopic and perfect; or costs year for overnight
+  # for example, set to [2020, 2030, 2040, 2050] for myopic foresight
+
+# CO2 budget as a fraction of 1990 emissions
+# this is over-ridden if CO2Lx is set in sector_opts
+co2_budget:
+  2020: 0.7011648746
+  2025: 0.5241935484
+  2030: 0.2970430108
+  2035: 0.1500896057
+  2040: 0.0712365591
+  2045: 0.0322580645
+  2050: 0
 
 # snapshots are originally set in PyPSA-Eur/config.yaml but used again by PyPSA-Eur-Sec
 snapshots:
@@ -43,9 +56,22 @@ electricity:
     battery: 6
     H2: 168
 
+# regulate what components with which carriers are kept from PyPSA-Eur;
+# some technologies are removed because they are implemented differently
+# or have different year-dependent costs in PyPSA-Eur-Sec
+pypsa_eur:
+  "Bus": ["AC"]
+  "Link": ["DC"]
+  "Generator": ["onwind", "offwind-ac", "offwind-dc", "solar", "ror"]
+  "StorageUnit": ["PHS","hydro"]
+
 biomass:
   year: 2030
   scenario: "Med"
+  classes:
+    solid biomass: ['Primary agricultural residues', 'Forestry energy residue', 'Secondary forestry residues', 'Secondary Forestry residues – sawdust', 'Forestry residues from landscape care biomass', 'Municipal waste']
+    not included: ['Bioethanol sugar beet biomass', 'Rapeseeds for biodiesel', 'sunflower and soya for Biodiesel', 'Starchy crops biomass', 'Grassy crops biomass', 'Willow biomass', 'Poplar biomass potential', 'Roundwood fuelwood', 'Roundwood Chips & Pellets']
+    biogas: ['Manure biomass potential', 'Sludge biomass']
 
 # only relevant for foresight = myopic or perfect
 existing_capacities:
@@ -56,8 +82,8 @@ existing_capacities:
 sector:
   'central' : True
   'central_fraction' : 0.6
-  'dsm_restriction_value' : 0.75  #Set to 0 for no restriction on BEV DSM
-  'dsm_restriction_time' : 7  #Time at which SOC of BEV has to be dsm_restriction_value
+  'bev_dsm_restriction_value' : 0.75  #Set to 0 for no restriction on BEV DSM
+  'bev_dsm_restriction_time' : 7  #Time at which SOC of BEV has to be dsm_restriction_value
   'transport_heating_deadband_upper' : 20.
   'transport_heating_deadband_lower' : 15.
   'ICE_lower_degree_factor' : 0.375    #in per cent increase in fuel consumption per degree above deadband
@@ -65,22 +91,46 @@ sector:
   'EV_lower_degree_factor' : 0.98
   'EV_upper_degree_factor' : 0.63
   'district_heating_loss' : 0.15
-  'bev' : True #turns on EV battery
+  'bev_dsm' : True #turns on EV battery
   'bev_availability' : 0.5  #How many cars do smart charging
   'v2g' : True #allows feed-in to grid from EV battery
-  'transport_fuel_cell_share' : 0.   #0 means all EVs, 1 means all FCs
+  #what is not EV or FCEV is fossil-fuelled
+  'land_transport_fuel_cell_share': # 1 means all FCEVs
+    2020: 0
+    2030: 0.05
+    2040: 0.1
+    2050: 0.15
+  'land_transport_electric_share': # 1 means all EVs
+    2020: 0
+    2030: 0.25
+    2040: 0.6
+    2050: 0.85
+  'transport_fuel_cell_efficiency': 0.5
+  'transport_internal_combustion_efficiency': 0.3
   'shipping_average_efficiency' : 0.4 #For conversion of fuel oil to propulsion in 2011
-  'time_dep_hp_cop' : True
-  'space_heating_fraction' : 1.0 #fraction of space heating active
-  'retrofitting' : False
-  'retroI-fraction' : 0.25
-  'retroII-fraction' : 0.55
-  'retrofitting-cost_factor' : 1.0
+  'time_dep_hp_cop' : True #time dependent heat pump coefficient of performance
+  'heat_pump_sink_T' : 55. # Celsius, based on DTU / large area radiators; used in build_cop_profiles.py
+   # conservatively high to cover hot water and space heating in poorly-insulated buildings
+  'retrofitting' :
+    'retro_exogen': True # space heat demand savings exogenously
+    'dE': # reduction of space heat demand (applied before losses in DH)
+      2020 : 0.
+      2030 : 0.15
+      2040 : 0.3
+      2050 : 0.4
+    'retro_endogen': False  # co-optimise space heat savings
+    'cost_factor' : 1.0
+    'interest_rate': 0.04  # for investment in building components
+    'annualise_cost': True  # annualise the investment costs
+    'tax_weighting': False   # weight costs depending on taxes in countries
+    'construction_index': True   # weight costs depending on labour/material costs per ct
+    'l_strength': ["0.076", "0.197"] # additional insulation thickness[m], determines number of retro steps(=generators per bus) and maximum possible savings
   'tes' : True
   'tes_tau' : 3.
   'boilers' : True
   'oil_boilers': False
   'chp' : True
+  'micro_chp' : False
   'solar_thermal' : True
   'solar_cf_correction': 0.788457  # =  >>> 1/1.2683
   'marginal_cost_storage' : 0. #1e-4
@@ -89,16 +139,19 @@ sector:
   'dac' : True
   'co2_vent' : True
   'SMR' : True
-  'ccs_fraction' : 0.9
+  'co2_sequestration_potential' : 200  #MtCO2/a sequestration potential for Europe
+  'co2_sequestration_cost' : 20   #EUR/tCO2 for transport and sequestration of CO2
+  'cc_fraction' : 0.9  # default fraction of CO2 captured with post-combustion capture
   'hydrogen_underground_storage' : True
   'use_fischer_tropsch_waste_heat' : True
   'use_fuel_cell_waste_heat' : True
   'electricity_distribution_grid' : False
   'electricity_distribution_grid_cost_factor' : 1.0  #multiplies cost in data/costs.csv
   'electricity_grid_connection' : True  # only applies to onshore wind and utility PV
+  'gas_distribution_grid' : True
+  'gas_distribution_grid_cost_factor' : 1.0  #multiplies cost in data/costs.csv
 
 costs:
-  year: 2030
   lifetime: 25 #default lifetime
   # From a Lion Hirth paper, also reflects average of Noothout et al 2016
   discountrate: 0.07
@@ -156,12 +209,17 @@ solving:
   mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
 
 industry:
-  'DRI_ratio' : 0.5 #ratio of today's blast-furnace steel (60% primary route, 40% secondary) to future assumption (30% primary, 70% secondary), transformed into DRI + electric arc
-  'H2_DRI' : 1.7   #H2 consumption in Direct Reduced Iron (DRI),  MWh_H2/ton_Steel from Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279
-  'Al_to_scrap' : 0.5 # ratio of primary-route Aluminum transformed into scrap (today 40% to future 20% primary route)
-  'H2_for_NH3' : 85000 # H2 in GWh/a for 17 MtNH3/a transformed from SMR to electrolyzed-H2, following Lechtenböhmer(2016)
+  'St_primary_fraction' : 0.3 # fraction of steel produced via primary route (DRI + EAF) versus secondary route (EAF); today fraction is 0.6
+  'H2_DRI' : 1.7   #H2 consumption in Direct Reduced Iron (DRI),  MWh_H2,LHV/ton_Steel from 51kgH2/tSt in Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279
+  'elec_DRI' : 0.322   #electricity consumption in Direct Reduced Iron (DRI) shaft, MWh/tSt HYBRIT brochure https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf
+  'Al_primary_fraction' : 0.2 # fraction of aluminium produced via the primary route versus scrap; today fraction is 0.4
+  'MWh_CH4_per_tNH3_SMR' : 10.8 # 2012's demand from https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf
+  'MWh_elec_per_tNH3_SMR' : 0.7 # same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3
+  'MWh_H2_per_tNH3_electrolysis' : 6.5 # from https://doi.org/10.1016/j.joule.2018.04.017, around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy)
+  'MWh_elec_per_tNH3_electrolysis' : 1.17 # from https://doi.org/10.1016/j.joule.2018.04.017 Table 13 (air separation and HB)
   'NH3_process_emissions' : 24.5 # in MtCO2/a from SMR for H2 production for NH3 from UNFCCC for 2015 for EU28
   'petrochemical_process_emissions' : 25.5 # in MtCO2/a for petrochemical and other from UNFCCC for 2015 for EU28
+  'HVC_primary_fraction' : 1.0 #fraction of current non-ammonia basic chemicals produced via primary route
 
 plotting:
   map:
@@ -258,7 +316,7 @@ plotting:
     "DAC" : "#E74C3C"
     "co2 stored" : "#123456"
     "CO2 sequestration" : "#123456"
-    "CCS" : "k"
+    "CC" : "k"
     "co2" : "#123456"
     "co2 vent" : "#654321"
     "solid biomass for industry co2 from atmosphere" : "#654321"
@@ -268,6 +326,7 @@ plotting:
     "Fischer-Tropsch" : "#44DD33"
     "kerosene for aviation": "#44BB11"
     "naphtha for industry" : "#44FF55"
+    "land transport fossil" : "#44DD33"
     "water tanks" : "#BBBBBB"
     "hot water storage" : "#BBBBBB"
     "hot water charging" : "#BBBBBB"
@@ -279,7 +338,6 @@ plotting:
     "Ambient" : "k"
     "Electric load" : "b"
     "Heat load" : "r"
-    "Transport load" : "grey"
     "heat" : "darkred"
     "rural heat" : "#880000"
     "central heat" : "#b22222"
@@ -294,15 +352,16 @@ plotting:
     "building retrofitting" : "purple"
     "BEV charger" : "grey"
     "V2G" : "grey"
-    "transport" : "grey"
+    "land transport EV" : "grey"
     "electricity" : "k"
     "gas for industry" : "#333333"
     "solid biomass for industry" : "#555555"
+    "industry electricity" : "#222222"
     "industry new electricity" : "#222222"
     "process emissions to stored" : "#444444"
     "process emissions to atmosphere" : "#888888"
     "process emissions" : "#222222"
-    "transport fuel cell" : "#AAAAAA"
+    "land transport fuel cell" : "#AAAAAA"
     "biogas" : "#800000"
     "solid biomass" : "#DAA520"
     "today" : "#D2691E"

config.myopic.yaml

@@ -1,328 +0,0 @@
-logging_level: INFO
-
-results_dir: 'results/'
-summary_dir: results
-costs_dir: '../technology-data/outputs/'
-run: 'your-run-name'  # use this to keep track of runs with different settings
-foresight: 'myopic' #options are overnight, myopic, perfect (perfect is not yet implemented)
-
-
-scenario:
-  sectors: [E]  # ignore this legacy setting
-  simpl: ['']  # only relevant for PyPSA-Eur
-  lv: [1.0,1.5]    # allowed transmission line volume expansion, can be any float >= 1.0 (today) or "opt"
-  clusters: [45,50] # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred
-  opts: ['']  # only relevant for PyPSA-Eur
-  sector_opts: [Co2L0-3H-H-B-solar3-dist1]  # this is where the main scenario settings are
-  # to really understand the options here, look in scripts/prepare_sector_network.py
-  # Co2Lx specifies the CO2 target in x% of the 1990 values; default will give default (5%);
-  # Co2L0p25 will give 25% CO2 emissions; Co2Lm0p05 will give 5% negative emissions
-  # xH is the temporal resolution; 3H is 3-hourly, i.e. one snapshot every 3 hours
-  # single letters are sectors: T for land transport, H for building heating,
-  # B for biomass supply, I for industry, shipping and aviation
-  # solarx or onwindx changes the available installable potential by factor x
-  # dist{n} includes distribution grids with investment cost of n times cost in data/costs.csv
-  planning_horizons : [2020, 2030, 2040, 2050] #investment years for myopic and perfect; or costs year for overnight
-  co2_budget_name: ['go'] #gives shape of CO2 budgets over planning horizon
-
-# snapshots are originally set in PyPSA-Eur/config.yaml but used again by PyPSA-Eur-Sec
-snapshots:
-  # arguments to pd.date_range
-  start: "2013-01-01"
-  end: "2014-01-01"
-  closed: 'left' # end is not inclusive
-
-atlite:
-  cutout_dir: '../pypsa-eur/cutouts'
-  cutout_name: "europe-2013-era5"
-
-# this information is NOT used but needed as an argument for
-# pypsa-eur/scripts/add_electricity.py/load_costs in make_summary.py
-electricity:
-  max_hours:
-    battery: 6
-    H2: 168
-
-biomass:
-  year: 2030
-  scenario: "Med"
-
-# only relevant for foresight = myopic or perfect
-existing_capacities:
-  grouping_years: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019]
-  threshold_capacity: 10
-  conventional_carriers: ['lignite', 'coal', 'oil', 'uranium']
-
-sector:
-  'central' : True
-  'central_fraction' : 0.6
-  'dsm_restriction_value' : 0.75  #Set to 0 for no restriction on BEV DSM
-  'dsm_restriction_time' : 7  #Time at which SOC of BEV has to be dsm_restriction_value
-  'transport_heating_deadband_upper' : 20.
-  'transport_heating_deadband_lower' : 15.
-  'ICE_lower_degree_factor' : 0.375    #in per cent increase in fuel consumption per degree above deadband
-  'ICE_upper_degree_factor' : 1.6
-  'EV_lower_degree_factor' : 0.98
-  'EV_upper_degree_factor' : 0.63
-  'district_heating_loss' : 0.15
-  'bev' : True #turns on EV battery
-  'bev_availability' : 0.5  #How many cars do smart charging
-  'v2g' : True #allows feed-in to grid from EV battery
-  'transport_fuel_cell_share' : 0.   #0 means all EVs, 1 means all FCs
-  'shipping_average_efficiency' : 0.4 #For conversion of fuel oil to propulsion in 2011
-  'time_dep_hp_cop' : True
-  'space_heating_fraction' : 1.0 #fraction of space heating active
-  'retrofitting' : False
-  'retroI-fraction' : 0.25
-  'retroII-fraction' : 0.55
-  'retrofitting-cost_factor' : 1.0
-  'tes' : True
-  'tes_tau' : 3.
-  'boilers' : True
-  'oil_boilers': False
-  'chp' : True
-  'solar_thermal' : True
-  'solar_cf_correction': 0.788457  # =  >>> 1/1.2683
-  'marginal_cost_storage' : 0. #1e-4
-  'methanation' : True
-  'helmeth' : True
-  'dac' : True
-  'co2_vent' : True
-  'SMR' : True
-  'ccs_fraction' : 0.9
-  'hydrogen_underground_storage' : True
-  'use_fischer_tropsch_waste_heat' : True
-  'use_fuel_cell_waste_heat' : True
-  'electricity_distribution_grid' : False
-  'electricity_distribution_grid_cost_factor' : 1.0  #multiplies cost in data/costs.csv
-  'electricity_grid_connection' : True  # only applies to onshore wind and utility PV
-
-costs:
-  year: 2030
-  lifetime: 25 #default lifetime
-  # From a Lion Hirth paper, also reflects average of Noothout et al 2016
-  discountrate: 0.07
-  # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html # noqa: E501
-  USD2013_to_EUR2013: 0.7532
-
-  # Marginal and capital costs can be overwritten
-  # capital_cost:
-  #   Wind: Bla
-  marginal_cost: #
-    solar: 0.01
-    onwind: 0.015
-    offwind: 0.015
-    hydro: 0.
-    H2: 0.
-    battery: 0.
-
-  emission_prices: # only used with the option Ep (emission prices)
-    co2: 0.
-
-  lines:
-    length_factor: 1.25 #to estimate offwind connection costs
-
-
-solving:
-  #tmpdir: "path/to/tmp"
-  options:
-    formulation: kirchhoff
-    clip_p_max_pu: 1.e-2
-    load_shedding: false
-    noisy_costs: true
-
-    min_iterations: 1
-    max_iterations: 1
-    # nhours: 1
-
-  solver:
-    name: gurobi
-    threads: 4
-    method: 2 # barrier
-    crossover: 0
-    BarConvTol: 1.e-5
-    Seed: 123
-    AggFill: 0
-    PreDual: 0
-    GURO_PAR_BARDENSETHRESH: 200
-    #FeasibilityTol: 1.e-6
-
-    #name: cplex
-    #threads: 4
-    #lpmethod: 4 # barrier
-    #solutiontype: 2 # non basic solution, ie no crossover
-    #barrier_convergetol: 1.e-5
-    #feasopt_tolerance: 1.e-6
-  mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
-
-industry:
-  'DRI_ratio' : 0.5 #ratio of today's blast-furnace steel (60% primary route, 40% secondary) to future assumption (30% primary, 70% secondary), transformed into DRI + electric arc
-  'H2_DRI' : 1.7   #H2 consumption in Direct Reduced Iron (DRI),  MWh_H2/ton_Steel from Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279
-  'Al_to_scrap' : 0.5 # ratio of primary-route Aluminum transformed into scrap (today 40% to future 20% primary route)
-  'H2_for_NH3' : 85000 # H2 in GWh/a for 17 MtNH3/a transformed from SMR to electrolyzed-H2, following Lechtenböhmer(2016)
-  'NH3_process_emissions' : 24.5 # in MtCO2/a from SMR for H2 production for NH3 from UNFCCC for 2015 for EU28
-  'petrochemical_process_emissions' : 25.5 # in MtCO2/a for petrochemical and other from UNFCCC for 2015 for EU28
-
-plotting:
-  map:
-    figsize: [7, 7]
-    boundaries: [-10.2, 29, 35,  72]
-    p_nom:
-      bus_size_factor: 5.e+4
-      linewidth_factor: 3.e+3 # 1.e+3  #3.e+3
-
-  costs_max: 1200
-  costs_threshold: 1
-
-
-  energy_max: 20000.
-  energy_min: -15000.
-  energy_threshold: 50.
-
-
-  vre_techs: ["onwind", "offwind-ac", "offwind-dc", "solar", "ror"]
-  renewable_storage_techs: ["PHS","hydro"]
-  conv_techs: ["OCGT", "CCGT", "Nuclear", "Coal"]
-  storage_techs: ["hydro+PHS", "battery", "H2"]
-  # store_techs: ["Li ion", "water tanks"]
-  load_carriers: ["AC load"] #, "heat load", "Li ion load"]
-  AC_carriers: ["AC line", "AC transformer"]
-  link_carriers: ["DC line", "Converter AC-DC"]
-  heat_links: ["heat pump", "resistive heater", "CHP heat", "CHP electric",
-               "gas boiler", "central heat pump", "central resistive heater", "central CHP heat",
-               "central CHP electric", "central gas boiler"]
-  heat_generators: ["gas boiler", "central gas boiler", "solar thermal collector", "central solar thermal collector"]
-  tech_colors:
-    "onwind" : "b"
-    "onshore wind" : "b"
-    'offwind' : "c"
-    'offshore wind' : "c"
-    'offwind-ac' : "c"
-    'offshore wind (AC)' : "c"
-    'offwind-dc' : "#009999"
-    'offshore wind (DC)' : "#009999"
-    'wave' : "#004444"
-    "hydro" : "#3B5323"
-    "hydro reservoir" : "#3B5323"
-    "ror" : "#78AB46"
-    "run of river" : "#78AB46"
-    'hydroelectricity' : '#006400'
-    'solar' : "y"
-    'solar PV' : "y"
-    'solar thermal' : 'coral'
-    'solar rooftop' : '#e6b800'
-    "OCGT" : "wheat"
-    "OCGT marginal" : "sandybrown"
-    "OCGT-heat" : "orange"
-    "gas boiler" : "orange"
-    "gas boilers" : "orange"
-    "gas boiler marginal" : "orange"
-    "gas-to-power/heat" : "orange"
-    "gas" : "brown"
-    "natural gas" : "brown"
-    "SMR" : "#4F4F2F"
-    "oil" : "#B5A642"
-    "oil boiler" : "#B5A677"
-    "lines" : "k"
-    "transmission lines" : "k"
-    "H2" : "m"
-    "hydrogen storage" : "m"
-    "battery" : "slategray"
-    "battery storage" : "slategray"
-    "home battery" : "#614700"
-    "home battery storage" : "#614700"
-    "Nuclear" : "r"
-    "Nuclear marginal" : "r"
-    "nuclear" : "r"
-    "uranium" : "r"
-    "Coal" : "k"
-    "coal" : "k"
-    "Coal marginal" : "k"
-    "Lignite" : "grey"
-    "lignite" : "grey"
-    "Lignite marginal" : "grey"
-    "CCGT" : "orange"
-    "CCGT marginal" : "orange"
-    "heat pumps" : "#76EE00"
-    "heat pump" : "#76EE00"
-    "air heat pump" : "#76EE00"
-    "ground heat pump" : "#40AA00"
-    "power-to-heat" : "#40AA00"
-    "resistive heater" : "pink"
-    "Sabatier" : "#FF1493"
-    "methanation" : "#FF1493"
-    "power-to-gas" : "#FF1493"
-    "power-to-liquid" : "#FFAAE9"
-    "helmeth" : "#7D0552"
-    "helmeth" : "#7D0552"
-    "DAC" : "#E74C3C"
-    "co2 stored" : "#123456"
-    "CO2 sequestration" : "#123456"
-    "CCS" : "k"
-    "co2" : "#123456"
-    "co2 vent" : "#654321"
-    "solid biomass for industry co2 from atmosphere" : "#654321"
-    "solid biomass for industry co2 to stored": "#654321"
-    "gas for industry co2 to atmosphere": "#654321"
-    "gas for industry co2 to stored": "#654321"
-    "Fischer-Tropsch" : "#44DD33"
-    "kerosene for aviation": "#44BB11"
-    "naphtha for industry" : "#44FF55"
-    "water tanks" : "#BBBBBB"
-    "hot water storage" : "#BBBBBB"
-    "hot water charging" : "#BBBBBB"
-    "hot water discharging" : "#999999"
-    "CHP" : "r"
-    "CHP heat" : "r"
-    "CHP electric" : "r"
-    "PHS" : "g"
-    "Ambient" : "k"
-    "Electric load" : "b"
-    "Heat load" : "r"
-    "Transport load" : "grey"
-    "heat" : "darkred"
-    "rural heat" : "#880000"
-    "central heat" : "#b22222"
-    "decentral heat" : "#800000"
-    "low-temperature heat for industry" : "#991111"
-    "process heat" : "#FF3333"
-    "heat demand" : "darkred"
-    "electric demand" : "k"
-    "Li ion" : "grey"
-    "district heating" : "#CC4E5C"
-    "retrofitting" : "purple"
-    "building retrofitting" : "purple"
-    "BEV charger" : "grey"
-    "V2G" : "grey"
-    "transport" : "grey"
-    "electricity" : "k"
-    "gas for industry" : "#333333"
-    "solid biomass for industry" : "#555555"
-    "industry new electricity" : "#222222"
-    "process emissions to stored" : "#444444"
-    "process emissions to atmosphere" : "#888888"
-    "process emissions" : "#222222"
-    "transport fuel cell" : "#AAAAAA"
-    "biogas" : "#800000"
-    "solid biomass" : "#DAA520"
-    "today" : "#D2691E"
-    "shipping" : "#6495ED"
-    "electricity distribution grid" : "#333333"
-  nice_names:
-    # OCGT: "Gas"
-    # OCGT marginal: "Gas (marginal)"
-    offwind: "offshore wind"
-    onwind: "onshore wind"
-    battery: "Battery storage"
-    lines: "Transmission lines"
-    AC line: "AC lines"
-    AC-AC: "DC lines"
-    ror: "Run of river"
-  nice_names_n:
-    offwind: "offshore\nwind"
-    onwind: "onshore\nwind"
-    # OCGT: "Gas"
-    H2: "Hydrogen\nstorage"
-    # OCGT marginal: "Gas (marginal)"
-    lines: "transmission\nlines"
-    ror: "run of river"

data/co2_budget.csv

@@ -1,8 +0,0 @@
-,go,wait
-2020,0.7011648746,0.7011648746
-2025,0.5241935484,0.6285842294
-2030,0.2970430108,0.3503584229
-2035,0.1500896057,0.0725806452
-2040,0.0712365591,0
-2045,0.0322580645,0
-2050,0,0

data/hydrogen_salt_cavern_potentials.csv

@@ -0,0 +1,31 @@
+ct,TWh
+AT,
+BA,
+BE,
+BG,
+CH,
+CZ,
+DE,4500
+DK,700
+EE,
+ES,350
+FI,
+FR,
+GB,1050
+GR,120
+HR,
+HU,
+IE,
+IT,
+LT,
+LU,
+LV,
+NL,150
+NO,
+PL,120
+PT,400
+RO,
+RS,
+SE,
+SI,
+SK,

data/retro/average_surface_components.csv

@@ -0,0 +1,7 @@
+,Dwelling,Ceilling,Standard component surfaces (m2),component,surfaces,(m2),,
+Building type,Space(m²),Height(m),Roof,Facade,Floor,Windows,,
+Single/two family house,120,2.5,90,166,63,29,,
+Large apartment house,1457,2.5,354,1189,354,380,,
+Apartment house,5276,,598.337,2992.1,598.337,756,tabula ,http://webtool.building-typology.eu/#pdfes
+,,,,,,,,
+"Source: https://link.springer.com/article/10.1007/s12053-010-9090-6  ,p.4",,,,,,,,

data/retro/comparative_level_investment.csv

@@ -0,0 +1,49 @@
+NA_ITEM,Price level indices (EU28=100),,,,,,,,,
+PPP_CAT,Actual individual consumption,,,,,,,,,
+,,,,,,,,,,
+GEO/TIME,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018
+European Union - 28 countries,100.0,100.0,100.0,100.0,100.0,100.0,100.0,100.0,100.0,100.0
+Belgium,113.6,111.9,112.4,111.5,111.0,108.9,106.3,110.3,112.3,112.5
+Bulgaria,47.1,45.7,45.5,45.0,44.2,42.6,42.2,43.2,45.1,46.3
+Czech Republic,64.5,66.6,68.9,66.9,63.3,58.3,58.4,60.5,62.4,65.0
+Denmark,141.7,140.0,139.9,140.0,139.3,138.5,135.0,140.0,138.9,138.1
+Germany,104.6,103.1,102.2,101.1,102.5,101.5,100.4,102.6,103.7,104.1
+Estonia,67.5,66.0,67.2,67.6,69.9,69.9,68.9,71.0,73.9,76.3
+Ireland,129.9,122.7,122.5,120.5,123.2,124.9,122.2,126.5,129.1,129.2
+Greece,93.6,95.4,94.9,91.9,87.8,83.8,81.0,82.3,83.0,81.8
+Spain,97.5,98.7,98.5,95.8,95.1,92.7,90.0,92.7,93.7,93.7
+France,111.2,109.9,109.6,108.7,107.0,106.0,104.0,105.8,107.1,107.4
+Croatia,70.2,70.1,68.1,65.5,64.5,62.5,60.7,61.3,63.0,64.0
+Italy,103.6,100.4,101.5,101.1,102.3,102.6,100.3,101.1,101.6,101.4
+Cyprus,92.0,94.6,95.8,96.0,95.2,92.0,88.5,89.8,91.2,90.6
+Latvia,68.1,62.3,65.5,65.9,66.0,66.0,64.2,66.9,68.3,69.5
+Lithuania,60.3,57.8,58.3,58.0,57.8,56.9,55.9,58.3,60.0,61.4
+Luxembourg,130.0,136.5,136.0,135.8,135.1,135.7,132.1,137.0,139.9,141.6
+Hungary,58.2,57.4,56.4,54.9,54.4,53.4,53.3,56.2,59.4,59.0
+Malta,75.8,76.6,78.0,78.0,80.8,80.5,79.8,81.4,81.9,83.4
+Netherlands,108.5,112.3,112.7,111.3,111.9,111.9,109.6,113.8,114.6,114.8
+Austria,109.9,109.2,110.1,108.9,109.1,109.1,107.2,110.2,112.8,113.7
+Poland,53.1,55.2,53.7,52.1,52.4,52.5,51.1,50.9,53.5,54.3
+Portugal,85.2,85.0,85.3,82.7,81.1,80.4,78.7,81.6,83.5,84.6
+Romania,49.1,46.9,47.7,45.6,47.8,47.6,47.2,46.8,48.0,48.6
+Slovenia,85.3,84.3,83.7,81.8,82.1,81.5,79.8,82.3,82.7,83.8
+Slovakia,66.6,62.5,63.4,63.4,63.4,63.3,62.3,63.6,65.4,66.1
+Finland,121.0,120.3,121.6,121.8,124.0,122.9,119.6,122.8,123.3,123.4
+Sweden,109.5,124.6,131.7,134.3,140.5,133.6,128.8,135.3,134.5,126.9
+United Kingdom,107.5,111.4,111.3,118.6,117.0,123.6,134.7,123.5,117.6,117.7
+Iceland,94.9,107.6,109.6,111.6,116.0,123.4,132.5,154.5,172.3,163.7
+Norway,142.4,158.8,165.3,172.5,166.9,157.2,152.2,155.0,157.3,155.4
+Switzerland,131.6,146.4,161.7,160.6,155.1,153.0,167.0,169.8,167.1,159.1
+Candidate and potential candidate countries except Turkey and Kosovo (under United Nations Security Council Resolution 1244/99),48.0,45.6,47.1,44.8,46.4,45.2,43.4,44.4,46.0,47.5
+Montenegro,52.3,49.5,49.3,50.1,50.5,49.3,48.0,48.7,50.5,51.1
+North Macedonia,41.4,41.3,42.7,42.1,42.5,41.9,40.9,41.7,43.2,43.3
+Albania,46.2,42.8,42.1,40.6,41.9,41.5,39.8,43.0,43.5,46.6
+Serbia,48.3,45.0,48.0,44.5,47.3,45.5,43.1,43.8,46.1,47.9
+Turkey,55.4,61.2,54.7,58.5,57.7,51.6,50.5,50.2,45.4,37.0
+Bosnia and Herzegovina,51.6,50.7,50.6,49.2,49.1,48.4,47.0,47.5,48.2,48.9
+Kosovo (under United Nations Security Council Resolution 1244/99),:,:,:,:,:,:,:,:,:,:
+United States,92.4,98,93.3,101.2,100.3,99,115.9,121.1,120.8,115.2
+Japan,115.1,126.1,127.8,133.8,101.7,94.8,96.5,113,109.4,103.9
+,,,,,,,,,,
+"Source: Eurostat Purchasing power parities (PPPs), price level indices and real expenditures for ESA 2010 aggregates (2019)",,,,,,,,,,
+https://ec.europa.eu/eurostat/statistics-explained/index.php?title=Comparative_price_levels_for_investment,,,,,,,,,,

data/retro/electricity_taxes_eu.csv

@@ -0,0 +1,164 @@
+Electricity prices for household consumers - bi-annual data (from 2007 onwards) [nrg_pc_204],,,,
+,,,,
+Last update,30.10.19,,,
+Extracted on,14.11.19,,,
+Source of data,Eurostat,,,
+,,,,
+PRODUCT,Electrical energy,,,
+CONSOM,Band DC : 2 500 kWh < Consumption < 5 000 kWh,,,
+UNIT,Kilowatt-hour,,,
+TIME,2018S1,,,
+,,,,
+CURRENCY,Euro,Euro,Euro,
+GEO/TAX,Excluding taxes and levies,Excluding VAT and other recoverable taxes and levies,All taxes and levies included,% cost without taxes
+European Union - 28 countries,0.1285,0.1756,0.2052,0.626218323586745
+"Euro area (EA11-2000, EA12-2006, EA13-2007, EA15-2008, EA16-2010, EA17-2013, EA18-2014, EA19)",0.1331,0.1855,0.2188,0.608318098720293
+Belgium,0.1903,0.2279,0.2733,0.696304427369191
+Bulgaria,0.0816,0.0816,0.0979,0.833503575076609
+Czech Republic,0.1286,0.1298,0.1573,0.817546090273363
+Denmark,0.1011,0.2501,0.3126,0.32341650671785
+Germany,0.1379,0.2510,0.2987,0.461667224640107
+Estonia,0.0989,0.1123,0.1348,0.733679525222552
+Ireland,0.1846,0.2087,0.2369,0.779231743351625
+Greece,0.1132,0.1482,0.1672,0.677033492822967
+Spain,0.1873,0.1969,0.2383,0.785984053713806
+France,0.1134,0.1492,0.1748,0.648741418764302
+Croatia,0.1020,0.1160,0.1311,0.778032036613272
+Italy,0.1285,0.1873,0.2067,0.621673923560716
+Cyprus,0.1445,0.1606,0.1893,0.763338615953513
+Latvia,0.1035,0.1266,0.1531,0.676028739386022
+Lithuania,0.0771,0.0906,0.1097,0.702825888787603
+Luxembourg,0.1283,0.1547,0.1671,0.767803710353082
+Hungary,0.0885,0.0885,0.1123,0.78806767586821
+Malta,0.1209,0.1224,0.1285,0.940856031128405
+Netherlands,0.1187,0.1410,0.1706,0.6957796014068
+Austria,0.1232,0.1638,0.1966,0.626653102746694
+Poland,0.0906,0.1146,0.1410,0.642553191489362
+Portugal,0.1007,0.1826,0.2246,0.448352626892253
+Romania,0.0990,0.1120,0.1333,0.742685671417854
+Slovenia,0.1108,0.1322,0.1613,0.686918784872908
+Slovakia,0.0942,0.1305,0.1566,0.601532567049808
+Finland,0.1074,0.1300,0.1612,0.666253101736973
+Sweden,0.1202,0.1513,0.1891,0.635642517186674
+United Kingdom,0.1347,0.1797,0.1887,0.713831478537361
+Iceland,0.1222,0.1246,0.1545,0.790938511326861
+Liechtenstein,:,:,:,#VALUE!
+Norway,0.1254,0.1434,0.1751,0.716162193032553
+Montenegro,0.0828,0.0844,0.1024,0.80859375
+North Macedonia,0.0662,0.0662,0.0781,0.847631241997439
+Albania,:,:,:,#VALUE!
+Serbia,0.0539,0.0587,0.0705,0.764539007092199
+Turkey,0.0727,0.0766,0.0904,0.804203539823009
+Bosnia and Herzegovina,0.0722,0.0738,0.0864,0.835648148148148
+Kosovo (under United Nations Security Council Resolution 1244/99),0.0569,0.0586,0.0633,0.898894154818325
+Moldova,0.1020,0.1020,0.1020,1
+Ukraine,0.0342,0.0342,0.0410,0.834146341463415
+,,,0.157271052631579,
+Special value:,,,,
+:,not available,,,
+,,,,
+PRODUCT,Electrical energy,,,
+CONSOM,Band DC : 2 500 kWh < Consumption < 5 000 kWh,,,
+UNIT,Kilowatt-hour,,,
+TIME,2018S2,,,
+,,,,
+CURRENCY,Euro,Euro,Euro,
+GEO/TAX,Excluding taxes and levies,Excluding VAT and other recoverable taxes and levies,All taxes and levies included,
+European Union - 28 countries,0.1329,0.1810,0.2113,
+"Euro area (EA11-2000, EA12-2006, EA13-2007, EA15-2008, EA16-2010, EA17-2013, EA18-2014, EA19)",0.1376,0.1902,0.2242,
+Belgium,0.1998,0.2429,0.2937,
+Bulgaria,0.0838,0.0838,0.1005,
+Czechia,0.1299,0.1311,0.1586,
+Denmark,0.1116,0.2499,0.3123,
+Germany (until 1990 former territory of the FRG),0.1378,0.2521,0.3000,
+Estonia,0.1048,0.1182,0.1418,
+Ireland,0.2006,0.2237,0.2539,
+Greece,0.1125,0.1458,0.1646,
+Spain,0.1947,0.2047,0.2477,
+France,0.1168,0.1537,0.1799,
+Croatia,0.1028,0.1169,0.1321,
+Italy,0.1416,0.1964,0.2161,
+Cyprus,0.1745,0.1850,0.2183,
+Latvia,0.1041,0.1249,0.1511,
+Lithuania,0.0771,0.0906,0.1097,
+Luxembourg,0.1302,0.1566,0.1691,
+Hungary,0.0880,0.0880,0.1118,
+Malta,0.1229,0.1244,0.1306,
+Netherlands,0.1212,0.1420,0.1707,
+Austria,0.1265,0.1676,0.2012,
+Poland,0.0889,0.1135,0.1396,
+Portugal,0.1028,0.1864,0.2293,
+Romania,0.0964,0.1107,0.1317,
+Slovenia,0.1125,0.1342,0.1638,
+Slovakia,0.0849,0.1218,0.1462,
+Finland,0.1144,0.1369,0.1698,
+Sweden,0.1287,0.1592,0.1990,
+United Kingdom,0.1401,0.1927,0.2024,
+Iceland,0.1152,0.1175,0.1457,
+Liechtenstein,:,:,:,
+Norway,0.1382,0.1562,0.1907,
+Montenegro,0.0829,0.0848,0.1030,
+North Macedonia,0.0667,0.0667,0.0787,
+Albania,0.0759,0.0759,0.0910,
+Serbia,0.0542,0.0591,0.0709,
+Turkey,0.0688,0.0726,0.0857,
+Bosnia and Herzegovina,0.0729,0.0744,0.0871,
+Kosovo (under United Nations Security Council Resolution 1244/99),0.0579,0.0591,0.0638,
+Moldova,0.0960,0.0960,0.1029,
+Ukraine,0.0342,0.0342,0.0410,
+,,,,
+Special value:,,,,
+:,not available,,,
+,,,,
+PRODUCT,Electrical energy,,,
+CONSOM,Band DC : 2 500 kWh < Consumption < 5 000 kWh,,,
+UNIT,Kilowatt-hour,,,
+TIME,2019S1,,,
+,,,,
+CURRENCY,Euro,Euro,Euro,
+GEO/TAX,Excluding taxes and levies,Excluding VAT and other recoverable taxes and levies,All taxes and levies included,
+European Union - 28 countries,0.1351,0.1841,0.2147,
+"Euro area (EA11-2000, EA12-2006, EA13-2007, EA15-2008, EA16-2010, EA17-2013, EA18-2014, EA19)",0.1396,0.1928,0.2270,
+Belgium,0.1965,0.2355,0.2839,
+Bulgaria,0.0831,0.0831,0.0997,
+Czechia,0.1433,0.1444,0.1748,
+Denmark,0.1084,0.2387,0.2984,
+Germany (until 1990 former territory of the FRG),0.1473,0.2595,0.3088,
+Estonia,0.0982,0.1131,0.1357,
+Ireland,0.2027,0.2134,0.2423,
+Greece,0.1139,0.1482,0.1650,
+Spain,0.1889,0.1986,0.2403,
+France,0.1138,0.1508,0.1765,
+Croatia,0.1028,0.1169,0.1321,
+Italy,0.1432,0.2090,0.2301,
+Cyprus,0.1762,0.1867,0.2203,
+Latvia,0.1136,0.1347,0.1629,
+Lithuania,0.0947,0.1037,0.1255,
+Luxembourg,0.1326,0.1666,0.1798,
+Hungary,0.0882,0.0882,0.1120,
+Malta,0.1228,0.1243,0.1305,
+Netherlands,0.1357,0.1708,0.2052,
+Austria,0.1316,0.1695,0.2034,
+Poland,0.0884,0.1092,0.1343,
+Portugal,0.1103,0.1751,0.2154,
+Romania,0.0983,0.1141,0.1358,
+Slovenia,0.1125,0.1339,0.1634,
+Slovakia,0.0962,0.1314,0.1577,
+Finland,0.1173,0.1398,0.1734,
+Sweden,0.1297,0.1612,0.2015,
+United Kingdom,0.1450,0.2021,0.2122,
+Iceland,0.1112,0.1134,0.1406,
+Liechtenstein,:,:,:,
+Norway,0.1360,0.1529,0.1867,
+Montenegro,0.0834,0.0850,0.1032,
+North Macedonia,:,:,:,
+Albania,:,:,:,
+Serbia,0.0541,0.0589,0.0706,
+Turkey,0.0684,0.0718,0.0847,
+Bosnia and Herzegovina,0.0729,0.0746,0.0873,
+Kosovo (under United Nations Security Council Resolution 1244/99),0.0537,0.0556,0.0600,
+Moldova,0.0936,0.0936,0.0936,
+Ukraine,0.0369,0.0369,0.0442,
+,,,,
+Special value:,,,,
+:,not available,,,

data/retro/floor_area_missing.csv

@@ -0,0 +1,17 @@
+country,sector,estimated,value,source,,comments,population [in Million],
+AL,residential,0,64,p.13 1.6 million m² = 2.5% of total floor area,https://www.buildup.eu/sites/default/files/content/sled_albania_residential_building_eng.pdf,,,
+AL,services,0,,,,,,
+BA,residential,0,125.89,Tabula,https://episcope.eu/building-typology/country/ba/,strong differences ? other source claims more than 300 Million m²,,https://www.buildup.eu/sites/default/files/content/sled_serbia_building_eng.pdf
+BA,services,0,,,,,,
+RS,residential,0,72.3,Odyssee(2011),https://odyssee.enerdata.net/database/,,,
+RS,services,0,,,,,,
+MK,residential,0,,"Worldbank p.7 Skopje 75% residential, 25% commercial",http://documents.albankaldawli.org/curated/ar/838951574180734318/pdf/Project-Information-Document-North-Macedonia-Public-Sector-Energy-Efficiency-Project-P149990.pdf,15 % live in illegal constructed buildings ? not part of the statistics,2.1,
+MK,services,0,,,,,,
+ME,residential,0,19.625,p.13 0.314 million m² = 1.6% of total floor area,buildup.eu/sites/default/files/content/sled_montenegro_building_eng.pdf,Only 50 % of the floor area is heated p.12,,buildup.eu/sites/default/files/content/sled_montenegro_building_eng.pdf
+ME,services,0,,,,,,
+CH,residential,0,99.45,Odyssee(2015),,,,
+CH,services,1,78.1392857142857,p.8 44%floor area is services,https://bta.climate-kic.org/wp-content/uploads/2018/04/171123-CK-BTA-DEF-BMB_SWITZERLAND_.pdf,,,
+NO,residential,0,121.55,Odyssee(2015),,,,
+NO,services,0,115.21,Odyssee(2015),,,,
+PL,residential,0,1028.41,EU Building Database,,,,
+PL,services,0,498.84,EU Building Database,,,,

data/retro/retro_cost_germany.csv

@@ -0,0 +1,7 @@
+component,cost_fix,cost_var,life_time,comment,additional source
+wall,70.34,2.36,40,Agora Energiewende p.110,
+floor,39.39,1.3,40,Agora Energiewende p.110,
+roof,75.61,1.3,40,Agora Energiewende p.110,https://www.baulinks.de/webplugin/2018/1524.php4
+window,nan,nan,35,,
+source: p.37 https://www.umweltbundesamt.de/sites/default/files/medien/1410/publikationen/2019-10-29_texte_132-2019_energieaufwand-gebaeudekonzepte.pdf,,,https://www.agora-energiewende.de/en/publications/building-sector-efficiency-a-crucial-component-of-the-energy-transition/,,
+,,,p.115,,

data/retro/u_values_poland.csv

@@ -0,0 +1,9 @@
+component,Before 1945,1945 - 1969,1970 - 1979,1980 - 1989,1990 - 1999,2000 - 2010,Post 2010,sector
+Walls,1.7,1.4,0.9,0.9,0.6,0.4,1.7,residential
+Windows,4.6,3.6,2.6,2.6,2.1,2.1,2.1,residential
+Roof,0.8,0.7,0.6,0.6,0.6,0.4,0.33,residential
+Floor,1.9,1.4,1.2,1.1,0.9,0.6,0.45,residential
+Walls,1.3,1.3,1.3,0.8,0.6,0.6,0.6,services
+Windows,4.7,3.7,2.6,2.6,2.3,2.1,2.1,services
+Roof,1,0.9,0.7,0.5,0.3,0.3,0.3,services
+Floor,1.6,1.2,1.2,1.1,1,0.7,0.7,services

data/retro/window_assumptions.csv

@@ -0,0 +1,8 @@
+strength,u_value,cost,u_limit,comment
+[m],[W/m^2K],EUR/m^2,[W/m^2K],
+0.076,1.34,180.08,3.5,Double-glazing
+0.197,0.8,225,1.3,Triple-glazing
+,,,,
+"source: https://www.agora-energiewende.de/en/publications/building-sector-efficiency-a-crucial-component-of-the-energy-transition/
+p.115
+",,,,

doc/conf.py

@@ -70,9 +70,9 @@ author = u'2019-2020 Tom Brown (KIT), Marta Victoria (Aarhus University), Lisa Z
 # built documents.
 #
 # The short X.Y version.
-version = u'0.1'
+version = u'0.4'
 # The full version, including alpha/beta/rc tags.
-release = u'0.1.0'
+release = u'0.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

doc/data.csv

@@ -0,0 +1,26 @@
+description,file/folder,licence,source
+JRC IDEES database,jrc-idees-2015/,CC BY 4.0,https://ec.europa.eu/jrc/en/potencia/jrc-idees
+urban/rural fraction,urban_percent.csv,unknown,unknown
+JRC biomass potentials,biomass/,unknown,https://doi.org/10.2790/39014
+EEA emission statistics,eea/,unknown,https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-14
+Eurostat Energy Balances,eurostat-energy_balances-*/,Eurostat,https://ec.europa.eu/eurostat/web/energy/data/energy-balances
+Swiss energy statistics from Swiss Federal Office of Energy,switzerland-sfoe/,unknown,http://www.bfe.admin.ch/themen/00526/00541/00542/02167/index.html?dossier_id=02169
+BASt emobility statistics,emobility/,unknown,http://www.bast.de/DE/Verkehrstechnik/Fachthemen/v2-verkehrszaehlung/Stundenwerte.html?nn=626916
+timezone mappings,timezone_mappings.csv,CC BY 4.0,Tom Brown
+BDEW heating profile,heat_load_profile_BDEW.csv,unknown,https://github.com/oemof/demandlib
+heating profiles for Aarhus,heat_load_profile_DK_AdamJensen.csv,unknown,Adam Jensen MA thesis at Aarhus University
+George Lavidas wind/wave costs,WindWaveWEC_GLTB.xlsx,unknown,George Lavidas
+country codes,Country_codes.csv,CC BY 4.0,Marta Victoria
+co2 budgets,co2_budget.csv,CC BY 4.0,https://arxiv.org/abs/2004.11009
+existing heating potentials,existing_infrastructure/existing_heating_raw.csv,unknown,https://ec.europa.eu/energy/studies/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment_en?redir=1
+IRENA existing VRE capacities,existing_infrastructure/{solar|onwind|offwind}_capcity_IRENA.csv,unknown,https://www.irena.org/Statistics/Download-Data
+USGS ammonia production,myb1-2017-nitro.xls,unknown,https://www.usgs.gov/centers/nmic/nitrogen-statistics-and-information
+hydrogen salt cavern potentials,hydrogen_salt_cavern_potentials.csv,CC BY 4.0,https://doi.org/10.1016/j.ijhydene.2019.12.161
+hotmaps industrial site database,Industrial_Database.csv,CC BY 4.0,https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database
+Hotmaps building stock data,data_building_stock.csv,CC BY 4.0,https://gitlab.com/hotmaps/building-stock
+U-values Poland,u_values_poland.csv,unknown,https://data.europa.eu/euodp/de/data/dataset/building-stock-observatory
+Floor area missing in hotmaps building stock data,floor_area_missing.csv,unknown,https://data.europa.eu/euodp/de/data/dataset/building-stock-observatory
+Comparative level investment,comparative_level_investment.csv,Eurostat,https://ec.europa.eu/eurostat/statistics-explained/index.php?title=Comparative_price_levels_for_investment
+Electricity taxes,electricity_taxes_eu.csv,Eurostat,https://appsso.eurostat.ec.europa.eu/nui/show.do?dataset=nrg_pc_204&lang=en
+Average surface components,average_surface_components.csv,unknown,http://webtool.building-typology.eu/#bm
+Retrofitting thermal envelope costs for Germany,retro_cost_germany.csv,unkown,https://www.iwu.de/forschung/handlungslogiken/kosten-energierelevanter-bau-und-anlagenteile-bei-modernisierung/

doc/index.rst

@@ -66,42 +66,6 @@ PyPSA-Eur-Sec is designed to be imported into the open toolbox `PyPSA <https://w
 This project is maintained by the `Energy System Modelling group <https://www.iai.kit.edu/english/2338.php>`_ at the `Institute for Automation and Applied Informatics <https://www.iai.kit.edu/english/index.php>`_ at the `Karlsruhe Institute of Technology <http://www.kit.edu/english/index.php>`_. The group is funded by the `Helmholtz Association <https://www.helmholtz.de/en/>`_ until 2024. Previous versions were developed by the `Renewable Energy Group <https://fias.uni-frankfurt.de/physics/schramm/renewable-energy-system-and-network-analysis/>`_ at `FIAS <https://fias.uni-frankfurt.de/>`_ to carry out simulations for the `CoNDyNet project <http://condynet.de/>`_, financed by the `German Federal Ministry for Education and Research (BMBF) <https://www.bmbf.de/en/index.html>`_ as part of the `Stromnetze Research Initiative <http://forschung-stromnetze.info/projekte/grundlagen-und-konzepte-fuer-effiziente-dezentrale-stromnetze/>`_.
 
 
-Spatial resolution of sectors
-=============================
-
-Not all of the sectors are at the full nodal resolution, and some are
-distributed to nodes using heuristics that need to be corrected. Some
-networks are copper-plated to reduce computational times.
-
-For example:
-
-Electricity network: nodal.
-
-Electricity demand: nodal, distributed in each country based on
-population and GDP.
-
-Building heating demand: nodal, distributed in each country based on
-population.
-
-Industry demand: nodal, distributed in each country based on
-population (will be corrected to real locations of industry, see
-github issue).
-
-Hydrogen network: nodal.
-
-Methane network: copper-plated for Europe, since future demand is so
-low and no bottlenecks are expected.
-
-Solid biomass: copper-plated until transport costs can be
-incorporated.
-
-CO2: copper-plated (but a transport and storage cost is added for
-sequestered CO2).
-
-Liquid hydrocarbons: copper-plated since transport costs are low.
-
-
-
 Documentation
 =============
 
@@ -116,6 +80,20 @@ Documentation
 
    installation
 
+**Implementation details**
+
+* :doc:`spatial_resolution`
+* :doc:`supply_demand`
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+   :caption: Implementation details
+
+   spatial_resolution
+   supply_demand
+
+
 **Foresight options**
 
 * :doc:`overnight`

doc/installation.rst

@@ -18,13 +18,17 @@ its dependencies. Clone the repository:
 
     projects % git clone git@github.com:PyPSA/pypsa-eur.git
 
-then download and unpack all the PyPSA-Eur data files.
+then download and unpack all the PyPSA-Eur data files by running the following snakemake rule:
+
+.. code:: bash
+
+    projects/pypsa-eur % snakemake -j 1 retrieve_databundle
 
 
 Clone technology-data repository
 ================================
 
-Create a parallel directory for the technology costs and other assumptions:
+Next install the technology assumptions database `technology-data <https://github.com/PyPSA/technology-data>`_ by creating a parallel directory:
 
 .. code:: bash
 
@@ -34,7 +38,7 @@ Create a parallel directory for the technology costs and other assumptions:
 Clone PyPSA-Eur-Sec repository
 ==============================
 
-Create a parallel directory for PyPSA-Eur-Sec with:
+Create a parallel directory for `PyPSA-Eur-Sec <https://github.com/PyPSA/pypsa-eur-sec>`_ with:
 
 .. code:: bash
 
@@ -54,19 +58,26 @@ atlite version 0.0.2.
 Data requirements
 =================
 
-The data requirements include the JRC-IDEES-2015 database, JRC biomass
-potentials, EEA emission statistics, Eurostat Energy Balances, urban
-district heating potentials, emobility statistics, timezone mappings
-and heating profiles.
+Small data files are included directly in the git repository, while
+larger ones are archived in a data bundle. The data bundle's size is
+around 640 MB.
 
-The data bundle is about 640 MB.
-
-To download and extract it on the command line:
+To download and extract the data bundle on the command line:
 
 .. code:: bash
 
-    projects/pypsa-eur-sec/data % wget "https://nworbmot.org/pypsa-eur-sec-data-bundle-190719.tar.gz"
-    projects/pypsa-eur-sec/data % tar xvzf pypsa-eur-sec-data-bundle-190719.tar.gz
+    projects/pypsa-eur-sec/data % wget "https://nworbmot.org/pypsa-eur-sec-data-bundle-201012.tar.gz"
+    projects/pypsa-eur-sec/data % tar xvzf pypsa-eur-sec-data-bundle-201012.tar.gz
+
+
+The data licences and sources are given in the following table.
+
+
+.. csv-table::
+   :header-rows: 1
+   :file: data.csv
+
+
 
 Set up the default configuration
 ================================

doc/myopic.rst

@@ -12,17 +12,19 @@ The current code applies the myopic approach to generators, storage technologies
 
 The transport sector and industry are not affected by the myopic code. In essence, the electrification of road and rail transport, the percentage of electric vehicles that allow demand-side management and vehicle-to-grid services, and the transformation in the different industrial subsectors do not evolve with time. They are kept fixed at the values specified in the configuration file. Including the transport sector and industry in the myopic code is planned for the near future.
 
-
+See also other `outstanding issues <https://github.com/PyPSA/pypsa-eur-sec/issues/19#issuecomment-678194802>`_.
 
 Configuration
 =================
 
-PyPSA-Eur-Sec has several configuration options which are collected in a config.yaml file located in the root directory. For myopic optimization, users should copy the provided myopic configuration ``config.myopic.yaml`` and make their own modifications and assumptions in the user-specific configuration file (``config.yaml``).
+PyPSA-Eur-Sec has several configuration options which are collected in a config.yaml file located in the root directory. For myopic optimization, users should copy the provided default configuration ``config.default.yaml`` and make their own modifications and assumptions in the user-specific configuration file (``config.yaml``).
 
 The following options included in the config.yaml file  are relevant for the myopic code.
 
 To activate the myopic option select ``foresight: 'myopic'`` in ``config.yaml``.
 
+To set the investment years which are sequentially simulated for the myopic investment planning, select for example ``planning_horizons : [2020, 2030, 2040, 2050]`` in ``config.yaml``.
+
 
 
 **existing capacities**

doc/overnight.rst

@@ -7,3 +7,5 @@ Overnight (greenfield) scenarios
 The default is to calculate a rebuilding of the energy system to meet demand, a so-called overnight or greenfield approach.
 
 For this, use ``foresight : 'overnight'`` in ``config.yaml``, like the example in ``config.default.yaml``.
+
+In this case, the ``planning_horizons : [2030]`` scenario parameter can be set to use the year from which cost and other technology assumptions are set (forecasts for 2030 in this case).

doc/release_notes.rst

@@ -2,13 +2,74 @@
 Release Notes
 ##########################################
 
-PyPSA-Eur-Sec 0.2.0 (TBD)
-==================================
 
-* Link to DEA cost database in another GitHub repository.
+PyPSA-Eur-Sec 0.4.0 (11th December 2020)
+=========================================
 
-* Myopic investment planning from Aarhus University.
+This release includes a more accurate nodal disaggregation of industry demand within each country, fixes to CHP and CCS representations, as well as changes to some configuration settings.
 
+It has been released to coincide with `PyPSA-Eur <https://github.com/PyPSA/pypsa-eur>`_ Version 0.3.0 and `Technology Data <https://github.com/PyPSA/technology-data>`_ Version 0.2.0, and is known to work with these releases.
+
+New features:
+
+* The `Hotmaps Industrial Database <https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database>`_ is used to disaggregate the industrial demand spatially to the nodes inside each country (previously it was distributed by population density).
+* Electricity demand from industry is now separated from the regular electricity demand and distributed according to the industry demand. Only the remaining regular electricity demand for households and services is distributed according to GDP and population.
+* A cost database for the retrofitting of the thermal envelope of residential and services buildings has been integrated, as well as endogenous optimisation of the level of retrofitting. This is described in the paper `Mitigating heat demand peaks in buildings in a highly renewable European energy system <https://arxiv.org/abs/2012.01831>`_. Retrofitting can be activated both exogenously and endogenously from the ``config.yaml``.
+* The biomass and gas combined heat and power (CHP) parameters ``c_v`` and ``c_b`` were read in assuming they were extraction plants rather than back pressure plants. The data is now corrected in `Technology Data <https://github.com/PyPSA/technology-data>`_ Version 0.2.0 to the correct DEA back pressure assumptions and they are now implemented as single links with a fixed ratio of electricity to heat output (even as extraction plants, they were always sitting on the backpressure line in simulations, so there was no point in modelling the full heat-electricity feasibility polygon). The old assumptions underestimated the heat output.
+* The Danish Energy Agency released `new assumptions for carbon capture <https://ens.dk/en/our-services/projections-and-models/technology-data/technology-data-industrial-process-heat-and>`_ in October 2020, which have now been incorporated in PyPSA-Eur-Sec, including direct air capture (DAC) and post-combustion capture on CHPs, cement kilns and other industrial facilities. The electricity and heat demand for DAC is modelled for each node (with heat coming from district heating), but currently the electricity and heat demand for industrial capture is not modelled very cleanly (for process heat, 10% of the energy is assumed to go to carbon capture) - a new issue will be opened on this.
+* Land transport is separated by energy carrier (fossil, hydrogen fuel cell electric vehicle, and electric vehicle), but still needs to be separated into heavy and light vehicles (the data is there, just not the code yet).
+* For assumptions that change with the investment year, there is a new time-dependent format in the ``config.yaml`` using a dictionary with keys for each year. Implemented examples include the CO2 budget, exogenous retrofitting share and land transport energy carrier; more parameters will be dynamised like this in future.
+* Some assumptions have been moved out of the code and into the ``config.yaml``, including the carbon sequestration potential and cost, the heat pump sink temperature, reductions in demand for high value chemicals, and some BEV DSM parameters and transport efficiencies.
+* Documentation on :doc:`supply_demand` options has been added.
+
+Many thanks to Fraunhofer ISI for opening the hotmaps database and to Lisa Zeyen (KIT) for implementing the building retrofitting.
+
+
+PyPSA-Eur-Sec 0.3.0 (27th September 2020)
+=========================================
+
+This releases focuses on improvements to industry demand and the generation of intermediate files for demand for basic materials. There are still inconsistencies with CCS and waste management that need to be improved.
+
+It is known to work with PyPSA-Eur v0.1.0 (commit bb3477cd69), PyPSA v0.17.1 and technology-data v0.1.0. Please note that the data bundle has also been updated.
+
+
+New features:
+
+* In previous version of PyPSA-Eur-Sec the energy demand for industry was calculated directly for each location. Now, instead, the production of each material (steel, cement, aluminium) at each location is calculated as an intermediate data file, before the energy demand is calculated from it. This allows us in future to have competing industrial processes for supplying the same material demand.
+* The script ``build_industrial_production_per_country_tomorrow.py`` determines the future industrial production of materials based on today's levels as well as assumed recycling and demand change measures.
+* The energy demand for each industry sector and each location in 2015 is also calculated, so that it can be later incorporated in the pathway optimization.
+* Ammonia production data is taken from the USGS and deducted from JRC-IDEES's "basic chemicals" so that it ammonia can be handled separately from the others (olefins, aromatics and chlorine).
+* Solid biomass is no longer allowed to be used for process heat in cement and basic chemicals, since the wastes and residues cannot be guaranteed to reach the high temperatures required. Instead, solid biomass is used in the paper and pulp as well as food, beverages and tobacco industries, where required temperatures are lower (see `DOI:10.1002/er.3436 <https://doi.org/10.1002/er.3436>`_ and `DOI:10.1007/s12053-017-9571-y <https://doi.org/10.1007/s12053-017-9571-y>`_).
+* National installable potentials for salt caverns are now applied.
+* When electricity distribution grids are activated, new industry electricity demand, resistive heaters and micro-CHPs are now connected to the lower voltage levels.
+* Gas distribution grid costs are included for gas boilers and micro-CHPs.
+* Installable potentials for rooftop PV are included with an assumption of 1 kWp per person.
+* Some intermediate files produced by scripts have been moved from the folder ``data`` to the folder ``resources``. Now ``data`` only includes input data, while ``resources`` only includes intermediate files necessary for building the network models. Please note that the data bundle has also been updated.
+* Biomass potentials for different years and scenarios from the JRC are generated in an intermediate file, so that a selection can be made more explicitly by specifying the biomass types from the ``config.yaml``.
+
+
+PyPSA-Eur-Sec 0.2.0 (21st August 2020)
+======================================
+
+This release introduces pathway optimization over many years (e.g. 2020, 2030, 2040, 2050) with myopic foresight, as well as outsourcing the technology assumptions to the `technology-data <https://github.com/PyPSA/technology-data>`_ repository.
+
+It is known to work with PyPSA-Eur v0.1.0 (commit bb3477cd69), PyPSA v0.17.1 and technology-data v0.1.0.
+
+New features:
+
+* Option for pathway optimization with myopic foresight, based on the paper `Early decarbonisation of the European Energy system pays off (2020) <https://arxiv.org/abs/2004.11009>`_. Investments are optimized sequentially for multiple years (e.g. 2020, 2030, 2040, 2050) taking account of existing assets built in previous years and their lifetimes. The script uses data on the existing assets for electricity and building heating technologies, but there are no assumptions yet for existing transport and industry (if you include these, the model will greenfield them). There are also some `outstanding issues <https://github.com/PyPSA/pypsa-eur-sec/issues/19#issuecomment-678194802>`_ on e.g. the distribution of existing wind, solar and heating technologies within each country. To use myopic foresight, set ``foresight : 'myopic'`` in the ``config.yaml`` instead of the default ``foresight : 'overnight'``. An example configuration can be found in ``config.myopic.yaml``. More details on the implementation can be found in :doc:`myopic`.
+
+* Technology assumptions (costs, efficiencies, etc.) are no longer stored in the repository. Instead, you have to install the `technology-data <https://github.com/PyPSA/technology-data>`_ database in a parallel directory. These assumptions are largely based on the `Danish Energy Agency Technology Data <https://ens.dk/en/our-services/projections-and-models/technology-data>`_. More details on the installation can be found in :doc:`installation`.
+
+* Logs and benchmarks are now stored with the other model outputs in ``results/run-name/``.
+
+* All buses now have a ``location`` attribute, e.g. bus ``DE0 3 urban central heat`` has a ``location`` of ``DE0 3``.
+
+* All assets have a ``lifetime`` attribute (integer in years). For the myopic foresight, a ``build_year`` attribute is also stored.
+
+* Costs for solar and onshore and offshore wind are recalculated by PyPSA-Eur-Sec based on the investment year, including the AC or DC connection costs for offshore wind.
+
+Many thanks to Marta Victoria for implementing the myopic foresight, and Marta Victoria, Kun Zhu and Lisa Zeyen for developing the technology assumptions database.
 
 
 PyPSA-Eur-Sec 0.1.0 (8th July 2020)
@@ -50,14 +111,20 @@ the additional sectors.
 Release Process
 ===============
 
-* Checkout a new release branch ``git checkout -b release-v0.x.x``.
-
 * Finalise release notes at ``doc/release_notes.rst``.
 
 * Update version number in ``doc/conf.py`` and ``*config.*.yaml``.
 
+* Make a ``git commit``.
+
 * Tag a release by running ``git tag v0.x.x``, ``git push``, ``git push --tags``. Include release notes in the tag message.
 
 * Make a `GitHub release <https://github.com/PyPSA/pypsa-eur-sec/releases>`_, which automatically triggers archiving by `zenodo <https://doi.org/10.5281/zenodo.3938042>`_.
 
 * Send announcement on the `PyPSA mailing list <https://groups.google.com/forum/#!forum/pypsa>`_.
+
+To make a new release of the data bundle, make an archive of the files in ``data`` which are not already included in the git repository:
+
+.. code:: bash
+
+    data % tar pczf pypsa-eur-sec-data-bundle-date.tar.gz eea switzerland-sfoe biomass eurostat-energy_balances-* jrc-idees-2015 emobility urban_percent.csv timezone_mappings.csv heat_load_profile_DK_AdamJensen.csv WindWaveWEC_GLTB.xlsx myb1-2017-nitro.xls Industrial_Database.csv

doc/spatial_resolution.rst

@@ -0,0 +1,54 @@
+.. _spatial_resolution:
+
+##########################################
+Spatial resolution
+##########################################
+
+The default nodal resolution of the model follows the electricity
+generation and transmission model `PyPSA-Eur
+<https://github.com/PyPSA/pypsa-eur>`_, which clusters down the
+electricity transmission substations in each European country based on
+the k-means algorithm. This gives nodes which correspond to major load
+and generation centres (typically cities).
+
+The total number of nodes for Europe is set in the ``config.yaml`` file
+under ``clusters``. The number of nodes can vary between 37, the number
+of independent countries / synchronous areas, and several
+hundred. With 200-300 nodes the model needs 100-150 GB RAM to solve
+with a commerical solver like Gurobi.
+
+
+Not all of the sectors are at the full nodal resolution, and some
+demand for some sectors is distributed to nodes using heuristics that
+need to be corrected. Some networks are copper-plated to reduce
+computational times.
+
+For example:
+
+Electricity network: nodal.
+
+Electricity residential and commercial demand: nodal, distributed in
+each country based on population and GDP.
+
+Electricity demand in industry: based on the location of industrial
+facilities from `HotMaps database <https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database>`_.
+
+Building heating demand: nodal, distributed in each country based on
+population.
+
+Industry demand: nodal, distributed in each country based on
+locations of industry from `HotMaps database <https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database>`_.
+
+Hydrogen network: nodal.
+
+Methane network: single node for Europe, since future demand is so
+low and no bottlenecks are expected.
+
+Solid biomass:  single node for Europe, until transport costs can be
+incorporated.
+
+CO2:  single node for Europe, but a transport and storage cost is added for
+sequestered CO2.
+
+Liquid hydrocarbons: single node for Europe, since transport costs for
+liquids are low.

doc/supply_demand.rst

@@ -0,0 +1,193 @@
+.. _supply_demand:
+
+##########################################
+Supply and demand
+##########################################
+
+An initial orientation to the supply and demand options in the model
+PyPSA-Eur-Sec can be found in the description of the model
+PyPSA-Eur-Sec-30 in the paper `Synergies of sector coupling and
+transmission reinforcement in a cost-optimised, highly renewable
+European energy system <https://arxiv.org/abs/1801.05290>`_ (2018).
+The latest version of PyPSA-Eur-Sec differs by including biomass,
+industry, industrial feedstocks, aviation, shipping, better carbon
+management, carbon capture and usage/sequestration, and gas networks.
+
+The basic supply (left column) and demand (right column) options in the model are described in this figure:
+
+.. image:: ../graphics/multisector_figure.png
+
+
+
+Electricity supply and demand
+=============================
+
+Electricity supply and demand follows the electricity generation and
+transmission model `PyPSA-Eur <https://github.com/PyPSA/pypsa-eur>`_,
+except that hydrogen storage is integrated into the hydrogen supply,
+demand and network, and PyPSA-Eur-Sec includes CHPs.
+
+Unlike PyPSA-Eur, PyPSA-Eur-Sec does not distribution electricity demand for industry according to population and GDP, but uses the
+geographical data from the `Hotmaps Industrial Database
+<https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database>`_.
+
+Also unlike PyPSA-Eur, PyPSA-Eur-Sec subtracts existing electrified heating from the existing electricity demand, so that power-to-heat can be optimised separately.
+
+The remaining electricity demand for households and services is distributed inside each country proportional to GDP and population.
+
+
+Heat demand
+=============================
+
+Heat demand is split into:
+
+* ``urban central``: large-scale district heating networks in urban areas with dense heat demand
+* ``residential/services urban decentral``: heating for individual buildings in urban areas
+* ``residential/services rural``: heating for individual buildings in rural areas
+
+
+Heat supply
+=======================
+
+Oil and gas boilers
+--------------------
+
+Heat pumps
+-------------
+
+Either air-to-water or ground-to-water heat pumps are implemented.
+
+They have coefficient of performance (COP) based on either the
+external air or the soil hourly temperature.
+
+Ground-source heat pumps are only allowed in rural areas because of
+space constraints.
+
+Only air-source heat pumps are allowed in urban areas. This is a
+conservative assumption, since there are many possible sources of
+low-temperature heat that could be tapped in cities (waste water,
+rivers, lakes, seas, etc.).
+
+Resistive heaters
+--------------------
+
+
+Large Combined Heat and Power (CHP) plants
+--------------------------------------------
+
+A good summary of CHP options that can be implemented in PyPSA can be found in the paper `Cost sensitivity of optimal sector-coupled district heating production systems <https://doi.org/10.1016/j.energy.2018.10.044>`_.
+
+PyPSA-Eur-Sec includes CHP plants fuelled by methane, hydrogen and solid biomass from waste and residues.
+
+Hydrogen CHPs are fuel cells.
+
+Methane and biomass CHPs are based on back pressure plants operating with a fixed ratio of electricity to heat output. The methane CHP is modelled on the Danish Energy Agency (DEA) "Gas turbine simple cycle (large)" while the solid biomass CHP is based on the DEA's "09b Wood Pellets Medium".
+
+The efficiencies of each are given on the back pressure line, where the back pressure coefficient ``c_b`` is the electricity output divided by the heat output. The plants are not allowed to deviate from the back pressure line and are implement as ``Link`` objects with a fixed ratio of heat to electricity output.
+
+
+NB: The old PyPSA-Eur-Sec-30 model assumed an extraction plant (like the DEA coal CHP) for gas which has flexible production of heat and electricity within the feasibility diagram of Figure 4 in the `Synergies paper <https://arxiv.org/abs/1801.05290>`_. We have switched to the DEA back pressure plants since these are more common for smaller plants for biomass, and because the extraction plants were on the back pressure line for 99.5% of the time anyway. The plants were all changed to back pressure in PyPSA-Eur-Sec v0.4.0.
+
+
+Micro-CHP for individual buildings
+-----------------------------------
+
+Optional.
+
+Waste heat from Fuel Cells, Methanation and Fischer-Tropsch plants
+-------------------------------------------------------------------
+
+
+Solar thermal collectors
+-------------------------
+
+Thermal energy storage using hot water tanks
+---------------------------------------------
+
+Small for decentral applications.
+
+Big pit storage for district heating.
+
+
+Hydrogen demand
+==================
+
+Stationary fuel cell CHP.
+
+Transport applications.
+
+Industry (ammonia, precursor to hydrocarbons for chemicals and iron/steel).
+
+
+Hydrogen supply
+=================
+
+SMR, SMR+CCS, electrolysers.
+
+
+Methane demand
+==================
+
+Can be used in boilers, in CHPs, in industry for high temperature heat, in OCGT.
+
+Not used in transport because of engine slippage.
+
+Methane supply
+=================
+
+Fossil, biogas, Sabatier (hydrogen to methane), HELMETH (directly power to methane with efficient heat integration).
+
+
+Solid biomass demand
+=====================
+
+Solid biomass provides process heat up to 500 Celsius in industry, as well as feeding CHP plants in district heating networks.
+
+Solid biomass supply
+=====================
+
+Only wastes and residues from the JRC biomass dataset.
+
+
+Oil product demand
+=====================
+
+Transport fuels and naphtha as a feedstock for the chemicals industry.
+
+Oil product supply
+======================
+
+Fossil or Fischer-Tropsch.
+
+
+Industry demand
+================
+
+Based on materials demand from JRC-IDEES and other sources such as the USGS for ammonia.
+
+Industry is split into many sectors, including iron and steel, ammonia, other basic chemicals, cement, non-metalic minerals, alumuninium, other non-ferrous metals, pulp, paper and printing, food, beverages and tobacco, and other more minor sectors.
+
+Inside each country the industrial demand is distributed using the `Hotmaps Industrial Database <https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database>`_.
+
+
+Industry supply
+================
+
+Process switching (e.g. from blast furnaces to direct reduction and electric arc furnaces for steel) is defined exogenously.
+
+Fuel switching for process heat is mostly also done exogenously.
+
+Solid biomass is used for up to 500 Celsius, mostly in paper and pulp and food and beverages.
+
+Higher temperatures are met with methane.
+
+
+Carbon dioxide capture, usage and sequestration (CCU/S)
+=========================================================
+
+Carbon dioxide can be captured from industry process emissions,
+emissions related to industry process heat, combined heat and power
+plants, and directly from the air (DAC).
+
+Carbon dioxide can be used as an input for methanation and
+Fischer-Tropsch fuels, or it can be sequestered underground.

scripts/add_brownfield.py

@@ -98,8 +98,8 @@ if __name__ == "__main__":
             output=['pypsa-eur-sec/results/test/prenetworks_brownfield/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{planning_horizons}.nc']
         )
         import yaml
-        with open('config.yaml') as f:
-            snakemake.config = yaml.load(f)
+        with open('config.yaml', encoding='utf8') as f:
+            snakemake.config = yaml.safe_load(f)
 
     print(snakemake.input.network_p)
     logging.basicConfig(level=snakemake.config['logging_level'])

scripts/add_existing_baseyear.py

@@ -170,10 +170,8 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
     df_agg.Fueltype = df_agg.Fueltype.map(rename_fuel)
 
     #assign clustered bus
-    busmap_s = pd.read_hdf(snakemake.input.clustermaps,
-                           key="/busmap_s")
-    busmap = pd.read_hdf(snakemake.input.clustermaps,
-                         key="/busmap")
+    busmap_s = pd.read_csv(snakemake.input.busmap_s, index_col=0).squeeze()
+    busmap = pd.read_csv(snakemake.input.busmap, index_col=0).squeeze()
     clustermaps = busmap_s.map(busmap)
     clustermaps.index = clustermaps.index.astype(int)
 
@@ -416,15 +414,16 @@ if __name__ == "__main__":
                            planning_horizons='2020'),
             input=dict(network='pypsa-eur-sec/results/test/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{co2_budget_name}_{planning_horizons}.nc',
                        powerplants='pypsa-eur/resources/powerplants.csv',
-                       clustermaps='pypsa-eur/resources/clustermaps_{network}_s{simpl}_{clusters}.h5',
+                       busmap_s='pypsa-eur/resources/busmap_{network}_s{simpl}.csv',
+                       busmap='pypsa-eur/resources/busmap_{network}_s{simpl}_{clusters}.csv',
                        costs='pypsa-eur-sec/data/costs/costs_{planning_horizons}.csv',
                        cop_air_total="pypsa-eur-sec/resources/cop_air_total_{network}_s{simpl}_{clusters}.nc",
                        cop_soil_total="pypsa-eur-sec/resources/cop_soil_total_{network}_s{simpl}_{clusters}.nc"),
             output=['pypsa-eur-sec/results/test/prenetworks_brownfield/{network}_s{simpl}_{clusters}_lv{lv}__{sector_opts}_{planning_horizons}.nc'],
         )
         import yaml
-        with open('config.yaml') as f:
-            snakemake.config = yaml.load(f)
+        with open('config.yaml', encoding='utf8') as f:
+            snakemake.config = yaml.safe_load(f)
 
 
     logging.basicConfig(level=snakemake.config['logging_level'])
@@ -443,7 +442,8 @@ if __name__ == "__main__":
     costs = prepare_costs(snakemake.input.costs,
                           snakemake.config['costs']['USD2013_to_EUR2013'],
                           snakemake.config['costs']['discountrate'],
-                          Nyears)
+                          Nyears,
+                          snakemake.config['costs']['lifetime'])
 
     grouping_years=snakemake.config['existing_capacities']['grouping_years']
     add_power_capacities_installed_before_baseyear(n, grouping_years, costs, baseyear)

scripts/build_ammonia_production.py

@@ -0,0 +1,45 @@
+
+
+import pandas as pd
+
+ammonia = pd.read_excel(snakemake.input.usgs,
+                        sheet_name="T12",
+                        skiprows=5,
+                        header=0,
+                        index_col=0,
+                        skipfooter=19)
+
+rename = {"Austriae" : "AT",
+          "Bulgaria" : "BG",
+          "Belgiume" : "BE",
+          "Croatia" : "HR",
+          "Czechia" : "CZ",
+          "Estonia" : "EE",
+          "Finland" : "FI",
+          "France" : "FR",
+          "Germany" : "DE",
+          "Greece" : "GR",
+          "Hungarye" : "HU",
+          "Italye" : "IT",
+          "Lithuania" : "LT",
+          "Netherlands" : "NL",
+          "Norwaye" : "NO",
+          "Poland" : "PL",
+          "Romania" : "RO",
+          "Serbia" : "RS",
+          "Slovakia" : "SK",
+          "Spain" : "ES",
+          "Switzerland" : "CH",
+          "United Kingdom" : "GB",
+}
+
+ammonia = ammonia.rename(rename)
+
+ammonia = ammonia.loc[rename.values(),[str(i) for i in range(2013,2018)]].astype(float)
+
+#convert from ktonN to ktonNH3
+ammonia = ammonia*17/14
+
+ammonia.index.name = "ktonNH3/a"
+
+ammonia.to_csv(snakemake.output.ammonia_production)

scripts/build_biomass_potentials.py

@@ -19,7 +19,10 @@ def build_biomass_potentials():
     for i in range(36):
         df_dict[df.iloc[i*16,1]] = df.iloc[1+i*16:(i+1)*16].astype(float)
 
-    df_new = pd.concat(df_dict)
+    #convert from PJ to MWh
+    df_new = pd.concat(df_dict).rename({"UK" : "GB", "BH" : "BA"})/3.6*1e6
+    df_new.index.name = "MWh/a"
+    df_new.to_csv(snakemake.output.biomass_potentials_all)
 
     #  solid biomass includes: Primary agricultural residues (MINBIOAGRW1),
     #  Forestry energy residue (MINBIOFRSF1),
@@ -31,17 +34,13 @@ def build_biomass_potentials():
     # biogas includes : Manure biomass potential (MINBIOGAS1),
     # Sludge biomass (MINBIOSLU1)
 
-    us_type = pd.Series(index=df_new.columns)
-    us_type.iloc[0:7] = "not included"
-    us_type.iloc[7:8] = "biogas"
-    us_type.iloc[8:9] = "solid biomass"
-    us_type.iloc[9:11] = "not included"
-    us_type.iloc[11:16] = "solid biomass"
-    us_type.iloc[16:17] = "biogas"
+    us_type = pd.Series("", df_new.columns)
 
+    for k,v in snakemake.config['biomass']['classes'].items():
+        us_type.loc[v] = k
 
-    #convert from PJ to MWh
-    biomass_potentials = df_new.loc[idx[:,snakemake.config['biomass']['year'],snakemake.config['biomass']['scenario']],:].groupby(us_type,axis=1).sum().groupby(level=0).sum().rename({"UK" : "GB", "BH" : "BA"})/3.6*1e6
+    biomass_potentials = df_new.swaplevel(0,2).loc[snakemake.config['biomass']['scenario'],snakemake.config['biomass']['year']].groupby(us_type,axis=1).sum()
+    biomass_potentials.index.name = "MWh/a"
     biomass_potentials.to_csv(snakemake.output.biomass_potentials)
 
 
@@ -58,7 +57,7 @@ if __name__ == "__main__":
         snakemake.input['jrc_potentials'] = "data/biomass/JRC Biomass Potentials.xlsx"
         snakemake.output = Dict()
         snakemake.output['biomass_potentials'] = 'data/biomass_potentials.csv'
-        with open('config.yaml') as f:
-            snakemake.config = yaml.load(f)
+        with open('config.yaml', encoding='utf8') as f:
+            snakemake.config = yaml.safe_load(f)
 
     build_biomass_potentials()

scripts/build_cop_profiles.py

@@ -9,16 +9,13 @@ import xarray as xr
 cop_f = {"air" : lambda d_t: 6.81 -0.121*d_t + 0.000630*d_t**2,
          "soil" : lambda d_t: 8.77 -0.150*d_t + 0.000734*d_t**2}
 
-sink_T = 55. # Based on DTU / large area radiators
-
-
 
 for area in ["total", "urban", "rural"]:
     for source in ["air", "soil"]:
 
         source_T = xr.open_dataarray(snakemake.input["temp_{}_{}".format(source,area)])
 
-        delta_T = sink_T - source_T
+        delta_T = snakemake.config['sector']['heat_pump_sink_T'] - source_T
 
         cop = cop_f[source](delta_T)
 

scripts/build_energy_totals.py

@@ -378,12 +378,12 @@ def build_energy_totals():
     clean_df.loc[missing,"total aviation passenger"] = clean_df.loc[missing,["total domestic aviation passenger","total international aviation passenger"]].sum(axis=1)
     clean_df.loc[missing,"total aviation freight"] = clean_df.loc[missing,["total domestic aviation freight","total international aviation freight"]].sum(axis=1)
 
+    if "BA" in clean_df.index:
+        #fix missing data for BA (services and road energy data)
+        missing = (clean_df.loc["BA"] == 0.)
 
-    #fix missing data for BA (services and road energy data)
-    missing = (clean_df.loc["BA"] == 0.)
-
-    #add back in proportional to RS with ratio of total residential demand
-    clean_df.loc["BA",missing] = clean_df.loc["BA","total residential"]/clean_df.loc["RS","total residential"]*clean_df.loc["RS",missing]
+        #add back in proportional to RS with ratio of total residential demand
+        clean_df.loc["BA",missing] = clean_df.loc["BA","total residential"]/clean_df.loc["RS","total residential"]*clean_df.loc["RS",missing]
 
     clean_df.to_csv(snakemake.output.energy_name)
 

scripts/build_industrial_demand.py

@@ -7,7 +7,7 @@ def build_industrial_demand():
     pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout,index_col=0)
     pop_layout["ct"] = pop_layout.index.str[:2]
     ct_total = pop_layout.total.groupby(pop_layout["ct"]).sum()
-    pop_layout["ct_total"] = pop_layout["ct"].map(ct_total.get)
+    pop_layout["ct_total"] = pop_layout["ct"].map(ct_total)
     pop_layout["fraction"] = pop_layout["total"]/pop_layout["ct_total"]
 
     industrial_demand_per_country = pd.read_csv(snakemake.input.industrial_demand_per_country,index_col=0)
@@ -33,7 +33,7 @@ if __name__ == "__main__":
         snakemake.input['industrial_demand_per_country']="resources/industrial_demand_per_country.csv"
         snakemake.output = Dict()
         snakemake.output['industrial_demand'] = "resources/industrial_demand_elec_s_128.csv"
-        with open('config.yaml') as f:
-            snakemake.config = yaml.load(f)
+        with open('config.yaml', encoding='utf8') as f:
+            snakemake.config = yaml.safe_load(f)
 
     build_industrial_demand()

scripts/build_industrial_distribution_key.py

@@ -0,0 +1,153 @@
+
+import pypsa
+import pandas as pd
+import geopandas as gpd
+from shapely import wkt, prepared
+from scipy.spatial import cKDTree as KDTree
+
+
+def prepare_hotmaps_database():
+
+    df = pd.read_csv(snakemake.input.hotmaps_industrial_database,
+                     sep=";",
+                     index_col=0)
+
+    #remove those sites without valid geometries
+    df.drop(df.index[df.geom.isna()],
+            inplace=True)
+
+    #parse geometry
+    #https://geopandas.org/gallery/create_geopandas_from_pandas.html?highlight=parse#from-wkt-format
+    df["Coordinates"] = df.geom.apply(lambda x : wkt.loads(x[x.find(";POINT")+1:]))
+
+    gdf = gpd.GeoDataFrame(df, geometry='Coordinates')
+
+    europe_shape = gpd.read_file(snakemake.input.europe_shape).loc[0, 'geometry']
+    europe_shape_prepped = prepared.prep(europe_shape)
+    not_in_europe = gdf.index[~gdf.geometry.apply(europe_shape_prepped.contains)]
+    print("Removing the following industrial facilities since they are not in European area:")
+    print(gdf.loc[not_in_europe])
+    gdf.drop(not_in_europe,
+             inplace=True)
+
+    country_to_code = {
+        'Belgium' : 'BE',
+        'Bulgaria' : 'BG',
+        'Czech Republic' : 'CZ',
+        'Denmark' : 'DK',
+        'Germany' : 'DE',
+        'Estonia' : 'EE',
+        'Ireland' : 'IE',
+        'Greece' : 'GR',
+        'Spain' : 'ES',
+        'France' : 'FR',
+        'Croatia' : 'HR',
+        'Italy' : 'IT',
+        'Cyprus' : 'CY',
+        'Latvia' : 'LV',
+        'Lithuania' : 'LT',
+        'Luxembourg' : 'LU',
+        'Hungary' : 'HU',
+        'Malta' : 'MA',
+        'Netherland' : 'NL',
+        'Austria' : 'AT',
+        'Poland' : 'PL',
+        'Portugal' : 'PT',
+        'Romania' : 'RO',
+        'Slovenia' : 'SI',
+        'Slovakia' : 'SK',
+        'Finland' : 'FI',
+        'Sweden' : 'SE',
+        'United Kingdom' : 'GB',
+        'Iceland' : 'IS',
+        'Norway' : 'NO',
+        'Montenegro' : 'ME',
+        'FYR of Macedonia' : 'MK',
+        'Albania' : 'AL',
+        'Serbia' : 'RS',
+        'Turkey' : 'TU',
+        'Bosnia and Herzegovina' : 'BA',
+        'Switzerland' : 'CH',
+        'Liechtenstein' : 'AT',
+    }
+    gdf["country_code"] = gdf.Country.map(country_to_code)
+
+    if gdf["country_code"].isna().any():
+        print("Warning, some countries not assigned an ISO code")
+
+    gdf["x"] = gdf.geometry.x
+    gdf["y"] = gdf.geometry.y
+
+    return gdf
+
+
+def assign_buses(gdf):
+
+    gdf["bus"] = ""
+
+    for c in n.buses.country.unique():
+        buses_i = n.buses.index[n.buses.country == c]
+        kdtree = KDTree(n.buses.loc[buses_i, ['x','y']].values)
+
+        industry_i = gdf.index[(gdf.country_code == c)]
+
+        if industry_i.empty:
+            print("Skipping country with no industry:",c)
+        else:
+            tree_i = kdtree.query(gdf.loc[industry_i, ['x','y']].values)[1]
+            gdf.loc[industry_i, 'bus'] = buses_i[tree_i]
+
+    if (gdf.bus == "").any():
+        print("Some industrial facilities have empty buses")
+    if gdf.bus.isna().any():
+        print("Some industrial facilities have NaN buses")
+
+
+def build_nodal_distribution_key(gdf):
+
+    sectors = ['Iron and steel','Chemical industry','Cement','Non-metallic mineral products','Glass','Paper and printing','Non-ferrous metals']
+
+    distribution_keys = pd.DataFrame(index=n.buses.index,
+                                    columns=sectors,
+                                    dtype=float)
+
+    pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout,index_col=0)
+    pop_layout["ct"] = pop_layout.index.str[:2]
+    ct_total = pop_layout.total.groupby(pop_layout["ct"]).sum()
+    pop_layout["ct_total"] = pop_layout["ct"].map(ct_total)
+    distribution_keys["population"] = pop_layout["total"]/pop_layout["ct_total"]
+
+    for c in n.buses.country.unique():
+        buses = n.buses.index[n.buses.country == c]
+        for sector in sectors:
+            facilities = gdf.index[(gdf.country_code == c) & (gdf.Subsector == sector)]
+            if not facilities.empty:
+                emissions = gdf.loc[facilities,"Emissions_ETS_2014"]
+                if emissions.sum() == 0:
+                    distribution_key = pd.Series(1/len(facilities),
+                                                 facilities)
+                else:
+                    #BEWARE: this is a strong assumption
+                    emissions = emissions.fillna(emissions.mean())
+                    distribution_key = emissions/emissions.sum()
+                distribution_key = distribution_key.groupby(gdf.loc[facilities,"bus"]).sum().reindex(buses,fill_value=0.)
+            else:
+                distribution_key = distribution_keys.loc[buses,"population"]
+
+            if abs(distribution_key.sum() - 1) > 1e-4:
+                print(c,sector,distribution_key)
+
+            distribution_keys.loc[buses,sector] = distribution_key
+
+    distribution_keys.to_csv(snakemake.output.industrial_distribution_key)
+
+if __name__ == "__main__":
+
+
+    n = pypsa.Network(snakemake.input.network)
+
+    hotmaps_database = prepare_hotmaps_database()
+
+    assign_buses(hotmaps_database)
+
+    build_nodal_distribution_key(hotmaps_database)

scripts/build_industrial_energy_demand_per_country.py

@@ -0,0 +1,83 @@
+
+import pandas as pd
+import numpy as np
+
+
+tj_to_ktoe = 0.0238845
+ktoe_to_twh = 0.01163
+
+eb_base_dir = "data/eurostat-energy_balances-may_2018_edition"
+jrc_base_dir = "data/jrc-idees-2015"
+
+# import EU ratios df as csv
+industry_sector_ratios=pd.read_csv(snakemake.input.industry_sector_ratios,
+                                   index_col=0)
+
+#material demand per country and industry (kton/a)
+countries_production = pd.read_csv(snakemake.input.industrial_production_per_country, index_col=0)
+
+#Annual energy consumption in Switzerland by sector in 2015 (in TJ)
+#From: Energieverbrauch in der Industrie und im Dienstleistungssektor, Der Bundesrat
+#http://www.bfe.admin.ch/themen/00526/00541/00543/index.html?lang=de&dossier_id=00775
+
+dic_Switzerland ={'Iron and steel': 7889.,
+          'Chemicals Industry': 26871.,
+          'Non-metallic mineral products': 15513.+3820.,
+          'Pulp, paper and printing': 12004.,
+          'Food, beverages and tobacco': 17728.,
+          'Non Ferrous Metals': 3037.,
+          'Transport Equipment': 14993.,
+          'Machinery Equipment': 4724.,
+          'Textiles and leather': 1742.,
+          'Wood and wood products': 0.,
+          'Other Industrial Sectors': 10825.,
+          'current electricity': 53760.}
+
+
+eb_names={'NO':'Norway', 'AL':'Albania', 'BA':'Bosnia and Herzegovina',
+          'MK':'FYR of Macedonia', 'GE':'Georgia', 'IS':'Iceland',
+          'KO':'Kosovo', 'MD':'Moldova', 'ME':'Montenegro', 'RS':'Serbia',
+          'UA':'Ukraine', 'TR':'Turkey', }
+
+jrc_names = {"GR" : "EL",
+             "GB" : "UK"}
+
+#final energy consumption per country and industry (TWh/a)
+countries_df = countries_production.dot(industry_sector_ratios.T)
+countries_df*= 0.001 #GWh -> TWh (ktCO2 -> MtCO2)
+
+
+
+non_EU = ['NO', 'CH', 'ME', 'MK', 'RS', 'BA', 'AL']
+
+
+# save current electricity consumption
+for country in countries_df.index:
+    if country in non_EU:
+        if country == 'CH':
+            countries_df.loc[country, 'current electricity']=dic_Switzerland['current electricity']*tj_to_ktoe*ktoe_to_twh
+        else:
+            excel_balances = pd.read_excel('{}/{}.XLSX'.format(eb_base_dir,eb_names[country]),
+                                      sheet_name='2016', index_col=1,header=0, skiprows=1 ,squeeze=True)
+
+            countries_df.loc[country, 'current electricity'] = excel_balances.loc['Industry', 'Electricity']*ktoe_to_twh
+
+    else:
+
+        excel_out = pd.read_excel('{}/JRC-IDEES-2015_Industry_{}.xlsx'.format(jrc_base_dir,jrc_names.get(country,country)),
+                                  sheet_name='Ind_Summary',index_col=0,header=0,squeeze=True) # the summary sheet
+
+        s_out = excel_out.iloc[27:48,-1]
+        countries_df.loc[country, 'current electricity'] = s_out['Electricity']*ktoe_to_twh
+
+
+rename_sectors = {'elec':'electricity',
+                  'biomass':'solid biomass',
+                  'heat':'low-temperature heat'}
+
+countries_df.rename(columns=rename_sectors,inplace=True)
+
+countries_df.index.name = "TWh/a (MtCO2/a)"
+
+countries_df.to_csv(snakemake.output.industrial_energy_demand_per_country,
+                    float_format='%.2f')

scripts/build_industrial_energy_demand_per_country_today.py

@@ -0,0 +1,140 @@
+
+import pandas as pd
+
+# sub-sectors as used in PyPSA-Eur-Sec and listed in JRC-IDEES industry sheets
+sub_sectors = {'Iron and steel' : ['Integrated steelworks','Electric arc'],
+               'Non-ferrous metals' : ['Alumina production','Aluminium - primary production','Aluminium - secondary production','Other non-ferrous metals'],
+               'Chemicals' : ['Basic chemicals', 'Other chemicals', 'Pharmaceutical products etc.', 'Basic chemicals feedstock'],
+               'Non-metalic mineral' : ['Cement','Ceramics & other NMM','Glass production'],
+               'Printing' : ['Pulp production','Paper production','Printing and media reproduction'],
+               'Food' : ['Food, beverages and tobacco'],
+               'Transport equipment' : ['Transport Equipment'],
+               'Machinery equipment' : ['Machinery Equipment'],
+               'Textiles and leather' : ['Textiles and leather'],
+               'Wood and wood products' : ['Wood and wood products'],
+               'Other Industrial Sectors' : ['Other Industrial Sectors'],
+}
+
+
+# name in JRC-IDEES Energy Balances
+eb_sheet_name = {'Integrated steelworks' : 'cisb',
+                 'Electric arc' : 'cise',
+                 'Alumina production' : 'cnfa',
+                 'Aluminium - primary production' : 'cnfp',
+                 'Aluminium - secondary production' : 'cnfs',
+                 'Other non-ferrous metals' : 'cnfo',
+                 'Basic chemicals' : 'cbch',
+                 'Other chemicals' : 'coch',
+                 'Pharmaceutical products etc.' : 'cpha',
+                 'Basic chemicals feedstock' : 'cpch',
+                 'Cement' : 'ccem',
+                 'Ceramics & other NMM' : 'ccer',
+                 'Glass production' : 'cgla',
+                 'Pulp production' : 'cpul',
+                 'Paper production' : 'cpap',
+                 'Printing and media reproduction' : 'cprp',
+                 'Food, beverages and tobacco' : 'cfbt',
+                 'Transport Equipment' : 'ctre',
+                 'Machinery Equipment' : 'cmae',
+                 'Textiles and leather' : 'ctel',
+                 'Wood and wood products' : 'cwwp',
+                 'Mining and quarrying' : 'cmiq',
+                 'Construction' : 'ccon',
+                 'Non-specified': 'cnsi',
+}
+
+
+
+fuels = {'all' : ['All Products'],
+         'solid' : ['Solid Fuels'],
+         'liquid' : ['Total petroleum products (without biofuels)'],
+         'gas' : ['Gases'],
+         'heat' : ['Nuclear heat','Derived heat'],
+         'biomass' : ['Biomass and Renewable wastes'],
+         'waste' : ['Wastes (non-renewable)'],
+         'electricity' : ['Electricity'],
+}
+
+ktoe_to_twh = 0.011630
+
+eu28 = ['FR', 'DE', 'GB', 'IT', 'ES', 'PL', 'SE', 'NL', 'BE', 'FI',
+        'DK', 'PT', 'RO', 'AT', 'BG', 'EE', 'GR', 'LV', 'CZ',
+        'HU', 'IE', 'SK', 'LT', 'HR', 'LU', 'SI', 'CY', 'MT']
+
+jrc_names = {"GR" : "EL",
+             "GB" : "UK"}
+
+year = 2015
+summaries = {}
+
+#for some reason the Energy Balances list Other Industrial Sectors separately
+ois_subs = ['Mining and quarrying','Construction','Non-specified']
+
+
+#MtNH3/a
+ammonia = pd.read_csv(snakemake.input.ammonia_production,
+                      index_col=0)/1e3
+
+
+
+for ct in eu28:
+    print(ct)
+    filename = 'data/jrc-idees-2015/JRC-IDEES-2015_EnergyBalance_{}.xlsx'.format(jrc_names.get(ct,ct))
+
+    summary = pd.DataFrame(index=list(fuels.keys()) + ['other'])
+
+    for sector in sub_sectors:
+        if sector == 'Other Industrial Sectors':
+            subs = ois_subs
+        else:
+            subs = sub_sectors[sector]
+
+        for sub in subs:
+            df = pd.read_excel(filename,
+                               sheet_name=eb_sheet_name[sub],
+                               index_col=0)
+
+            s = df[year].astype(float)
+
+            for fuel in fuels:
+                summary.at[fuel,sub] = s[fuels[fuel]].sum()
+                summary.at['other',sub] = summary.at['all',sub] - summary.loc[summary.index^['all','other'],sub].sum()
+
+    summary['Other Industrial Sectors'] = summary[ois_subs].sum(axis=1)
+    summary.drop(columns=ois_subs,inplace=True)
+
+    summary.drop(index=['all'],inplace=True)
+
+    summary *= ktoe_to_twh
+
+    summary['Basic chemicals'] += summary['Basic chemicals feedstock']
+    summary.drop(columns=['Basic chemicals feedstock'], inplace=True)
+
+    summary['Ammonia'] = 0.
+    summary.at['gas','Ammonia'] = snakemake.config['industry']['MWh_CH4_per_tNH3_SMR']*ammonia[str(year)].get(ct,0.)
+    summary.at['electricity','Ammonia'] = snakemake.config['industry']['MWh_elec_per_tNH3_SMR']*ammonia[str(year)].get(ct,0.)
+    summary['Basic chemicals (without ammonia)'] = summary['Basic chemicals'] - summary['Ammonia']
+    summary.loc[summary['Basic chemicals (without ammonia)'] < 0, 'Basic chemicals (without ammonia)'] = 0.
+    summary.drop(columns=['Basic chemicals'], inplace=True)
+
+    summaries[ct] = summary
+
+final_summary = pd.concat(summaries,axis=1)
+
+# add in the non-EU28 based on their output (which is derived from their energy too)
+# output in MtMaterial/a
+output = pd.read_csv(snakemake.input.industrial_production_per_country,
+                     index_col=0)/1e3
+
+eu28_averages = final_summary.groupby(level=1,axis=1).sum().divide(output.loc[eu28].sum(),axis=1)
+
+non_eu28 = output.index^eu28
+
+for ct in non_eu28:
+    print(ct)
+    final_summary = pd.concat((final_summary,pd.concat({ct : eu28_averages.multiply(output.loc[ct],axis=1)},axis=1)),axis=1)
+
+
+final_summary.index.name = 'TWh/a'
+
+final_summary.to_csv(snakemake.output.industrial_energy_demand_per_country_today)

scripts/build_industrial_energy_demand_per_node.py

@@ -0,0 +1,33 @@
+
+import pandas as pd
+import numpy as np
+
+# import EU ratios df as csv
+industry_sector_ratios=pd.read_csv(snakemake.input.industry_sector_ratios,
+                                   index_col=0)
+
+#material demand per node and industry (kton/a)
+nodal_production = pd.read_csv(snakemake.input.industrial_production_per_node,
+                               index_col=0)
+
+#energy demand today to get current electricity
+nodal_today = pd.read_csv(snakemake.input.industrial_energy_demand_per_node_today,
+                          index_col=0)
+
+#final energy consumption per node and industry (TWh/a)
+nodal_df = nodal_production.dot(industry_sector_ratios.T)
+nodal_df*= 0.001 #GWh -> TWh (ktCO2 -> MtCO2)
+
+
+rename_sectors = {'elec':'electricity',
+                  'biomass':'solid biomass',
+                  'heat':'low-temperature heat'}
+
+nodal_df.rename(columns=rename_sectors,inplace=True)
+
+nodal_df["current electricity"] = nodal_today["electricity"]
+
+nodal_df.index.name = "TWh/a (MtCO2/a)"
+
+nodal_df.to_csv(snakemake.output.industrial_energy_demand_per_node,
+                float_format='%.2f')

scripts/build_industrial_energy_demand_per_node_today.py

@@ -0,0 +1,54 @@
+
+import pandas as pd
+import numpy as np
+
+def build_nodal_demand():
+
+    industrial_demand = pd.read_csv(snakemake.input.industrial_energy_demand_per_country_today,
+                                    header=[0,1],
+                                    index_col=0)
+
+    distribution_keys = pd.read_csv(snakemake.input.industrial_distribution_key,
+                                        index_col=0)
+    distribution_keys["country"] = distribution_keys.index.str[:2]
+
+    nodal_demand = pd.DataFrame(0.,
+                                index=distribution_keys.index,
+                                columns=industrial_demand.index,
+                                dtype=float)
+
+    #map JRC/our sectors to hotmaps sector, where mapping exist
+    sector_mapping = {'Electric arc' : 'Iron and steel',
+                      'Integrated steelworks' : 'Iron and steel',
+                      'DRI + Electric arc' : 'Iron and steel',
+                      'Ammonia' : 'Chemical industry',
+                      'Basic chemicals (without ammonia)' : 'Chemical industry',
+                      'Other chemicals' : 'Chemical industry',
+                      'Pharmaceutical products etc.' : 'Chemical industry',
+                      'Cement' : 'Cement',
+                      'Ceramics & other NMM' : 'Non-metallic mineral products',
+                      'Glass production' : 'Glass',
+                      'Pulp production' : 'Paper and printing',
+                      'Paper production' : 'Paper and printing',
+                      'Printing and media reproduction' : 'Paper and printing',
+                      'Alumina production' : 'Non-ferrous metals',
+                      'Aluminium - primary production' : 'Non-ferrous metals',
+                      'Aluminium - secondary production' : 'Non-ferrous metals',
+                      'Other non-ferrous metals' : 'Non-ferrous metals',
+    }
+
+    for c in distribution_keys.country.unique():
+        buses = distribution_keys.index[distribution_keys.country == c]
+        for sector in industrial_demand.columns.levels[1]:
+            distribution_key = distribution_keys.loc[buses,sector_mapping.get(sector,"population")]
+            demand = industrial_demand[c,sector]
+            outer = pd.DataFrame(np.outer(distribution_key,demand),index=distribution_key.index,columns=demand.index)
+            nodal_demand.loc[buses] += outer
+
+    nodal_demand.index.name = "TWh/a"
+
+    nodal_demand.to_csv(snakemake.output.industrial_energy_demand_per_node_today)
+
+if __name__ == "__main__":
+
+    build_nodal_demand()

scripts/build_industrial_production_per_country.py

@@ -1,26 +1,17 @@
 
-
-#%matplotlib inline
 import pandas as pd
 import numpy as np
 
 
+tj_to_ktoe = 0.0238845
+ktoe_to_twh = 0.01163
 
 jrc_base_dir = "data/jrc-idees-2015"
 eb_base_dir = "data/eurostat-energy_balances-may_2018_edition"
 
-
-
-tj_to_ktoe = 0.0238845
-
-ktoe_to_twh = 0.01163
-
-# import EU ratios df as csv
-df=pd.read_csv('resources/industry_sector_ratios.csv', sep=';', index_col=0)
-
-
-
-
+# year for which data is retrieved
+raw_year = 2015
+year = raw_year-2016
 
 sub_sheet_name_dict = { 'Iron and steel':'ISI',
                         'Chemicals Industry':'CHI',
@@ -36,20 +27,17 @@ sub_sheet_name_dict = { 'Iron and steel':'ISI',
 
 index = ['elec','biomass','methane','hydrogen','heat','naphtha','process emission','process emission from feedstock']
 
-countries_df = pd.DataFrame(columns=index) #data frame final energy consumption per country and source
-
-
 non_EU = ['NO', 'CH', 'ME', 'MK', 'RS', 'BA', 'AL']
 
-rename = {"GR" : "EL",
-          "GB" : "UK"}
+jrc_names = {"GR" : "EL",
+             "GB" : "UK"}
 
-eu28 = ['FR', 'DE', 'GB', 'IT', 'ES', 'PL', 'SE', 'NL', 'BE', 'FI', 'CZ',
-        'DK', 'PT', 'RO', 'AT', 'BG', 'EE', 'GR', 'LV',
-        'HU', 'IE', 'SK', 'LT', 'HR', 'LU', 'SI'] + ['CY','MT']
+eu28 = ['FR', 'DE', 'GB', 'IT', 'ES', 'PL', 'SE', 'NL', 'BE', 'FI',
+        'DK', 'PT', 'RO', 'AT', 'BG', 'EE', 'GR', 'LV', 'CZ',
+        'HU', 'IE', 'SK', 'LT', 'HR', 'LU', 'SI', 'CY', 'MT']
 
 
-countries = non_EU + [rename.get(eu,eu) for eu in eu28[:-2]]
+countries = non_EU + eu28
 
 
 sectors = ['Iron and steel','Chemicals Industry','Non-metallic mineral products',
@@ -69,6 +57,14 @@ sect2sub = {'Iron and steel':['Electric arc','Integrated steelworks'],
             'Wood and wood products' :['Wood and wood products'],
             'Other Industrial Sectors':['Other Industrial Sectors']}
 
+subsectors = [ss for s in sectors for ss in sect2sub[s]]
+
+#material demand per country and industry (kton/a)
+countries_demand = pd.DataFrame(index=countries,
+                                columns=subsectors,
+                                dtype=float)
+
+
 out_dic ={'Electric arc': 'Electric arc',
           'Integrated steelworks': 'Integrated steelworks',
           'Basic chemicals': 'Basic chemicals (kt ethylene eq.)',
@@ -117,10 +113,11 @@ dic_sec_summary = {'Iron and steel': 'Iron and steel',
                    'Other Industrial Sectors': ' Other Industrial Sectors'}
 
 #countries=['CH']
-dic_countries={'NO':'Norway', 'AL':'Albania', 'BA':'Bosnia and Herzegovina',
-              'MK':'FYR of Macedonia', 'GE':'Georgia', 'IS':'Iceland',
-              'KO':'Kosovo', 'MD':'Moldova', 'ME':'Montenegro', 'RS':'Serbia',
-               'UA':'Ukraine', 'TR':'Turkey', }
+eb_names={'NO':'Norway', 'AL':'Albania', 'BA':'Bosnia and Herzegovina',
+          'MK':'FYR of Macedonia', 'GE':'Georgia', 'IS':'Iceland',
+          'KO':'Kosovo', 'MD':'Moldova', 'ME':'Montenegro', 'RS':'Serbia',
+          'UA':'Ukraine', 'TR':'Turkey', }
+
 dic_sec ={'Iron and steel':'Iron & steel industry',
           'Chemicals Industry': 'Chemical and Petrochemical industry',
           'Non-metallic mineral products': 'Non-ferrous metal industry',
@@ -153,8 +150,8 @@ dic_Switzerland ={'Iron and steel': 7889.,
 
 dic_sec_position={}
 for country in countries:
-    countries_df.loc[country] = 0
-    print (country)
+    countries_demand.loc[country] = 0.
+    print(country)
     for sector in sectors:
         if country in non_EU:
             if country == 'CH':
@@ -162,14 +159,14 @@ for country in countries:
             else:
                 # estimate physical output
                 #energy consumption in the sector and country
-                excel_balances = pd.read_excel('{}/{}.XLSX'.format(eb_base_dir,dic_countries[country]),
+                excel_balances = pd.read_excel('{}/{}.XLSX'.format(eb_base_dir,eb_names[country]),
                                       sheet_name='2016', index_col=2,header=0, skiprows=1 ,squeeze=True)
                 e_country = excel_balances.loc[dic_sec[sector], 'Total all products']
 
             #energy consumption in the sector and EU28
             excel_sum_out = pd.read_excel('{}/JRC-IDEES-2015_Industry_EU28.xlsx'.format(jrc_base_dir),
                                   sheet_name='Ind_Summary', index_col=0,header=0,squeeze=True) # the summary sheet
-            s_sum_out = excel_sum_out.iloc[49:76,-1]
+            s_sum_out = excel_sum_out.iloc[49:76,year]
             e_EU28 = s_sum_out[dic_sec_summary[sector]]
 
             ratio_country_EU28=e_country/e_EU28
@@ -177,62 +174,45 @@ for country in countries:
             excel_out = pd.read_excel('{}/JRC-IDEES-2015_Industry_EU28.xlsx'.format(jrc_base_dir),
                                       sheet_name=sub_sheet_name_dict[sector],index_col=0,header=0,squeeze=True) # the summary sheet
 
-            s_out = excel_out.iloc[loc_dic[sector][0]:loc_dic[sector][1],-1]
+            s_out = excel_out.iloc[loc_dic[sector][0]:loc_dic[sector][1],year]
 
             for subsector in sect2sub[sector]:
-                output = ratio_country_EU28*s_out[out_dic[subsector]]
-
-                for ind in index:
-                    countries_df.loc[country, ind] += float(output*df.loc[ind, subsector]) # kton * MWh = GWh (# kton * tCO2 = ktCO2)
+                countries_demand.loc[country,subsector] = ratio_country_EU28*s_out[out_dic[subsector]]
 
         else:
 
             # read the input sheets
-            excel_out = pd.read_excel('{}/JRC-IDEES-2015_Industry_{}.xlsx'.format(jrc_base_dir,country), sheet_name=sub_sheet_name_dict[sector],index_col=0,header=0,squeeze=True) # the summary sheet
+            excel_out = pd.read_excel('{}/JRC-IDEES-2015_Industry_{}.xlsx'.format(jrc_base_dir,jrc_names.get(country,country)), sheet_name=sub_sheet_name_dict[sector],index_col=0,header=0,squeeze=True) # the summary sheet
 
-            s_out = excel_out.iloc[loc_dic[sector][0]:loc_dic[sector][1],-1]
+            s_out = excel_out.iloc[loc_dic[sector][0]:loc_dic[sector][1],year]
 
             for subsector in sect2sub[sector]:
-                output = s_out[out_dic[subsector]]
-                for ind in index:
-                    countries_df.loc[country, ind] += output*df.loc[ind, subsector] #kton * MWh = GWh (# kton * tCO2 = ktCO2)
-
-countries_df*= 0.001 #GWh -> TWh (ktCO2 -> MtCO2)
-
-# save current electricity consumption
-for country in countries:
-    if country in non_EU:
-        if country == 'CH':
-            countries_df.loc[country, 'current electricity']=dic_Switzerland['current electricity']*tj_to_ktoe*ktoe_to_twh
-        else:
-            excel_balances = pd.read_excel('{}/{}.XLSX'.format(eb_base_dir,dic_countries[country]),
-                                      sheet_name='2016', index_col=1,header=0, skiprows=1 ,squeeze=True)
-
-            countries_df.loc[country, 'current electricity'] = excel_balances.loc['Industry', 'Electricity']*ktoe_to_twh
-
-    else:
-
-        excel_out = pd.read_excel('{}/JRC-IDEES-2015_Industry_{}.xlsx'.format(jrc_base_dir,country),
-                                  sheet_name='Ind_Summary',index_col=0,header=0,squeeze=True) # the summary sheet
-
-        s_out = excel_out.iloc[27:48,-1]
-        countries_df.loc[country, 'current electricity'] = s_out['Electricity']*ktoe_to_twh
-        print(countries_df.loc[country, 'current electricity'])
+                countries_demand.loc[country,subsector] = s_out[out_dic[subsector]]
 
 
+#include ammonia demand separately and remove ammonia from basic chemicals
 
-# save df as csv
-for ind in index:
-    countries_df[ind]=countries_df[ind].astype('float')
-countries_df = countries_df.round(3)
+ammonia = pd.read_csv(snakemake.input.ammonia_production,
+                      index_col=0)
 
-countries_df.rename(index={value : key for key,value in rename.items()},inplace=True)
+there = ammonia.index.intersection(countries_demand.index)
+missing = countries_demand.index^there
 
-rename_sectors = {'elec':'electricity',
-                  'biomass':'solid biomass',
-                  'heat':'low-temperature heat'}
+print("Following countries have no ammonia demand:", missing)
 
-countries_df.rename(columns=rename_sectors,inplace=True)
+countries_demand.insert(2,"Ammonia",0.)
 
-countries_df.to_csv('resources/industrial_demand_per_country.csv',
-                    float_format='%.2f')
+countries_demand.loc[there,"Ammonia"] = ammonia.loc[there, str(raw_year)]
+
+countries_demand["Basic chemicals"] -= countries_demand["Ammonia"]
+
+#EE, HR and LT got negative demand through subtraction - poor data
+countries_demand.loc[countries_demand["Basic chemicals"] < 0.,"Basic chemicals"] = 0.
+
+countries_demand.rename(columns={"Basic chemicals" : "Basic chemicals (without ammonia)"},
+                        inplace=True)
+
+countries_demand.index.name = "kton/a"
+
+countries_demand.to_csv(snakemake.output.industrial_production_per_country,
+                        float_format='%.2f')

scripts/build_industrial_production_per_country_tomorrow.py

@@ -0,0 +1,29 @@
+
+import pandas as pd
+
+industrial_production = pd.read_csv(snakemake.input.industrial_production_per_country,
+                                    index_col=0)
+
+total_steel = industrial_production[["Integrated steelworks","Electric arc"]].sum(axis=1)
+
+fraction_primary_stays_primary = snakemake.config["industry"]["St_primary_fraction"]*total_steel.sum()/industrial_production["Integrated steelworks"].sum()
+
+industrial_production.insert(2, "DRI + Electric arc",
+                             fraction_primary_stays_primary*industrial_production["Integrated steelworks"])
+
+industrial_production["Electric arc"] = total_steel - industrial_production["DRI + Electric arc"]
+industrial_production["Integrated steelworks"] = 0.
+
+
+total_aluminium = industrial_production[["Aluminium - primary production","Aluminium - secondary production"]].sum(axis=1)
+
+fraction_primary_stays_primary = snakemake.config["industry"]["Al_primary_fraction"]*total_aluminium.sum()/industrial_production["Aluminium - primary production"].sum()
+
+industrial_production["Aluminium - primary production"] = fraction_primary_stays_primary*industrial_production["Aluminium - primary production"]
+industrial_production["Aluminium - secondary production"] = total_aluminium - industrial_production["Aluminium - primary production"]
+
+industrial_production["Basic chemicals (without ammonia)"] *= snakemake.config["industry"]['HVC_primary_fraction']
+
+
+industrial_production.to_csv(snakemake.output.industrial_production_per_country_tomorrow,
+                             float_format='%.2f')

scripts/build_industrial_production_per_node.py

@@ -0,0 +1,47 @@
+
+import pandas as pd
+
+def build_nodal_industrial_production():
+
+    industrial_production = pd.read_csv(snakemake.input.industrial_production_per_country_tomorrow,
+                                        index_col=0)
+
+    distribution_keys = pd.read_csv(snakemake.input.industrial_distribution_key,
+                                        index_col=0)
+    distribution_keys["country"] = distribution_keys.index.str[:2]
+
+    nodal_industrial_production = pd.DataFrame(index=distribution_keys.index,
+                                               columns=industrial_production.columns,
+                                               dtype=float)
+
+    #map JRC/our sectors to hotmaps sector, where mapping exist
+    sector_mapping = {'Electric arc' : 'Iron and steel',
+                      'Integrated steelworks' : 'Iron and steel',
+                      'DRI + Electric arc' : 'Iron and steel',
+                      'Ammonia' : 'Chemical industry',
+                      'Basic chemicals (without ammonia)' : 'Chemical industry',
+                      'Other chemicals' : 'Chemical industry',
+                      'Pharmaceutical products etc.' : 'Chemical industry',
+                      'Cement' : 'Cement',
+                      'Ceramics & other NMM' : 'Non-metallic mineral products',
+                      'Glass production' : 'Glass',
+                      'Pulp production' : 'Paper and printing',
+                      'Paper production' : 'Paper and printing',
+                      'Printing and media reproduction' : 'Paper and printing',
+                      'Alumina production' : 'Non-ferrous metals',
+                      'Aluminium - primary production' : 'Non-ferrous metals',
+                      'Aluminium - secondary production' : 'Non-ferrous metals',
+                      'Other non-ferrous metals' : 'Non-ferrous metals',
+    }
+
+    for c in distribution_keys.country.unique():
+        buses = distribution_keys.index[distribution_keys.country == c]
+        for sector in industrial_production.columns:
+            distribution_key = distribution_keys.loc[buses,sector_mapping.get(sector,"population")]
+            nodal_industrial_production.loc[buses,sector] = industrial_production.at[c,sector]*distribution_key
+
+    nodal_industrial_production.to_csv(snakemake.output.industrial_production_per_node)
+
+if __name__ == "__main__":
+
+    build_nodal_industrial_production()

scripts/build_industry_sector_ratios.py

@@ -5,11 +5,11 @@ import numpy as np
 
 base_dir = "data/jrc-idees-2015"
 
-# year for wich data is retrieved
-year = 2015
-year = year-2016
+# year for which data is retrieved
+raw_year = 2015
+year = raw_year-2016
 
-conv_factor=11.630 #ktoe/kton -> MWh/ton
+conv_factor=11.630 #GWh/ktoe OR MWh/toe
 
 country = 'EU28'
 
@@ -26,7 +26,7 @@ sub_sheet_name_dict = { 'Iron and steel':'ISI',
                         'Wood and wood products': 'WWP',
                         'Other Industrial Sectors': 'OIS'}
 
-index = ['elec','biomass','methane','hydrogen','heat','naphtha','process emission','process emission from feedstock']
+index = ['elec','coal','coke','biomass','methane','hydrogen','heat','naphtha','process emission','process emission from feedstock']
 
 df = pd.DataFrame(index=index)
 
@@ -58,7 +58,7 @@ excel_emi = pd.read_excel('{}/JRC-IDEES-2015_Industry_{}.xlsx'.format(base_dir,c
 
 sector = 'Electric arc'
 
-df[sector] = 0
+df[sector] = 0.
 
 # read the corresponding lines
 s_fec = excel_fec.iloc[51:57,year]
@@ -150,21 +150,122 @@ df.loc['process emission',sector] = s_emi['Process emissions']/s_out[sector] # u
 # final energy consumption per t
 df.loc[['elec','heat','methane'],sector] = df.loc[['elec','heat','methane'],sector]*conv_factor/s_out[sector] # unit MWh/t material
 
+### For primary route: DRI with H2 + EAF
 
-
-## Integrated steelworks is converted to Electric arc
-#
-#> Electric arc uses scrap metal and Direct Reduced Iron
-#
-#> We assume that when substituting Integrated Steelworks by Electric arc furnaces.
-#> 50% of Integrated steelworks is substituted by scrap metal + electric furnaces
-#> 50% of Integrated steelworks is substituted by Direct Reduce Iron (with Hydrogen) + electric furnaces
-
-df['Integrated steelworks']=df['Electric arc']
+df['DRI + Electric arc'] = df['Electric arc']
 
 # adding the Hydrogen necessary for the Direct Reduction of Iron. consumption 1.7 MWh H2 /ton steel
-#(0.5 because only half of the steel requires DRI, the rest is scrap  metal)
-df.loc['hydrogen', 'Integrated steelworks'] =snakemake.config["industry"]["H2_DRI"] * snakemake.config["industry"]["DRI_ratio"]
+df.loc['hydrogen', 'DRI + Electric arc'] = snakemake.config["industry"]["H2_DRI"]
+# add electricity consumption in DRI shaft (0.322 MWh/tSl)
+df.loc['elec', 'DRI + Electric arc'] += snakemake.config["industry"]["elec_DRI"]
+
+
+### Integrated steelworks (could be used in combination with CCS)
+### Assume existing fuels are kept, except for furnaces, refining, rolling, finishing
+### Ignore 'derived gases' since these are top gases from furnaces
+
+sector = 'Integrated steelworks'
+
+df['Integrated steelworks']= 0.
+
+# read the corresponding lines
+s_fec = excel_fec.iloc[3:9,year]
+
+assert s_fec.index[0] == sector
+
+# Lighting, Air compressors, Motor drives, Fans and pumps
+df.loc['elec',sector] += s_fec[['Lighting','Air compressors','Motor drives','Fans and pumps']].sum()
+
+# Low enthalpy heat
+df.loc['heat',sector] += s_fec['Low enthalpy heat']
+
+
+#### Steel: Sinter/Pellet making
+
+subsector = 'Steel: Sinter/Pellet making'
+
+# read the corresponding lines
+s_fec = excel_fec.iloc[13:19,year]
+
+s_ued = excel_ued.iloc[13:19,year]
+
+assert s_fec.index[0] == subsector
+
+df.loc['elec',sector] += s_fec['Electricity']
+df.loc['methane',sector] += s_fec['Natural gas (incl. biogas)']
+df.loc['methane',sector] += s_fec['Residual fuel oil']
+df.loc['coal',sector] += s_fec['Solids']
+
+
+#### Steel: Blast / Basic Oxygen Furnace
+
+subsector = 'Steel: Blast /Basic oxygen furnace'
+
+# read the corresponding lines
+s_fec = excel_fec.iloc[19:25,year]
+
+s_ued = excel_ued.iloc[19:25,year]
+
+assert s_fec.index[0] == subsector
+
+df.loc['methane',sector] += s_fec['Natural gas (incl. biogas)']
+df.loc['methane',sector] += s_fec['Residual fuel oil']
+df.loc['coal',sector] += s_fec['Solids']
+df.loc['coke',sector] += s_fec['Coke']
+
+
+#### Steel: Furnaces, Refining and Rolling
+#> assume fully electrified
+#
+#> other processes are scaled by the used energy
+
+subsector = 'Steel: Furnaces, Refining and Rolling'
+
+# read the corresponding lines
+s_fec = excel_fec.iloc[25:32,year]
+
+s_ued = excel_ued.iloc[25:32,year]
+
+assert s_fec.index[0] == subsector
+
+# this process can be electrified
+eff = s_ued['Steel: Furnaces, Refining and Rolling - Electric']/s_fec['Steel: Furnaces, Refining and Rolling - Electric']
+
+df.loc['elec',sector] += s_ued[subsector]/eff
+
+#### Steel: Products finishing
+#> assume fully electrified
+
+subsector = 'Steel: Products finishing'
+
+# read the corresponding lines
+s_fec = excel_fec.iloc[32:49,year]
+
+s_ued = excel_ued.iloc[32:49,year]
+
+assert s_fec.index[0] == subsector
+
+# this process can be electrified
+eff = s_ued['Steel: Products finishing - Electric']/s_fec['Steel: Products finishing - Electric']
+
+df.loc['elec',sector] += s_ued[subsector]/eff
+
+
+#### Process emissions (per physical output)
+
+s_emi = excel_emi.iloc[3:50,year]
+
+assert s_emi.index[0] == sector
+
+s_out = excel_out.iloc[6:7,year]
+
+assert sector in str(s_out.index)
+
+df.loc['process emission',sector] = s_emi['Process emissions']/s_out[sector] # unit tCO2/t material
+
+# final energy consumption per t
+df.loc[['elec','heat','methane','coke','coal'],sector] = df.loc[['elec','heat','methane','coke','coal'],sector]*conv_factor/s_out[sector] # unit MWh/t material
+
 
 
 ## Chemicals Industry
@@ -186,6 +287,8 @@ excel_emi = pd.read_excel('{}/JRC-IDEES-2015_Industry_{}.xlsx'.format(base_dir,c
 
 ### Basic chemicals
 
+## Ammonia is separated afterwards
+
 sector = 'Basic chemicals'
 
 df[sector] = 0
@@ -204,9 +307,8 @@ df.loc['heat',sector] += s_fec['Low enthalpy heat']
 #### Chemicals: Feedstock (energy used as raw material)
 #> There are Solids, Refinery gas, LPG, Diesel oil, Residual fuel oil, Other liquids, Naphtha, Natural gas for feedstock.
 #
-#>  Naphta represents 47%, methane 17%. LPG (18%) solids, refinery gas, diesel oil, residual fuel oils and other liquids are asimilated to Napthta
-#
-#> Following Lechtenbohmer 2016, the 85 TWh/year of methane for the ammonia industry are substited by hydrogen.
+#>  Naphta represents 47%, methane 17%. LPG (18%) solids, refinery gas, diesel oil, residual fuel oils and other liquids are asimilated to Naphtha
+
 
 subsector = 'Chemicals: Feedstock (energy used as raw material)'
 
@@ -219,19 +321,16 @@ assert s_fec.index[0] == subsector
 df.loc['naphtha',sector] += s_fec['Naphtha']
 
 # natural gas
-# 85 TWh/year of methane for the ammonia industry are substituted by hydrogen
-df.loc['methane',sector] += s_fec['Natural gas'] - snakemake.config["industry"]["H2_for_NH3"]/conv_factor
-df.loc['hydrogen',sector] += snakemake.config["industry"]["H2_for_NH3"]/conv_factor
-# 1 ktoe = 11630 MWh
+df.loc['methane',sector] += s_fec['Natural gas']
 
 # LPG and other feedstock materials are assimilated to naphtha since they will be produced trough Fischer-Tropsh process
 df.loc['naphtha',sector] += (s_fec['Solids'] + s_fec['Refinery gas'] + s_fec['LPG'] + s_fec['Diesel oil']
                             + s_fec['Residual fuel oil'] + s_fec['Other liquids'])
 
 #### Chemicals: Steam processing
-#> All the final energy consumption in the Stem processing is converted to biomass.
+#> All the final energy consumption in the Steam processing is converted to methane, since we need >1000 C temperatures here.
 #
-#> The current efficiency of biomass is assumed in the conversion.
+#> The current efficiency of methane is assumed in the conversion.
 
 subsector = 'Chemicals: Steam processing'
 
@@ -242,11 +341,11 @@ s_ued = excel_ued.iloc[22:33,year]
 
 assert s_fec.index[0] == subsector
 
-# efficiency of biomass
-eff_bio = s_ued['Biomass']/s_fec['Biomass']
+# efficiency of natural gas
+eff_ch4 = s_ued['Natural gas (incl. biogas)']/s_fec['Natural gas (incl. biogas)']
 
-# replace all fec by biomass
-df.loc['biomass',sector] += s_ued[subsector]/eff_bio
+# replace all fec by methane
+df.loc['methane',sector] += s_ued[subsector]/eff_ch4
 
 #### Chemicals: Furnaces
 #> assume fully electrified
@@ -298,10 +397,25 @@ s_emi = excel_emi.iloc[3:57,year]
 
 assert s_emi.index[0] == sector
 
+
+## Correct everything by subtracting 2015's ammonia demand and putting in ammonia demand for H2 and electricity separately
+
 s_out = excel_out.iloc[8:9,year]
 
 assert sector in str(s_out.index)
 
+ammonia = pd.read_csv(snakemake.input.ammonia_production,
+                      index_col=0)
+
+eu28 = ['FR', 'DE', 'GB', 'IT', 'ES', 'PL', 'SE', 'NL', 'BE', 'FI',
+        'DK', 'PT', 'RO', 'AT', 'BG', 'EE', 'GR', 'LV', 'CZ',
+        'HU', 'IE', 'SK', 'LT', 'HR', 'LU', 'SI', 'CY', 'MT']
+
+#ktNH3/a
+total_ammonia = ammonia.loc[ammonia.index.intersection(eu28),str(raw_year)].sum()
+
+s_out -= total_ammonia
+
 df.loc['process emission',sector] += (s_emi['Process emissions'] - snakemake.config["industry"]['petrochemical_process_emissions']*1e3 - snakemake.config["industry"]['NH3_process_emissions']*1e3)/s_out.values # unit tCO2/t material
 
 #these are emissions originating from feedstock, i.e. could be non-fossil origin
@@ -311,8 +425,24 @@ df.loc['process emission from feedstock',sector] += (snakemake.config["industry"
 # final energy consumption per t
 sources=['elec','biomass', 'methane', 'hydrogen', 'heat','naphtha']
 
-df.loc[sources,sector] = df.loc[sources,sector]*conv_factor/s_out.values# unit MWh/t material
-# 1 ktoe = 11630 MWh
+#convert from ktoe/a to GWh/a
+df.loc[sources,sector] *= conv_factor
+
+df.loc['methane',sector] -= total_ammonia*snakemake.config['industry']['MWh_CH4_per_tNH3_SMR']
+df.loc['elec',sector] -= total_ammonia*snakemake.config['industry']['MWh_elec_per_tNH3_SMR']
+
+df.loc[sources,sector] = df.loc[sources,sector]/s_out.values # unit MWh/t material
+
+df.rename(columns={sector : sector + " (without ammonia)"},
+          inplace=True)
+
+sector = 'Ammonia'
+
+df[sector] = 0.
+
+df.loc['hydrogen',sector] = snakemake.config['industry']['MWh_H2_per_tNH3_electrolysis']
+df.loc['elec',sector] = snakemake.config['industry']['MWh_elec_per_tNH3_electrolysis']
+
 
 ### Other chemicals
 
@@ -405,7 +535,7 @@ df.loc['process emission',sector] += s_emi['Process emissions']/s_out.values # u
 # final energy consumption per t
 sources=['elec','biomass', 'methane', 'hydrogen', 'heat','naphtha']
 
-df.loc[sources,sector] = df.loc[sources,sector]*11.630/s_out.values # unit MWh/t material
+df.loc[sources,sector] = df.loc[sources,sector]*conv_factor/s_out.values # unit MWh/t material
 # 1 ktoe = 11630 MWh
 
 ### Pharmaceutical products etc.
@@ -495,7 +625,7 @@ df.loc['process emission',sector] += 0 # unit tCO2/t material
 # final energy consumption per t
 sources=['elec','biomass', 'methane', 'hydrogen', 'heat', 'naphtha']
 
-df.loc[sources,sector] = df.loc[sources,sector]*11.630/s_out.values # unit MWh/t material
+df.loc[sources,sector] = df.loc[sources,sector]*conv_factor/s_out.values # unit MWh/t material
 # 1 ktoe = 11630 MWh
 
 ## Non-metallic mineral products
@@ -527,7 +657,7 @@ excel_emi = pd.read_excel('{}/JRC-IDEES-2015_Industry_{}.xlsx'.format(base_dir,c
 #
 #> Temperatures above 1400C are required for procesing limestone and sand into clinker.
 #
-#> Everything (except current electricity and heat consumption) is transformed into biomass
+#> Everything (except current electricity and heat consumption and existing biomass) is transformed into methane for high T.
 
 sector = 'Cement'
 
@@ -546,10 +676,11 @@ df.loc['elec',sector] += s_fec[['Lighting','Air compressors','Motor drives','Fan
 # Low enthalpy heat
 df.loc['heat',sector] += s_fec['Low enthalpy heat']
 
-# Efficiency changes due to biomass
-eff_bio=s_ued['Biomass']/s_fec['Biomass']
+# pre-processing: keep existing elec and biomass, rest to methane
+df.loc['elec', sector] += s_fec['Cement: Grinding, milling of raw material']
+df.loc['biomass', sector] += s_fec['Biomass']
+df.loc['methane', sector] += s_fec['Cement: Pre-heating and pre-calcination'] - s_fec['Biomass']
 
-df.loc['biomass', sector] += s_ued[['Cement: Grinding, milling of raw material', 'Cement: Pre-heating and pre-calcination']].sum()/eff_bio
 
 #### Cement: Clinker production (kilns)
 
@@ -562,10 +693,10 @@ s_ued = excel_ued.iloc[34:43,year]
 
 assert s_fec.index[0] == subsector
 
-# Efficiency changes due to biomass
-eff_bio=s_ued['Biomass']/s_fec['Biomass']
+df.loc['biomass', sector] += s_fec['Biomass']
+df.loc['methane', sector] += s_fec['Cement: Clinker production (kilns)'] - s_fec['Biomass']
+df.loc['elec', sector] += s_fec['Cement: Grinding, packaging']
 
-df.loc['biomass', sector] += s_ued[['Cement: Clinker production (kilns)', 'Cement: Grinding, packaging']].sum()/eff_bio
 
 #### Process-emission came from the calcination of limestone to chemically reactive calcium oxide (lime).
 #> Calcium carbonate -> lime + CO2
@@ -635,7 +766,7 @@ df.loc['process emission',sector] += s_emi['Process emissions']/s_out.values # u
 # final energy consumption per t
 sources=['elec','biomass', 'methane', 'hydrogen', 'heat','naphtha']
 
-df.loc[sources,sector] = df.loc[sources,sector]*11.630/s_out.values # unit MWh/t material
+df.loc[sources,sector] = df.loc[sources,sector]*conv_factor/s_out.values # unit MWh/t material
 # 1 ktoe = 11630 MWh
 
 ### Glass production
@@ -707,7 +838,7 @@ excel_out = pd.read_excel('{}/JRC-IDEES-2015_Industry_{}.xlsx'.format(base_dir,c
 #
 #> Includes three subcategories: (a) Wood preparation, grinding; (b) Pulping;  (c) Cleaning.
 #
-#> (b) Pulping is electrified. The efficiency is calculated from the pulping process that is already electric.
+#> (b) Pulping is either biomass or electric; left like this (dominated by biomass).
 #
 #> (a) Wood preparation, grinding and (c) Cleaning represent only 10% their current energy consumption is assumed to be electrified without any change in efficiency
 
@@ -729,14 +860,11 @@ df.loc['elec', sector] += s_fec[['Lighting','Air compressors','Motor drives','Fa
 df.loc['heat', sector] += s_fec['Low enthalpy heat']
 
 # Industry-specific
-df.loc['elec', sector] += s_fec[['Pulp: Wood preparation, grinding', 'Pulp: Cleaning']].sum()
+df.loc['elec', sector] += s_fec[['Pulp: Wood preparation, grinding', 'Pulp: Cleaning', 'Pulp: Pulping electric']].sum()
 
-# Efficiency changes due to electrification
-eff_elec=s_ued['Pulp: Pulping electric']/s_fec['Pulp: Pulping electric']
-df.loc['elec', sector] += s_ued['Pulp: Pulping thermal']/eff_elec
-
-# add electricity from process that is already electrified
-df.loc['elec', sector] += s_fec['Pulp: Pulping electric']
+# Efficiency changes due to biomass
+eff_bio=s_ued['Biomass']/s_fec['Biomass']
+df.loc['biomass', sector] += s_ued['Pulp: Pulping thermal']/eff_bio
 
 s_out = excel_out.iloc[8:9,year]
 
@@ -751,7 +879,7 @@ df.loc[sources,sector] = df.loc[sources,sector]*conv_factor/s_out['Pulp producti
 #
 #> Includes three subcategories: (a) Stock preparation; (b) Paper machine;  (c) Product finishing.
 #
-#> (b) Paper machine and (c) Product finishing are electrified. The efficiency is calculated from the pulping process that is already electric.
+#> (b) Paper machine and (c) Product finishing are left electric and thermal is moved to biomass. The efficiency is calculated from the pulping process that is already biomass.
 #
 #> (a) Stock preparation represents only 7% and its current energy consumption is assumed to be electrified without any change in efficiency.
 
@@ -775,16 +903,35 @@ df.loc['heat', sector] += s_fec['Low enthalpy heat']
 # Industry-specific
 df.loc['elec', sector] += s_fec['Paper: Stock preparation']
 
-# Efficiency changes due to electrification
-eff_elec=s_ued['Paper: Paper machine - Electricity']/s_fec['Paper: Paper machine - Electricity']
-df.loc['elec', sector] += s_ued['Paper: Paper machine - Steam use']/eff_elec
-
-eff_elec=s_ued['Paper: Product finishing - Electricity']/s_fec['Paper: Product finishing - Electricity']
-df.loc['elec', sector] += s_ued['Paper: Product finishing - Steam use']/eff_elec
-
 # add electricity from process that is already electrified
 df.loc['elec', sector] += s_fec['Paper: Paper machine - Electricity']
 
+# add electricity from process that is already electrified
+df.loc['elec', sector] += s_fec['Paper: Product finishing - Electricity']
+
+
+s_fec = excel_fec.iloc[53:64,year]
+
+s_ued = excel_ued.iloc[53:64,year]
+
+assert s_fec.index[0] == 'Paper: Paper machine - Steam use'
+
+# Efficiency changes due to biomass
+eff_bio=s_ued['Biomass']/s_fec['Biomass']
+df.loc['biomass', sector] += s_ued['Paper: Paper machine - Steam use']/eff_bio
+
+
+s_fec = excel_fec.iloc[66:77,year]
+
+s_ued = excel_ued.iloc[66:77,year]
+
+assert s_fec.index[0] == 'Paper: Product finishing - Steam use'
+
+# Efficiency changes due to biomass
+eff_bio=s_ued['Biomass']/s_fec['Biomass']
+df.loc['biomass', sector] += s_ued['Paper: Product finishing - Steam use']/eff_bio
+
+
 # read the corresponding lines
 s_out = excel_out.iloc[9:10,year]
 
@@ -878,8 +1025,8 @@ df.loc['elec', sector] += s_ued['Food: Drying']/eff_elec
 eff_elec=s_ued['Food: Electric cooling']/s_fec['Food: Electric cooling']
 df.loc['elec', sector] += s_ued['Food: Process cooling and refrigeration']/eff_elec
 
-# Steam processing is electrified without change in efficiency
-df.loc['elec', sector] += s_fec['Food: Steam processing']
+# Steam processing goes all to biomass without change in efficiency
+df.loc['biomass', sector] += s_fec['Food: Steam processing']
 
 # add electricity from process that is already electrified
 df.loc['elec', sector] += s_fec['Food: Electric machinery']
@@ -1052,9 +1199,6 @@ sources=['elec','biomass', 'methane', 'hydrogen', 'heat','naphtha']
 df.loc[sources,sector] = df.loc[sources,sector]*conv_factor/s_out['Aluminium - secondary production'] # unit MWh/t material
 # 1 ktoe = 11630 MWh
 
-# primary route is divided into 50% remains as today and 50% is transformed into secondary route
-df.loc[sources,'Aluminium - primary production'] = (1-snakemake.config["industry"]["Al_to_scrap"])*df.loc[sources,'Aluminium - primary production'] + snakemake.config["industry"]["Al_to_scrap"]*df.loc[sources,'Aluminium - secondary production']
-df.loc['process emission','Aluminium - primary production'] = (1-snakemake.config["industry"]["Al_to_scrap"])*df.loc['process emission','Aluminium - primary production']
 
 ### Other non-ferrous metals
 
@@ -1372,4 +1516,5 @@ sources=['elec','biomass', 'methane', 'hydrogen', 'heat','naphtha']
 df.loc[sources,sector] = df.loc[sources,sector]*conv_factor/s_out['Physical output (index)'] # unit MWh/t material
 
 
-df.to_csv('resources/industry_sector_ratios.csv', sep=';')
+df.index.name = "MWh/tMaterial"
+df.to_csv('resources/industry_sector_ratios.csv')

scripts/build_retro_cost.py

@@ -0,0 +1,484 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Jan 20 14:57:21 2020
+
+@author: bw0928
+
+*****************************************************************************
+This script calculates cost-energy_saving-curves for retrofitting
+for the EU-37 countries, based on the building stock data from hotmaps and
+the EU building stock database
+*****************************************************************************
+
+Structure:
+
+    (1) set assumptions and parameters
+    (2) read and prepare data
+    (3) calculate (€-dE-curves)
+    (4) save in csv
+
+*****************************************************************************
+"""
+
+import pandas as pd
+import matplotlib.pyplot as plt
+
+#%% ************ FUCNTIONS ***************************************************
+
+# windows ---------------------------------------------------------------
+def window_limit(l, window_assumptions):
+    """
+    define limit u value from which on window is retrofitted
+    """
+    m = (window_assumptions.diff()["u_limit"] /
+         window_assumptions.diff()["strength"]).dropna().iloc[0]
+    a = window_assumptions["u_limit"][0] - m * window_assumptions["strength"][0]
+    return m*l + a
+
+def u_retro_window(l, window_assumptions):
+    """
+    define retrofitting value depending on renovation strength
+    """
+    m = (window_assumptions.diff()["u_value"] /
+         window_assumptions.diff()["strength"]).dropna().iloc[0]
+    a = window_assumptions["u_value"][0] - m * window_assumptions["strength"][0]
+    return max(m*l + a, 0.8)
+
+def window_cost(u, cost_retro, window_assumptions):
+    """
+    get costs for new windows depending on u value
+
+    """
+    m = (window_assumptions.diff()["cost"] /
+         window_assumptions.diff()["u_value"]).dropna().iloc[0]
+    a = window_assumptions["cost"][0] - m * window_assumptions["u_value"][0]
+    window_cost = m*u + a
+    if annualise_cost:
+        window_cost = window_cost * interest_rate / (1 - (1 + interest_rate)
+                      ** -cost_retro.loc["Windows", "life_time"])
+    return window_cost
+
+#  functions for intermediate steps (~l, ~area)  -----------------------------
+def calculate_new_u(u_values, l, l_weight, k=0.035):
+    """
+    calculate U-values after building retrofitting, depending on the old
+    U-values (u_values).
+    They depend for the components Roof, Wall, Floor on the additional
+    insulation thickness (l), and the weighting for the corresponding
+    component (l_weight).
+    Windows are renovated to new ones with U-value (function: u_retro_window(l))
+    only if the are worse insulated than a certain limit value
+    (function: window_limit).
+
+    Parameters
+    ----------
+    u_values: pd.DataFrame
+    l: string
+    l_weight: pd.DataFrame (component, weight)
+    k: thermal conductivity
+
+    """
+    return u_values.apply(lambda x:
+                                 k / ((k / x.value) +
+                                      (float(l) * l_weight.loc[x.type][0]))
+                                 if x.type!="Windows"
+                             else (min(x.value, u_retro_window(float(l), window_assumptions))
+                                   if x.value>window_limit(float(l), window_assumptions) else x.value),
+                             axis=1)
+
+def calculate_dE(u_values, l, average_surface_w):
+    """
+    returns energy demand after retrofit (per unit of unrefurbished energy
+    demand) depending on current and retrofitted U-values, this energy demand
+    is weighted depending on the average surface of each component for the
+    building type of the assumend subsector
+    """
+    return u_values.apply(lambda x: x[l] / x.value *
+                                     average_surface_w.loc[x.assumed_subsector,
+                                                           x.type],
+                                     axis=1)
+
+
+def calculate_costs(u_values, l, cost_retro, average_surface):
+    """
+    returns costs for a given retrofitting strength weighted by the average
+    surface/volume ratio of the component for each building type
+    """
+    return u_values.apply(lambda x: (cost_retro.loc[x.type, "cost_var"] *
+                                     100 * float(l) * l_weight.loc[x.type][0]
+                                     + cost_retro.loc[x.type, "cost_fix"]) *
+                              average_surface.loc[x.assumed_subsector, x.type] /
+                              average_surface.loc[x.assumed_subsector, "surface"]
+                              if x.type!="Windows"
+                              else (window_cost(x[l], cost_retro, window_assumptions) *
+                                               average_surface.loc[x.assumed_subsector, x.type] /
+                                               average_surface.loc[x.assumed_subsector, "surface"]
+                                    if x.value>window_limit(float(l), window_assumptions) else 0),
+                             axis=1)
+
+
+# ---------------------------------------------------------------------------
+def prepare_building_stock_data():
+    """
+    reads building stock data and cleans up the format, returns
+    --------
+    u_values:          pd.DataFrame current U-values
+    average_surface:   pd.DataFrame (index= building type,
+                                     columns = [surface [m],height [m],
+                                                components area [m^2]])
+    average_surface_w: pd.DataFrame weighted share of the components per
+                       building type
+    area_tot:          heated floor area per country and sector [Mm²]
+    area:              heated floor area [Mm²] for country, sector, building
+                       type and period
+
+    """
+
+    building_data = pd.read_csv(snakemake.input.building_stock,
+                                usecols=list(range(13)))
+
+    # standardize data
+    building_data["type"].replace(
+                {'Covered area: heated  [Mm²]': 'Heated area [Mm²]',
+                 'Windows ': 'Windows',
+                 'Walls ': 'Walls',
+                 'Roof ': 'Roof',
+                 'Floor ': 'Floor'}, inplace=True)
+
+    building_data.country_code = building_data.country_code.str.upper()
+    building_data["subsector"].replace({'Hotels and Restaurants':
+                                        'Hotels and restaurants'}, inplace=True)
+    building_data["sector"].replace({'Residential sector': 'residential',
+                                     'Service sector': 'services'},
+                                    inplace=True)
+    # extract u-values
+    u_values = building_data[(building_data.feature.str.contains("U-values"))
+                             & (building_data.subsector != "Total")]
+
+    components = list(u_values.type.unique())
+
+    country_iso_dic = building_data.set_index("country")["country_code"].to_dict()
+
+    # add missing /rename countries
+    country_iso_dic.update({'Norway': 'NO',
+                            'Iceland': 'IS',
+                            'Montenegro': 'ME',
+                            'Serbia': 'RS',
+                            'Albania': 'AL',
+                            'United Kingdom': 'GB',
+                            'Bosnia and Herzegovina': 'BA',
+                            'Switzerland': 'CH'})
+
+    # heated floor area ----------------------------------------------------------
+    area = building_data[(building_data.type == 'Heated area [Mm²]') &
+                         (building_data.subsector != "Total")]
+    area_tot = area.groupby(["country", "sector"]).sum()
+    area = pd.concat([area, area.apply(lambda x: x.value /
+                                          area_tot.value.loc[(x.country, x.sector)],
+                                          axis=1).rename("weight")],axis=1)
+    area = area.groupby(['country', 'sector', 'subsector', 'bage']).sum()
+    area_tot.rename(index=country_iso_dic, inplace=True)
+
+    # add for some missing countries floor area from other data sources
+    area_missing = pd.read_csv(snakemake.input.floor_area_missing,
+                               index_col=[0, 1], usecols=[0, 1, 2, 3])
+    area_tot = area_tot.append(area_missing.unstack(level=-1).dropna().stack())
+    area_tot = area_tot.loc[~area_tot.index.duplicated(keep='last')]
+
+    # for still missing countries calculate floor area by population size
+    pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0)
+    pop_layout["ct"] = pop_layout.index.str[:2]
+    ct_total = pop_layout.total.groupby(pop_layout["ct"]).sum()
+
+    area_per_pop = area_tot.unstack().reindex(index=ct_total.index).apply(lambda x: x / ct_total[x.index])
+    missing_area_ct = ct_total.index.difference(area_tot.index.levels[0])
+    for ct in (missing_area_ct & ct_total.index):
+        averaged_data = pd.DataFrame(
+            area_per_pop.value.reindex(map_for_missings[ct]).mean()
+            * ct_total[ct],
+            columns=["value"])
+        index = pd.MultiIndex.from_product([[ct], averaged_data.index.to_list()])
+        averaged_data.index = index
+        averaged_data["estimated"] = 1
+        if ct not in area_tot.index.levels[0]:
+            area_tot = area_tot.append(averaged_data, sort=True)
+        else:
+            area_tot.loc[averaged_data.index] = averaged_data
+
+    # u_values for Poland are missing -> take them from eurostat -----------
+    u_values_PL = pd.read_csv(snakemake.input.u_values_PL)
+    area_PL = area.loc["Poland"].reset_index()
+    data_PL = pd.DataFrame(columns=u_values.columns, index=area_PL.index)
+    data_PL["country"] = "Poland"
+    data_PL["country_code"] = "PL"
+    # data from area
+    for col in ["sector", "subsector", "bage"]:
+        data_PL[col] = area_PL[col]
+    data_PL["btype"] = area_PL["subsector"]
+
+    data_PL_final = pd.DataFrame()
+    for component in components:
+        data_PL["type"] = component
+        data_PL["value"] = data_PL.apply(lambda x: u_values_PL[(u_values_PL.component==component)
+                                                               & (u_values_PL.sector==x["sector"])]
+                                         [x["bage"]].iloc[0], axis=1)
+        data_PL_final = data_PL_final.append(data_PL)
+
+    u_values = pd.concat([u_values,
+                          data_PL_final]).reset_index(drop=True)
+
+    # clean data ---------------------------------------------------------------
+    # smallest possible today u values for windows 0.8 (passive house standard)
+    # maybe the u values for the glass and not the whole window including frame
+    # for those types assumed in the dataset
+    u_values.loc[(u_values.type=="Windows") & (u_values.value<0.8), "value"] = 0.8
+    # drop unnecessary columns
+    u_values.drop(['topic', 'feature','detail', 'estimated','unit'],
+                  axis=1, inplace=True, errors="ignore")
+    #  only take in config.yaml specified countries into account
+    countries = ct_total.index
+    area_tot = area_tot.loc[countries]
+
+    # average component surface --------------------------------------------------
+    average_surface = (pd.read_csv(snakemake.input.average_surface,
+                                   nrows=3,
+                                   header=1,
+                                   index_col=0).rename(
+                                       {'Single/two family house': 'Single family- Terraced houses',
+                                        'Large apartment house': 'Multifamily houses',
+                                        'Apartment house': 'Appartment blocks'},
+                                    axis="index")).iloc[:, :6]
+    average_surface.columns = ["surface", "height", "Roof",
+                               "Walls", "Floor", "Windows"]
+    # get area share of component
+    average_surface_w = average_surface[components].apply(lambda x: x / x.sum(),
+                                                          axis=1)
+
+    return (u_values, average_surface,
+            average_surface_w, area_tot, area, country_iso_dic, countries)
+
+
+def prepare_cost_retro():
+    """
+    read and prepare retro costs, annualises them if annualise_cost=True
+    """
+    cost_retro = pd.read_csv(snakemake.input.cost_germany,
+                         nrows=4, index_col=0, usecols=[0, 1, 2, 3])
+    cost_retro.index = cost_retro.index.str.capitalize()
+    cost_retro.rename(index={"Window": "Windows", "Wall": "Walls"}, inplace=True)
+
+    window_assumptions = pd.read_csv(snakemake.input.window_assumptions,
+                                     skiprows=[1], usecols=[0,1,2,3], nrows=2)
+
+    if annualise_cost:
+        cost_retro[["cost_fix", "cost_var"]] = (cost_retro[["cost_fix", "cost_var"]]
+                                                .apply(lambda x: x * interest_rate /
+                                                       (1 - (1 + interest_rate)
+                                                        ** -cost_retro.loc[x.index,
+                                                                           "life_time"])))
+
+    return cost_retro, window_assumptions
+
+
+def calculate_cost_energy_curve(u_values, l_strength, l_weight, average_surface_w,
+                                average_surface, area, country_iso_dic,
+                                countries):
+    """
+    returns energy demand per unit of unrefurbished (dE) and cost for given
+    renovation strength (l_strength), data for missing countries is
+    approximated by countries with similar building stock (dict:map_for_missings)
+
+    parameter
+    -------- input -----------
+    u_values:          pd.DataFrame current U-values
+    l_strength:        list of strings (strength of retrofitting)
+    l_weight:          pd.DataFrame (component, weight)
+    average_surface:   pd.DataFrame (index= building type,
+                                     columns = [surface [m],height [m],
+                                                components area [m^2]])
+    average_surface_w: pd.DataFrame weighted share of the components per
+                       building type
+    area:              heated floor area [Mm²] for country, sector, building
+                       type and period
+    country_iso_dic:   dict (maps country name to 2-letter-iso-code)
+    countries:         pd.Index (specified countries in config.yaml)
+    -------- output ----------
+    res:               pd.DataFrame(index=pd.MultiIndex([country, sector]),
+                                    columns=pd.MultiIndex([(dE/cost), l_strength]))
+    """
+
+    energy_saved = u_values[['country', 'sector', 'subsector', 'bage', 'type']]
+    costs = u_values[['country', 'sector', 'subsector', 'bage', 'type']]
+
+    for l in l_strength:
+        u_values[l] = calculate_new_u(u_values, l, l_weight)
+        energy_saved = pd.concat([energy_saved,
+                                  calculate_dE(u_values, l, average_surface_w).rename(l)],
+                                 axis=1)
+        costs = pd.concat([costs,
+                           calculate_costs(u_values, l, cost_retro, average_surface).rename(l)],
+                          axis=1)
+
+    # energy and costs per country, sector, subsector and year
+    e_tot = energy_saved.groupby(['country', 'sector', 'subsector', 'bage']).sum()
+    cost_tot = costs.groupby(['country', 'sector', 'subsector', 'bage']).sum()
+
+    # weighting by area -> energy and costs per country and sector
+    # in case of missing data first concat
+    energy_saved = pd.concat([e_tot, area.weight], axis=1)
+    cost_res = pd.concat([cost_tot, area.weight], axis=1)
+    energy_saved = (energy_saved.apply(lambda x: x * x.weight, axis=1)
+                    .groupby(level=[0, 1]).sum())
+    cost_res = (cost_res.apply(lambda x: x * x.weight, axis=1)
+                .groupby(level=[0, 1]).sum())
+
+    res = pd.concat([energy_saved[l_strength], cost_res[l_strength]],
+                    axis=1, keys=["dE", "cost"])
+    res.rename(index=country_iso_dic, inplace=True)
+
+    res = res.reindex(index=countries, level=0)
+    # reset index because otherwise not considered countries still in index.levels[0]
+    res = res.reset_index().set_index(["country", "sector"])
+
+    # map missing countries
+    for ct in pd.Index(map_for_missings.keys()) & countries:
+        averaged_data = res.reindex(index=map_for_missings[ct], level=0).mean(level=1)
+        index = pd.MultiIndex.from_product([[ct], averaged_data.index.to_list()])
+        averaged_data.index = index
+        if ct not in res.index.levels[0]:
+            res = res.append(averaged_data)
+        else:
+            res.loc[averaged_data.index] = averaged_data
+
+    return res
+
+
+# %% **************** MAIN ************************************************
+if __name__ == "__main__":
+    #  for testing
+    if 'snakemake' not in globals():
+        import yaml
+        import os
+        from vresutils.snakemake import MockSnakemake
+        snakemake = MockSnakemake(
+            wildcards=dict(
+                network='elec',
+                simpl='',
+                clusters='37',
+                lv='1',
+                opts='Co2L-3H',
+                sector_opts="[Co2L0p0-168H-T-H-B-I]"),
+            input=dict(
+                building_stock="data/retro/data_building_stock.csv",
+                u_values_PL="data/retro/u_values_poland.csv",
+                tax_w="data/retro/electricity_taxes_eu.csv",
+                construction_index="data/retro/comparative_level_investment.csv",
+                average_surface="data/retro/average_surface_components.csv",
+                floor_area_missing="data/retro/floor_area_missing.csv",
+                clustered_pop_layout="resources/pop_layout_{network}_s{simpl}_{clusters}.csv",
+                cost_germany="data/retro/retro_cost_germany.csv",
+                window_assumptions="data/retro/window_assumptions.csv"),
+            output=dict(
+                retro_cost="resources/retro_cost_{network}_s{simpl}_{clusters}.csv",
+                floor_area="resources/floor_area_{network}_s{simpl}_{clusters}.csv")
+        )
+        with open('config.yaml', encoding='utf8') as f:
+            snakemake.config = yaml.safe_load(f)
+
+#  ******** (1) ASSUMPTIONS - PARAMETERS **********************************
+    retro_opts =  snakemake.config["sector"]["retrofitting"]
+    interest_rate = retro_opts["interest_rate"]
+    annualise_cost = retro_opts["annualise_cost"]  # annualise the investment costs
+    tax_weighting = retro_opts["tax_weighting"]   # weight costs depending on taxes in countries
+    construction_index = retro_opts["construction_index"]   # weight costs depending on labour/material costs per ct
+    # additional insulation thickness, determines maximum possible savings
+    l_strength = retro_opts["l_strength"]
+
+    k = 0.035   # thermal conductivity standard value
+    # strenght of relative retrofitting depending on the component
+    # determined by historical data of insulation thickness for retrofitting
+    l_weight = pd.DataFrame({"weight": [1.95, 1.48, 1.]},
+                            index=["Roof", "Walls", "Floor"])
+
+
+    # mapping missing countries by neighbours
+    map_for_missings = {
+        "AL": ["BG", "RO", "GR"],
+        "BA": ["HR"],
+        "RS": ["BG", "RO", "HR", "HU"],
+        "MK": ["BG", "GR"],
+        "ME": ["BA", "AL", "RS", "HR"],
+        "CH": ["SE", "DE"],
+        "NO": ["SE"],
+        }
+
+# %% ************ (2) DATA ***************************************************
+
+    # building stock data -----------------------------------------------------
+    (u_values, average_surface, average_surface_w,
+     area_tot, area, country_iso_dic, countries) = prepare_building_stock_data()
+
+    # costs for retrofitting -------------------------------------------------
+    cost_retro, window_assumptions = prepare_cost_retro()
+
+    # weightings of costs
+    if construction_index:
+        cost_w = pd.read_csv(snakemake.input.construction_index,
+                             skiprows=3, nrows=32, index_col=0)
+        # since German retrofitting costs are assumed
+        cost_w = ((cost_w["2018"] / cost_w.loc["Germany", "2018"])
+                  .rename(index=country_iso_dic))
+
+    if tax_weighting:
+        tax_w = pd.read_csv(snakemake.input.tax_w,
+                            header=12, nrows=39, index_col=0, usecols=[0, 4])
+        tax_w.rename(index=country_iso_dic, inplace=True)
+        tax_w = tax_w.apply(pd.to_numeric, errors='coerce').iloc[:, 0]
+        tax_w.dropna(inplace=True)
+
+# %% ********** (3) CALCULATE COST-ENERGY-CURVES ****************************
+
+    # for missing weighting of surfaces of building types assume MultiFamily houses
+    u_values["assumed_subsector"] = u_values.subsector
+    u_values.loc[~u_values.subsector.isin(average_surface.index),
+                 "assumed_subsector"] = 'Multifamily houses'
+
+    dE_and_cost =  calculate_cost_energy_curve(u_values, l_strength, l_weight,
+                                          average_surface_w, average_surface, area,
+                                          country_iso_dic, countries)
+    # reset index because otherwise not considered countries still in index.levels[0]
+    dE_and_cost = dE_and_cost.reset_index().set_index(["country", "sector"])
+
+    # weights costs after construction index
+    if construction_index:
+        for ct in list(map_for_missings.keys() - cost_w.index):
+            cost_w.loc[ct] = cost_w.reindex(index=map_for_missings[ct]).mean()
+        dE_and_cost.cost = dE_and_cost.cost.apply(lambda x: x * cost_w[x.index.levels[0]])
+
+    # weights cost depending on country taxes
+    if tax_weighting:
+        for ct in list(map_for_missings.keys() - tax_w.index):
+            tax_w[ct] = tax_w.reindex(index=map_for_missings[ct]).mean()
+        dE_and_cost.cost = dE_and_cost.cost.apply(lambda x: x * tax_w[x.index.levels[0]])
+
+    # get share of residential and sevice floor area
+    sec_w = (area_tot / area_tot.groupby(["country"]).sum())["value"]
+    # get the total cost-energy-savings weight by sector area
+    tot = dE_and_cost.apply(lambda col: col * sec_w, axis=0).groupby(level=0).sum()
+    tot.set_index(pd.MultiIndex.from_product([list(tot.index), ["tot"]]),
+                  inplace=True)
+    dE_and_cost = dE_and_cost.append(tot).unstack().stack()
+
+    summed_area = pd.DataFrame(area_tot.groupby("country").sum())
+    summed_area.set_index(pd.MultiIndex.from_product(
+                          [list(summed_area.index), ["tot"]]), inplace=True)
+    area_tot = area_tot.append(summed_area).unstack().stack()
+
+# %% ******* (4) SAVE   ************************************************
+
+    dE_and_cost.to_csv(snakemake.output.retro_cost)
+    area_tot.to_csv(snakemake.output.floor_area)
+
+
+

scripts/make_summary.py

@@ -28,10 +28,13 @@ opt_name = {"Store": "e", "Line" : "s", "Transformer" : "s"}
 override_component_attrs = pypsa.descriptors.Dict({k : v.copy() for k,v in pypsa.components.component_attrs.items()})
 override_component_attrs["Link"].loc["bus2"] = ["string",np.nan,np.nan,"2nd bus","Input (optional)"]
 override_component_attrs["Link"].loc["bus3"] = ["string",np.nan,np.nan,"3rd bus","Input (optional)"]
+override_component_attrs["Link"].loc["bus4"] = ["string",np.nan,np.nan,"4th bus","Input (optional)"]
 override_component_attrs["Link"].loc["efficiency2"] = ["static or series","per unit",1.,"2nd bus efficiency","Input (optional)"]
 override_component_attrs["Link"].loc["efficiency3"] = ["static or series","per unit",1.,"3rd bus efficiency","Input (optional)"]
+override_component_attrs["Link"].loc["efficiency4"] = ["static or series","per unit",1.,"4th bus efficiency","Input (optional)"]
 override_component_attrs["Link"].loc["p2"] = ["series","MW",0.,"2nd bus output","Output"]
 override_component_attrs["Link"].loc["p3"] = ["series","MW",0.,"3rd bus output","Output"]
+override_component_attrs["Link"].loc["p4"] = ["series","MW",0.,"4th bus output","Output"]
 override_component_attrs["StorageUnit"].loc["p_dispatch"] = ["series","MW",0.,"Storage discharging.","Output"]
 override_component_attrs["StorageUnit"].loc["p_store"] = ["series","MW",0.,"Storage charging.","Output"]
 
@@ -186,14 +189,6 @@ def calculate_costs(n,label,costs):
 
         costs.loc[marginal_costs_grouped.index,label] = marginal_costs_grouped
 
-    #add back in costs of links if there is a line volume limit
-    if label[1] != "opt":
-        costs.loc[("links-added","capital","transmission lines"),label] = ((costs_db.at['HVDC overhead', 'fixed']*n.links.length + costs_db.at['HVDC inverter pair', 'fixed'])*n.links.p_nom_opt)[n.links.carrier == "DC"].sum()
-        costs.loc[("lines-added","capital","transmission lines"),label] = costs_db.at["HVAC overhead", "fixed"]*(n.lines.length*n.lines.s_nom_opt).sum()
-    else:
-        costs.loc[("links-added","capital","transmission lines"),label] = (costs_db.at['HVDC inverter pair', 'fixed']*n.links.p_nom_opt)[n.links.carrier == "DC"].sum()
-
-
     #add back in all hydro
     #costs.loc[("storage_units","capital","hydro"),label] = (0.01)*2e6*n.storage_units.loc[n.storage_units.group=="hydro","p_nom"].sum()
     #costs.loc[("storage_units","capital","PHS"),label] = (0.01)*2e6*n.storage_units.loc[n.storage_units.group=="PHS","p_nom"].sum()
@@ -530,7 +525,7 @@ outputs = ["nodal_costs",
 
 def make_summaries(networks_dict):
 
-    columns = pd.MultiIndex.from_tuples(networks_dict.keys(),names=["cluster","lv","opt", "co2_budget_name","planning_horizon"])
+    columns = pd.MultiIndex.from_tuples(networks_dict.keys(),names=["cluster","lv","opt","planning_horizon"])
 
     df = {}
 
@@ -565,8 +560,8 @@ if __name__ == "__main__":
         from vresutils import Dict
         import yaml
         snakemake = Dict()
-        with open('config.yaml') as f:
-            snakemake.config = yaml.load(f)
+        with open('config.yaml', encoding='utf8') as f:
+            snakemake.config = yaml.safe_load(f)
 
         #overwrite some options
         snakemake.config["run"] = "test"
@@ -579,21 +574,19 @@ if __name__ == "__main__":
         for item in outputs:
             snakemake.output[item] = snakemake.config['summary_dir'] + '/{name}/csvs/{item}.csv'.format(name=snakemake.config['run'],item=item)
 
-    networks_dict = {(cluster,lv,opt+sector_opt, co2_budget_name, planning_horizon) :
-                     snakemake.config['results_dir'] + snakemake.config['run'] + '/postnetworks/elec_s{simpl}_{cluster}_lv{lv}_{opt}_{sector_opt}_{co2_budget_name}_{planning_horizon}.nc'\
+    networks_dict = {(cluster, lv, opt+sector_opt, planning_horizon) :
+                     snakemake.config['results_dir'] + snakemake.config['run'] + '/postnetworks/elec_s{simpl}_{cluster}_lv{lv}_{opt}_{sector_opt}_{planning_horizon}.nc'\
                      .format(simpl=simpl,
                              cluster=cluster,
                              opt=opt,
                              lv=lv,
                              sector_opt=sector_opt,
-                             co2_budget_name=co2_budget_name,
                              planning_horizon=planning_horizon)\
                      for simpl in snakemake.config['scenario']['simpl'] \
                      for cluster in snakemake.config['scenario']['clusters'] \
                      for opt in snakemake.config['scenario']['opts'] \
                      for sector_opt in snakemake.config['scenario']['sector_opts'] \
                      for lv in snakemake.config['scenario']['lv'] \
-                     for co2_budget_name in snakemake.config['scenario']['co2_budget_name'] \
                      for planning_horizon in snakemake.config['scenario']['planning_horizons']}
 
     print(networks_dict)
@@ -602,7 +595,8 @@ if __name__ == "__main__":
     costs_db = prepare_costs(snakemake.input.costs,
                              snakemake.config['costs']['USD2013_to_EUR2013'],
                              snakemake.config['costs']['discountrate'],
-                             Nyears)
+                             Nyears,
+                             snakemake.config['costs']['lifetime'])
 
     df = make_summaries(networks_dict)
 

scripts/plot_network.py

@@ -253,9 +253,7 @@ def plot_h2_map(network):
 
     elec = n.links.index[n.links.carrier == "H2 Electrolysis"]
 
-    bus_sizes = pd.Series(0., index=n.buses.index)
-    bus_sizes.loc[elec.str.replace(" H2 Electrolysis", "")] = \
-                        n.links.loc[elec, "p_nom_opt"].values / bus_size_factor
+    bus_sizes = n.links.loc[elec,"p_nom_opt"].groupby(n.links.loc[elec,"bus0"]).sum() / bus_size_factor
 
     # make a fake MultiIndex so that area is correct for legend
     bus_sizes.index = pd.MultiIndex.from_product(
@@ -326,7 +324,7 @@ def plot_map_without(network):
     fig.set_size_inches(7, 6)
 
     # PDF has minimum width, so set these to zero
-    line_lower_threshold = 0.
+    line_lower_threshold = 200.
     line_upper_threshold = 1e4
     linewidth_factor = 2e3
     ac_color = "gray"
@@ -345,19 +343,19 @@ def plot_map_without(network):
         line_widths = n.lines.s_nom_min
         link_widths = n.links.p_nom_min
 
-    line_widths[line_widths < line_upper_threshold] = 0.
-    link_widths[link_widths < line_upper_threshold] = 0.
+    line_widths[line_widths < line_lower_threshold] = 0.
+    link_widths[link_widths < line_lower_threshold] = 0.
 
     line_widths[line_widths > line_upper_threshold] = line_upper_threshold
     link_widths[link_widths > line_upper_threshold] = line_upper_threshold
 
-    n.plot(bus_sizes=10,
-           bus_colors="k",
+    n.plot(bus_colors="k",
            line_colors=ac_color,
            link_colors=dc_color,
            line_widths=line_widths / linewidth_factor,
            link_widths=link_widths / linewidth_factor,
-           ax=ax,  boundaries=(-10, 30, 34, 70))
+           ax=ax,  boundaries=(-10, 30, 34, 70),
+           color_geomap={'ocean': 'lightblue', 'land': "palegoldenrod"})
 
     handles = []
     labels = []

scripts/plot_summary.py

@@ -22,7 +22,7 @@ def rename_techs(label):
                                "retrofitting" : "building retrofitting",
                                "H2" : "hydrogen storage",
                                "battery" : "battery storage",
-                               "CCS" : "CCS"}
+                               "CC" : "CC"}
 
     rename = {"solar" : "solar PV",
               "Sabatier" : "methanation",
@@ -188,11 +188,11 @@ def plot_balances():
         df = df/1e6
 
         #remove trailing link ports
-        df.index = [i[:-1] if i[-1:] in ["0","1","2","3"] else i for i in df.index]
+        df.index = [i[:-1] if ((i != "co2") and (i[-1:] in ["0","1","2","3"])) else i for i in df.index]
 
         df = df.groupby(df.index.map(rename_techs)).sum()
 
-        to_drop = df.index[df.abs().max(axis=1) < snakemake.config['plotting']['energy_threshold']]
+        to_drop = df.index[df.abs().max(axis=1) < snakemake.config['plotting']['energy_threshold']/10]
 
         print("dropping")
 
@@ -245,8 +245,8 @@ if __name__ == "__main__":
         from vresutils import Dict
         import yaml
         snakemake = Dict()
-        with open('config.yaml') as f:
-            snakemake.config = yaml.load(f)
+        with open('config.yaml', encoding='utf8') as f:
+            snakemake.config = yaml.safe_load(f)
         snakemake.input = Dict()
         snakemake.output = Dict()
 
@@ -256,9 +256,10 @@ if __name__ == "__main__":
         snakemake.input["balances"] = snakemake.config['summary_dir'] + '/test/csvs/supply_energy.csv'
         snakemake.output["balances"] = snakemake.config['summary_dir'] + '/test/graphs/balances-energy.csv'
 
-    n_header = 5 
+    n_header = 4
+
     plot_costs()
 
     plot_energy()
 
-    #plot_balances()
+    plot_balances()

scripts/solve_network.py

@@ -28,10 +28,13 @@ from vresutils.benchmark import memory_logger
 override_component_attrs = pypsa.descriptors.Dict({k : v.copy() for k,v in pypsa.components.component_attrs.items()})
 override_component_attrs["Link"].loc["bus2"] = ["string",np.nan,np.nan,"2nd bus","Input (optional)"]
 override_component_attrs["Link"].loc["bus3"] = ["string",np.nan,np.nan,"3rd bus","Input (optional)"]
+override_component_attrs["Link"].loc["bus4"] = ["string",np.nan,np.nan,"4th bus","Input (optional)"]
 override_component_attrs["Link"].loc["efficiency2"] = ["static or series","per unit",1.,"2nd bus efficiency","Input (optional)"]
 override_component_attrs["Link"].loc["efficiency3"] = ["static or series","per unit",1.,"3rd bus efficiency","Input (optional)"]
+override_component_attrs["Link"].loc["efficiency4"] = ["static or series","per unit",1.,"4th bus efficiency","Input (optional)"]
 override_component_attrs["Link"].loc["p2"] = ["series","MW",0.,"2nd bus output","Output"]
 override_component_attrs["Link"].loc["p3"] = ["series","MW",0.,"3rd bus output","Output"]
+override_component_attrs["Link"].loc["p4"] = ["series","MW",0.,"4th bus output","Output"]
 
 
 
@@ -351,7 +354,7 @@ def solve_network(n, config=None, solver_log=None, opts=None):
         #     fn = os.path.basename(snakemake.output[0])
         #     n.export_to_netcdf('/home/vres/data/jonas/playground/pypsa-eur/' + fn)
 
-    status, termination_condition = run_lopf(n, fix_ext_lines=True)
+    status, termination_condition = run_lopf(n, allow_warning_status=True, fix_ext_lines=True)
 
     # Drop zero lines from network
     # zero_lines_i = n.lines.index[(n.lines.s_nom_opt == 0.) & n.lines.s_nom_extendable]
@@ -378,8 +381,8 @@ if __name__ == "__main__":
                      python="logs/{network}_s{simpl}_{clusters}_lv{lv}_{sector_opts}_{co2_budget_name}_{planning_horizons}_python-test.log")
         )
         import yaml
-        with open('config.yaml') as f:
-            snakemake.config = yaml.load(f)
+        with open('config.yaml', encoding='utf8') as f:
+            snakemake.config = yaml.safe_load(f)
     tmpdir = snakemake.config['solving'].get('tmpdir')
     if tmpdir is not None:
         patch_pyomo_tmpdir(tmpdir)