Changes to make solving and post-processing work

Snakefile

@@ -16,6 +16,14 @@ subworkflow pypsaeur:
     snakefile: "../pypsa-eur/Snakefile"
     configfile: "../pypsa-eur/config.yaml"
 
+rule all:
+     input: config['summary_dir'] + '/' + config['run'] + '/graphs/costs.pdf'
+
+
+rule solve_all_elec_networks:
+    input:
+        expand(config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}.nc",
+               **config['scenario'])
 
 rule test_script:
     input:
@@ -177,3 +185,63 @@ rule prepare_sector_network:
     resources: mem=1000
     benchmark: "benchmarks/prepare_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}"
     script: "scripts/prepare_sector_network.py"
+
+
+rule solve_network:
+    input: config['results_dir'] +  config['run'] + "/prenetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}.nc"
+    output: config['results_dir'] +  config['run'] + "/postnetworks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}.nc"
+    shadow: "shallow"
+    log:
+        solver="logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_solver.log",
+        python="logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_python.log",
+        memory="logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_memory.log"
+    benchmark: "benchmarks/solve_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}"
+    threads: 4
+    resources: mem=95000 #memory; 40 GB enough for 45+B+I; 100 GB based on RESI usage for 128
+    # group: "solve" # with group, threads is ignored https://bitbucket.org/snakemake/snakemake/issues/971/group-job-description-does-not-contain
+    script: "scripts/solve_network.py"
+
+rule plot_network:
+    input:
+        network="results/networks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}.nc",
+        supply_regions='data/supply_regions/supply_regions.shp',
+        resarea=lambda w: config['data']['resarea'][w.resarea],
+        costs='data/costs.csv'
+    output:
+        only_map="results/plots/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{attr}.pdf",
+        ext="results/plots/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_{attr}_ext.pdf"
+    script: "scripts/plot_network.py"
+
+rule make_summary:
+    input:
+        expand(config['results_dir'] + config['run'] + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}.nc",
+               **config['scenario'])
+        #heat_demand_name='data/heating/daily_heat_demand.h5'
+    output:
+        costs=config['summary_dir'] + '/' + config['run'] + '/csvs/costs.csv',
+        curtailment=config['summary_dir'] + '/' + config['run'] + '/csvs/curtailment.csv',
+        energy=config['summary_dir'] + '/' + config['run'] + '/csvs/energy.csv',
+        supply=config['summary_dir'] + '/' + config['run'] + '/csvs/supply.csv',
+        supply_energy=config['summary_dir'] + '/' + config['run'] + '/csvs/supply_energy.csv',
+        prices=config['summary_dir'] + '/' + config['run'] + '/csvs/prices.csv',
+        weighted_prices=config['summary_dir'] + '/' + config['run'] + '/csvs/weighted_prices.csv',
+        market_values=config['summary_dir'] + '/' + config['run'] + '/csvs/market_values.csv',
+        price_statistics=config['summary_dir'] + '/' + config['run'] + '/csvs/price_statistics.csv',
+        metrics=config['summary_dir'] + '/' + config['run'] + '/csvs/metrics.csv'
+    threads: 2
+    resources: mem_mb=10000
+    script:
+        'scripts/make_summary.py'
+
+
+rule plot_summary:
+    input:
+        costs=config['summary_dir'] + '/' + config['run'] + '/csvs/costs.csv',
+        energy=config['summary_dir'] + '/' + config['run'] + '/csvs/energy.csv'
+    output:
+        costs=config['summary_dir'] + '/' + config['run'] + '/graphs/costs.pdf',
+        energy=config['summary_dir'] + '/' + config['run'] + '/graphs/energy.pdf'
+    threads: 2
+    resources: mem_mb=10000
+    script:
+        'scripts/plot_summary.py'

config.yaml

@@ -10,7 +10,7 @@ scenario:
   lv: [1.0,1.25]#[1.0, 1.125, 1.25, 1.5, 2.0, opt]# or opt
   clusters: [128] #[90, 128, 181] #[45, 64, 90, 128, 181, 256] #, 362] # (2**np.r_[5.5:9:.5]).astype(int) minimum is 37
   opts: ['']  #for pypsa-eur
-  sector_opts: [Co2L0-3H-T-H-B-I,Co2L0-3H-T-H-B-I-onwind0,Co2L0-3H-T-H-B-I-onwind0p1,Co2L0p1-3H-T-H-B-I,Co2L0p1-3H-T-H-B-I-onwind0,Co2L0p1-3H-T-H-B-I-onwind0p1]#,Co2L0p05-3H-T-H-B-I,Co2L0p10-3H-T-H-B-I,Co2L0p20-3H-T-H-B-I,Co2L0p30-3H-T-H-B-I,Co2L0p50-3H-T-H-B-I]#[Co2L-3H-T-H,Co2L0p10-3H-T-H,Co2L0-3H-T-H,Co2L0p20-3H-T-H]   #Co2L-3H-T-H,Co2L0p10-3H-T-H,Co2L0p20-3H-T-HCo2L-3H-T-H,Co2L0p10-3H-T-H,Co2L0p30-3H-T-H,Co2L0p50-3H-T-H] #Co2L-3H,Co2L-3H-T,, LC-FL, LC-T, Ep-T, Co2L-T]
+  sector_opts: [Co2L0-3H-T-H-B-I,Co2L0-3H-T-H-B-I-onwind0,Co2L0p1-3H-T-H-B-I]#,Co2L0p05-3H-T-H-B-I,Co2L0p10-3H-T-H-B-I,Co2L0p20-3H-T-H-B-I,Co2L0p30-3H-T-H-B-I,Co2L0p50-3H-T-H-B-I]#[Co2L-3H-T-H,Co2L0p10-3H-T-H,Co2L0-3H-T-H,Co2L0p20-3H-T-H]   #Co2L-3H-T-H,Co2L0p10-3H-T-H,Co2L0p20-3H-T-HCo2L-3H-T-H,Co2L0p10-3H-T-H,Co2L0p30-3H-T-H,Co2L0p50-3H-T-H] #Co2L-3H,Co2L-3H-T,, LC-FL, LC-T, Ep-T, Co2L-T]
   # Co2L will give default (5%); Co2L0p25 will give 25% CO2 emissions; Co2Lm0p05 will give 5% negative emissions
 
 
@@ -20,6 +20,20 @@ snapshots:
   end: "2014-01-01"
   closed: 'left' # end is not inclusive
 
+
+electricity:
+  voltages: [220., 300., 380.]
+
+  extendable_carriers:
+    Generator: [] # [OCGT]
+    StorageUnit: [] # [battery, H2, CAES]
+
+  max_hours:
+    battery: 6
+    H2: 168
+
+  conventional_carriers: [] # nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]
+
 atlite:
   nprocesses: 4
   cutout_dir: '../pypsa-eur/cutouts'
@@ -155,3 +169,159 @@ costs:
 
   emission_prices: # only used with the option Ep (emission prices)
     co2: 0.
+
+
+
+solving:
+  options:
+    formulation: kirchhoff
+    clip_p_max_pu: 1.e-2
+    load_shedding: false
+    noisy_costs: true
+
+    min_iterations: 1
+    max_iterations: 1
+    # max_iterations: 1
+    # nhours: 1
+
+  solver:
+    name: gurobi
+    threads: 4
+    method: 2 # barrier
+    crossover: 0
+    BarConvTol: 1.e-5
+    FeasibilityTol: 1.e-6
+  # solver:
+  #   name: cplex
+  #   threads: 4
+  #   lpmethod: 4 # barrier
+  #   solutiontype: 2 # non basic solution, ie no crossover
+  #   barrier.convergetol: 1.e-5
+  #   feasopt.tolerance: 1.e-6
+
+plotting:
+  map:
+    figsize: [7, 7]
+    boundaries: [-10.2, 29, 35,  72]
+    p_nom:
+      bus_size_factor: 5.e+4
+      linewidth_factor: 3.e+3 # 1.e+3  #3.e+3
+
+  costs_max: 1200
+  costs_threshold: 1
+
+
+  energy_max: 15000.
+  energy_min: -15000.
+  energy_threshold: 50.
+
+
+  vre_techs: ["onwind", "offwind", "solar", "ror"]
+  conv_techs: ["OCGT", "CCGT", "Nuclear", "Coal"]
+  storage_techs: ["hydro+PHS", "battery", "H2"]
+  # store_techs: ["Li ion", "water tanks"]
+  load_carriers: ["AC load"] #, "heat load", "Li ion load"]
+  AC_carriers: ["AC line", "AC transformer"]
+  link_carriers: ["DC line", "Converter AC-DC"]
+  heat_links: ["heat pump", "resistive heater", "CHP heat", "CHP electric",
+               "gas boiler", "central heat pump", "central resistive heater", "central CHP heat",
+               "central CHP electric", "central gas boiler"]
+  heat_generators: ["gas boiler", "central gas boiler", "solar thermal collector", "central solar thermal collector"]
+  tech_colors:
+    "onwind" : "b"
+    "onshore wind" : "b"
+    'offwind' : "c"
+    'offshore wind' : "c"
+    "hydro" : "#3B5323"
+    "hydro reservoir" : "#3B5323"
+    "ror" : "#78AB46"
+    "run of river" : "#78AB46"
+    'hydroelectricity' : '#006400'
+    'solar' : "y"
+    'solar PV' : "y"
+    'solar thermal' : 'coral'
+    "OCGT" : "wheat"
+    "OCGT marginal" : "sandybrown"
+    "OCGT-heat" : "orange"
+    "gas boiler" : "orange"
+    "gas boilers" : "orange"
+    "gas boiler marginal" : "orange"
+    "gas-to-power/heat" : "orange"
+    "gas" : "brown"
+    "natural gas" : "brown"
+    "lines" : "k"
+    "transmission lines" : "k"
+    "H2" : "m"
+    "hydrogen storage" : "m"
+    "battery" : "slategray"
+    "battery storage" : "slategray"
+    "Nuclear" : "r"
+    "Nuclear marginal" : "r"
+    "Coal" : "k"
+    "Coal marginal" : "k"
+    "Lignite" : "grey"
+    "Lignite marginal" : "grey"
+    "CCGT" : "orange"
+    "CCGT marginal" : "orange"
+    "heat pumps" : "#76EE00"
+    "heat pump" : "#76EE00"
+    "air heat pump" : "#76EE00"
+    "ground heat pump" : "#40AA00"
+    "power-to-heat" : "#40AA00"
+    "resistive heater" : "pink"
+    "Sabatier" : "#FF1493"
+    "methanation" : "#FF1493"
+    "power-to-gas" : "#FF1493"
+    "power-to-liquid" : "#FFAAE9"
+    "helmeth" : "#7D0552"
+    "helmeth" : "#7D0552"
+    "DAC" : "#E74C3C"
+    "co2 stored" : "#123456"
+    "CO2 sequestration" : "#123456"
+    "co2" : "#123456"
+    "Fischer-Tropsch" : "#44DD33"
+    "nuclear" : "#303030"
+    "water tanks" : "#BBBBBB"
+    "hot water storage" : "#BBBBBB"
+    "hot water charging" : "#BBBBBB"
+    "hot water discharging" : "#999999"
+    "CHP" : "r"
+    "CHP heat" : "r"
+    "CHP electric" : "r"
+    "PHS" : "g"
+    "Ambient" : "k"
+    "Electric load" : "b"
+    "Heat load" : "r"
+    "Transport load" : "grey"
+    "heat" : "darkred"
+    "heat demand" : "darkred"
+    "electric demand" : "k"
+    "Li ion" : "grey"
+    "district heating" : "#CC4E5C"
+    "retrofitting" : "purple"
+    "building retrofitting" : "purple"
+    "BEV charger" : "grey"
+    "V2G" : "grey"
+    "transport" : "grey"
+    "electricity" : "k"
+    "transport fuel cell" : "#AAAAAA"
+    "biogas" : "#800000"
+    "solid biomass" : "#DAA520"
+  nice_names:
+    # OCGT: "Gas"
+    # OCGT marginal: "Gas (marginal)"
+    offwind: "offshore wind"
+    onwind: "onshore wind"
+    battery: "Battery storage"
+    lines: "Transmission lines"
+    AC line: "AC lines"
+    AC-AC: "DC lines"
+    ror: "Run of river"
+  nice_names_n:
+    offwind: "offshore\nwind"
+    onwind: "onshore\nwind"
+    # OCGT: "Gas"
+    H2: "Hydrogen\nstorage"
+    # OCGT marginal: "Gas (marginal)"
+    lines: "transmission\nlines"
+    ror: "run of river"

scripts/make_summary.py

@@ -3,7 +3,7 @@ from six import iteritems
 
 import sys
 
-sys.path = ['/home/vres/lib/python3.5/site-packages'] + sys.path
+sys.path.append("../pypsa-eur/scripts")
 
 import pandas as pd
 
@@ -13,7 +13,7 @@ import pypsa
 
 from vresutils.costdata import annuity
 
-from prepare_network import generate_periodic_profiles
+from prepare_sector_network import generate_periodic_profiles
 
 from add_electricity import load_costs
 
@@ -428,13 +428,15 @@ if __name__ == "__main__":
         for item in outputs:
             snakemake.output[item] = snakemake.config['summary_dir'] + '/{name}/csvs/{item}.csv'.format(name=snakemake.config['run'],item=item)
 
-    networks_dict = {(cluster,lv,opt) :
+    networks_dict = {(cluster,lv,opt+sector_opt) :
-                     snakemake.config['results_dir'] + snakemake.config['run'] + '/postnetworks/elec_s_{cluster}_lv{lv}_{opt}.nc'\
+                     snakemake.config['results_dir'] + snakemake.config['run'] + '/postnetworks/elec_s_{cluster}_lv{lv}_{opt}_{sector_opt}.nc'\
                      .format(cluster=cluster,
                              opt=opt,
-                             lv=lv)\
+                             lv=lv,
+                             sector_opt=sector_opt)\
                      for cluster in snakemake.config['scenario']['clusters'] \
                      for opt in snakemake.config['scenario']['opts'] \
+                     for sector_opt in snakemake.config['scenario']['sector_opts'] \
                      for lv in snakemake.config['scenario']['lv']}
     print(networks_dict)