From 6a613ea483d9d86190183e1aa5fbe9ccf19ba404 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonas=20H=C3=B6rsch?= <hoersch@fias.uni-frankfurt.de>
Date: Sat, 10 Feb 2018 17:16:20 +0100
Subject: [PATCH] build_powerplants: Separate loading of power plant list into
 new rule

---
 Snakefile                    | 12 ++++++++++--
 scripts/add_electricity.py   | 14 ++++----------
 scripts/build_powerplants.py | 29 +++++++++++++++++++++++++++++
 scripts/cluster_network.py   |  2 +-
 scripts/prepare_network.py   |  2 +-
 scripts/solve_network.py     |  3 ++-
 6 files changed, 47 insertions(+), 15 deletions(-)
 create mode 100644 scripts/build_powerplants.py

diff --git a/Snakefile b/Snakefile
index 6933447b..bba6cbf2 100644
--- a/Snakefile
+++ b/Snakefile
@@ -1,6 +1,6 @@
 configfile: "config.yaml"
 
-localrules: all, prepare_links_p_nom, base_network, add_electricity, add_sectors, extract_summaries, plot_network, scenario_comparions
+localrules: all, prepare_links_p_nom, base_network, build_powerplants, add_electricity, add_sectors, prepare_network, extract_summaries, plot_network, scenario_comparions
 
 wildcard_constraints:
     lv="[0-9\.]+",
@@ -41,6 +41,13 @@ rule base_network:
     resources: mem_mb=500
     script: "scripts/base_network.py"
 
+rule build_powerplants:
+    input: base_network="networks/base.nc"
+    output: "resources/powerplants.csv"
+    threads: 1
+    resources: mem_mb=500
+    script: "scripts/build_powerplants.py"
+
 rule build_bus_regions:
     input:
         base_network="networks/base.nc"
@@ -73,12 +80,13 @@ rule add_electricity:
         base_network='networks/base.nc',
         tech_costs='data/costs/costs.csv',
         regions="resources/regions_onshore.geojson",
+        powerplants='resources/powerplants.csv',
         **{'profile_' + t: "resources/profile_" + t + ".nc"
            for t in config['renewable']}
     output: "networks/elec.nc"
     benchmark: "benchmarks/add_electricity"
     threads: 1
-    resources: mem_mb=1000
+    resources: mem_mb=3000
     script: "scripts/add_electricity.py"
 
 rule simplify_network:
diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index aa1329f0..2a1afaec 100644
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -81,16 +81,10 @@ def load_costs(Nyears=1.):
 
     return costs
 
-def load_powerplants(n):
-    ppl = ppm.collection.MATCHED_dataset(aggregated_hydros=False, include_unavailables=True,
-                                         subsume_uncommon_fueltypes=True)
-    ppl = ppl.loc[ppl.lon.notnull() & ppl.lat.notnull()]
-
-    substation_lv_i = n.buses.index[n.buses['substation_lv']]
-    kdtree = sp.spatial.cKDTree(n.buses.loc[substation_lv_i, ['x','y']].values)
-    ppl = ppl.assign(bus=substation_lv_i[kdtree.query(ppl[['lon','lat']].values)[1]])
-
-    return ppl
+def load_powerplants(n, ppl_fn=None):
+    if ppl_fn is None:
+        ppl_fn = snakemake.input.powerplants
+    return pd.read_csv(ppl_fn, index_col=0, dtype={'bus': 'str'})
 
 # ## Attach components
 
diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py
new file mode 100644
index 00000000..e9732ed3
--- /dev/null
+++ b/scripts/build_powerplants.py
@@ -0,0 +1,29 @@
+# coding: utf-8
+
+import logging
+import pandas as pd
+from scipy.spatial import cKDTree as KDTree
+
+import pypsa
+import powerplantmatching as ppm
+
+if 'snakemake' not in globals():
+    from vresutils.snakemake import MockSnakemake, Dict
+
+    snakemake = MockSnakemake(
+        input=Dict(base_network='networks/base.nc'),
+        output=['resources/powerplants.csv']
+    )
+
+logging.basicConfig(level=snakemake.config['logging_level'])
+
+n = pypsa.Network(snakemake.input.base_network)
+
+ppl = ppm.collection.MATCHED_dataset(include_unavailables=True)
+ppl = ppl.loc[ppl.lon.notnull() & ppl.lat.notnull()]
+
+substation_lv_i = n.buses.index[n.buses['substation_lv']]
+kdtree = KDTree(n.buses.loc[substation_lv_i, ['x','y']].values)
+ppl = ppl.assign(bus=substation_lv_i[kdtree.query(ppl[['lon','lat']].values)[1]])
+
+ppl.to_csv(snakemake.output[0])
diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py
index eb8a2617..e8fd4d58 100644
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@@ -149,7 +149,7 @@ if __name__ == "__main__":
             )
         )
 
-    logging.basicConfig(snakemake.config['logging_level'])
+    logging.basicConfig(level=snakemake.config['logging_level'])
 
     n = pypsa.Network(snakemake.input.network)
 
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index ba7eaaaa..679ace3c 100644
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -90,7 +90,7 @@ if __name__ == "__main__":
             output=['networks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}.nc']
         )
 
-    logging.basicConfig(snakemake.config['logging_level'])
+    logging.basicConfig(level=snakemake.config['logging_level'])
 
     opts = snakemake.wildcards.opts.split('-')
 
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index d9257780..9dc29aba 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -242,7 +242,8 @@ if __name__ == "__main__":
     if tmpdir is not None:
         patch_pyomo_tmpdir(tmpdir)
 
-    logging.basicConfig(filename=snakemake.log.python, level=logging.INFO)
+    logging.basicConfig(filename=snakemake.log.python,
+                        level=snakemake.config['logging_level'])
 
     n = pypsa.Network(snakemake.input[0])