build_powerplants: Separate loading of power plant list into new rule

Snakefile

@@ -1,6 +1,6 @@
 configfile: "config.yaml"
 
-localrules: all, prepare_links_p_nom, base_network, add_electricity, add_sectors, extract_summaries, plot_network, scenario_comparions
+localrules: all, prepare_links_p_nom, base_network, build_powerplants, add_electricity, add_sectors, prepare_network, extract_summaries, plot_network, scenario_comparions
 
 wildcard_constraints:
     lv="[0-9\.]+",
@@ -41,6 +41,13 @@ rule base_network:
     resources: mem_mb=500
     script: "scripts/base_network.py"
 
+rule build_powerplants:
+    input: base_network="networks/base.nc"
+    output: "resources/powerplants.csv"
+    threads: 1
+    resources: mem_mb=500
+    script: "scripts/build_powerplants.py"
+
 rule build_bus_regions:
     input:
         base_network="networks/base.nc"
@@ -73,12 +80,13 @@ rule add_electricity:
         base_network='networks/base.nc',
         tech_costs='data/costs/costs.csv',
         regions="resources/regions_onshore.geojson",
+        powerplants='resources/powerplants.csv',
         **{'profile_' + t: "resources/profile_" + t + ".nc"
            for t in config['renewable']}
     output: "networks/elec.nc"
     benchmark: "benchmarks/add_electricity"
     threads: 1
-    resources: mem_mb=1000
+    resources: mem_mb=3000
     script: "scripts/add_electricity.py"
 
 rule simplify_network:

scripts/add_electricity.py

@@ -81,16 +81,10 @@ def load_costs(Nyears=1.):
 
     return costs
 
-def load_powerplants(n):
+def load_powerplants(n, ppl_fn=None):
-    ppl = ppm.collection.MATCHED_dataset(aggregated_hydros=False, include_unavailables=True,
+    if ppl_fn is None:
-                                         subsume_uncommon_fueltypes=True)
+        ppl_fn = snakemake.input.powerplants
-    ppl = ppl.loc[ppl.lon.notnull() & ppl.lat.notnull()]
+    return pd.read_csv(ppl_fn, index_col=0, dtype={'bus': 'str'})
-
-    substation_lv_i = n.buses.index[n.buses['substation_lv']]
-    kdtree = sp.spatial.cKDTree(n.buses.loc[substation_lv_i, ['x','y']].values)
-    ppl = ppl.assign(bus=substation_lv_i[kdtree.query(ppl[['lon','lat']].values)[1]])
-
-    return ppl
 
 # ## Attach components
 

scripts/build_powerplants.py

@@ -0,0 +1,29 @@
+# coding: utf-8
+
+import logging
+import pandas as pd
+from scipy.spatial import cKDTree as KDTree
+
+import pypsa
+import powerplantmatching as ppm
+
+if 'snakemake' not in globals():
+    from vresutils.snakemake import MockSnakemake, Dict
+
+    snakemake = MockSnakemake(
+        input=Dict(base_network='networks/base.nc'),
+        output=['resources/powerplants.csv']
+    )
+
+logging.basicConfig(level=snakemake.config['logging_level'])
+
+n = pypsa.Network(snakemake.input.base_network)
+
+ppl = ppm.collection.MATCHED_dataset(include_unavailables=True)
+ppl = ppl.loc[ppl.lon.notnull() & ppl.lat.notnull()]
+
+substation_lv_i = n.buses.index[n.buses['substation_lv']]
+kdtree = KDTree(n.buses.loc[substation_lv_i, ['x','y']].values)
+ppl = ppl.assign(bus=substation_lv_i[kdtree.query(ppl[['lon','lat']].values)[1]])
+
+ppl.to_csv(snakemake.output[0])

scripts/cluster_network.py

@@ -149,7 +149,7 @@ if __name__ == "__main__":
             )
         )
 
-    logging.basicConfig(snakemake.config['logging_level'])
+    logging.basicConfig(level=snakemake.config['logging_level'])
 
     n = pypsa.Network(snakemake.input.network)
 

scripts/prepare_network.py

@@ -90,7 +90,7 @@ if __name__ == "__main__":
             output=['networks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}.nc']
         )
 
-    logging.basicConfig(snakemake.config['logging_level'])
+    logging.basicConfig(level=snakemake.config['logging_level'])
 
     opts = snakemake.wildcards.opts.split('-')
 

scripts/solve_network.py

@@ -242,7 +242,8 @@ if __name__ == "__main__":
     if tmpdir is not None:
         patch_pyomo_tmpdir(tmpdir)
 
-    logging.basicConfig(filename=snakemake.log.python, level=logging.INFO)
+    logging.basicConfig(filename=snakemake.log.python,
+                        level=snakemake.config['logging_level'])
 
     n = pypsa.Network(snakemake.input[0])