Finish solve_operations_network

Snakefile

@@ -137,8 +137,7 @@ rule prepare_network:
     script: "scripts/prepare_network.py"
 
 def partition(w):
-    n_clusters = int(w.clusters[:-1] if w.clusters.endswith('m') else w.clusters)
+    return 'vres' if memory(w) >= 60000 else 'x-men'
-    return 'vres' if n_clusters >= 256 else 'x-men'
 
 def memory(w):
     if w.clusters.endswith('m'):
@@ -164,23 +163,29 @@ rule solve_network:
         vres=lambda w: 1 if partition(w) == 'vres' else 0
     script: "scripts/solve_network.py"
 
+def partition_op(w):
+    return 'vres' if memory_op(w) >= 60000 else 'x-men'
+
+def memory_op(w):
+    return 5000 + 372 * int(w.clusters)
+
 rule solve_operations_network:
     input:
         unprepared="networks/{network}_s{simpl}_{clusters}.nc",
         optimized="results/networks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}.nc"
     output: "results/networks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op.nc"
     shadow: "shallow"
-    params: partition=partition
+    params: partition=partition_op
     log:
-        gurobi="logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op_gurobi.log",
+        gurobi="logs/solve_operations_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op_gurobi.log",
-        python="logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op_python.log",
+        python="logs/solve_operations_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op_python.log",
-        memory="logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op_memory.log"
+        memory="logs/solve_operations_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op_memory.log"
-    benchmark: "benchmarks/solve_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op"
+    benchmark: "benchmarks/solve_operations_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}"
     threads: 4
     resources:
-        mem_mb=memory,
+        mem_mb=memory_op,
-        x_men=lambda w: 1 if partition(w) == 'x-men' else 0,
+        x_men=lambda w: 1 if partition_op(w) == 'x-men' else 0,
-        vres=lambda w: 1 if partition(w) == 'vres' else 0
+        vres=lambda w: 1 if partition_op(w) == 'vres' else 0
     script: "scripts/solve_operations_network.py"
 
 rule plot_network:

cluster.yaml

@@ -7,3 +7,6 @@ __default__:
 solve_network:
   partition: "{params.partition}"
   time: "8-0"
+solve_operations_network:
+  partition: "{params.partition}"
+  time: "8-0"

scripts/solve_network.py

@@ -106,7 +106,7 @@ def fix_branches(n, lines_s_nom=None, links_p_nom=None):
         if isinstance(n.opt, pypsa.opf.PersistentSolver):
             n.opt.update_var(n.model.link_p_nom)
 
-def solve_network(n, config=None, gurobi_log=None):
+def solve_network(n, config=None, gurobi_log=None, opts=None):
     if config is None:
         config = snakemake.config['solving']
     solve_opts = config['options']
@@ -122,7 +122,7 @@ def solve_network(n, config=None, gurobi_log=None):
 
         if not hasattr(n, 'opt') or not isinstance(n.opt, pypsa.opf.PersistentSolver):
             pypsa.opf.network_lopf_build_model(n, formulation=solve_opts['formulation'])
-            add_opts_constraints(n)
+            add_opts_constraints(n, opts)
             add_lv_constraint(n)
             add_eps_storage_constraint(n)
 
@@ -231,6 +231,11 @@ def solve_network(n, config=None, gurobi_log=None):
 
         logger.info("Starting last run with fixed extendable lines")
 
+        # Not really needed, could also be taken out
+        # if 'snakemake' in globals():
+        #     fn = os.path.basename(snakemake.output[0])
+        #     n.export_to_netcdf('/home/vres/data/jonas/playground/pypsa-eur/' + fn)
+
     status, termination_condition = run_lopf(n, fix_ext_lines=True)
 
     # Drop zero lines from network

scripts/solve_operations_network.py

@@ -1,20 +1,27 @@
 import pypsa
+import numpy as np
+import re
 import logging
 logger = logging.getLogger(__name__)
 
-from solve_network import patch_pyomo_tmpdir, log_memory, solve_network, prepare_network
+from vresutils.benchmark import memory_logger
+from solve_network import patch_pyomo_tmpdir, solve_network, prepare_network
 
 def set_parameters_from_optimized(n, n_optim):
     lines_typed_i = n.lines.index[n.lines.type != '']
-    n.lines.loc[lines_typed_i, 'num_parallel'] = n_optim.lines.loc[lines_typed_i, 'num_parallel']
+    n.lines.loc[lines_typed_i, 'num_parallel'] = n_optim.lines['num_parallel'].reindex(lines_typed_i, fill_value=0.)
+    n.lines.loc[lines_typed_i, 's_nom'] = (
+        np.sqrt(3) * n.lines['type'].map(n.line_types.i_nom) *
+        n.lines.bus0.map(n.buses.v_nom) * n.lines.num_parallel
+    )
     lines_untyped_i = n.lines.index[n.lines.type == '']
     for attr in ('s_nom', 'r', 'x'):
-        n.lines.loc[lines_untyped_i, attr] = n_optim.lines.loc[lines_untyped_i, attr]
+        n.lines.loc[lines_untyped_i, attr] = n_optim.lines[attr].reindex(lines_untyped_i, fill_value=0.)
     n.lines['s_nom_extendable'] = False
 
-    links_extend_i = n.links.index[n.links.p_nom_extendable]
+    links_dc_i = n.links.index[n.links.carrier == 'DC']
-    n.links.loc[links_extend_i, 'p_nom'] = n_optim.links['p_nom_opt'].reindex(links_extend_i, fill_value=0.)
+    n.links.loc[links_dc_i, 'p_nom'] = n_optim.links['p_nom_opt'].reindex(links_dc_i, fill_value=0.)
-    n.links.loc[links_extend_i, 'p_nom_extendable'] = False
+    n.links.loc[links_dc_i, 'p_nom_extendable'] = False
 
     gen_extend_i = n.generators.index[n.generators.p_nom_extendable]
     n.generators.loc[gen_extend_i, 'p_nom'] = n_optim.generators['p_nom_opt'].reindex(gen_extend_i, fill_value=0.)
@@ -29,10 +36,10 @@ def set_parameters_from_optimized(n, n_optim):
 if __name__ == "__main__":
     # Detect running outside of snakemake and mock snakemake for testing
     if 'snakemake' not in globals():
-        from vresutils.snakemake import MockSnakemake, Dict
+        from vresutils.snakemake import MockSnakemake
         snakemake = MockSnakemake(
             wildcards=dict(network='elec', simpl='', clusters='45', lv='1.5', opts='Co2L-3H'),
-            input=Dict(unprepared="networks/{network}_s{simpl}_{clusters}.nc",
+            input=dict(unprepared="networks/{network}_s{simpl}_{clusters}.nc",
                        optimized="results/networks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}.nc"),
             output=["results/networks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op.nc"],
             log=dict(gurobi="logs/s{simpl}_{clusters}_lv{lv}_{opts}_op_gurobi.log",
@@ -52,10 +59,13 @@ if __name__ == "__main__":
     n = set_parameters_from_optimized(n, n_optim)
     del n_optim
 
-    with log_memory(filename=getattr(snakemake.log, 'memory', None), interval=30., max_usage=True) as mem:
+    opts = [o
-        n = prepare_network(n, solve_opts=snakemake.config['solving']['options'])
+            for o in snakemake.wildcards.opts.split('-')
-        n = solve_network(n, config=snakemake.config['solving'], gurobi_log=snakemake.log.gurobi)
+            if not re.match(r'^\d+h$', o, re.IGNORECASE)]
 
+    with memory_logger(filename=getattr(snakemake.log, 'memory', None), interval=30.) as mem:
+        n = prepare_network(n, solve_opts=snakemake.config['solving']['options'])
+        n = solve_network(n, config=snakemake.config['solving'], gurobi_log=snakemake.log.gurobi, opts=opts)
         n.export_to_netcdf(snakemake.output[0])
 
-    logger.info("Maximum memory usage: {}".format(mem.mem_usage[0]))
+    logger.info("Maximum memory usage: {}".format(mem.mem_usage))