From 4bde9417dc7ec2d1e1066de78175445e4fd27656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6rsch?= Date: Tue, 13 Mar 2018 10:47:47 +0100 Subject: [PATCH] Finish solve_operations_network --- Snakefile | 25 ++++++++++++--------- cluster.yaml | 3 +++ scripts/solve_network.py | 9 ++++++-- scripts/solve_operations_network.py | 34 +++++++++++++++++++---------- 4 files changed, 47 insertions(+), 24 deletions(-) diff --git a/Snakefile b/Snakefile index 998954db..5f164fdd 100644 --- a/Snakefile +++ b/Snakefile @@ -137,8 +137,7 @@ rule prepare_network: script: "scripts/prepare_network.py" def partition(w): - n_clusters = int(w.clusters[:-1] if w.clusters.endswith('m') else w.clusters) - return 'vres' if n_clusters >= 256 else 'x-men' + return 'vres' if memory(w) >= 60000 else 'x-men' def memory(w): if w.clusters.endswith('m'): @@ -164,23 +163,29 @@ rule solve_network: vres=lambda w: 1 if partition(w) == 'vres' else 0 script: "scripts/solve_network.py" +def partition_op(w): + return 'vres' if memory_op(w) >= 60000 else 'x-men' + +def memory_op(w): + return 5000 + 372 * int(w.clusters) + rule solve_operations_network: input: unprepared="networks/{network}_s{simpl}_{clusters}.nc", optimized="results/networks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}.nc" output: "results/networks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op.nc" shadow: "shallow" - params: partition=partition + params: partition=partition_op log: - gurobi="logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op_gurobi.log", - python="logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op_python.log", - memory="logs/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op_memory.log" - benchmark: "benchmarks/solve_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op" + gurobi="logs/solve_operations_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op_gurobi.log", + python="logs/solve_operations_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op_python.log", + memory="logs/solve_operations_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op_memory.log" + benchmark: "benchmarks/solve_operations_network/{network}_s{simpl}_{clusters}_lv{lv}_{opts}" threads: 4 resources: - mem_mb=memory, - x_men=lambda w: 1 if partition(w) == 'x-men' else 0, - vres=lambda w: 1 if partition(w) == 'vres' else 0 + mem_mb=memory_op, + x_men=lambda w: 1 if partition_op(w) == 'x-men' else 0, + vres=lambda w: 1 if partition_op(w) == 'vres' else 0 script: "scripts/solve_operations_network.py" rule plot_network: diff --git a/cluster.yaml b/cluster.yaml index 2e2f5ae9..a56786da 100644 --- a/cluster.yaml +++ b/cluster.yaml @@ -7,3 +7,6 @@ __default__: solve_network: partition: "{params.partition}" time: "8-0" +solve_operations_network: + partition: "{params.partition}" + time: "8-0" diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 80b6f9ab..7a8893ca 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -106,7 +106,7 @@ def fix_branches(n, lines_s_nom=None, links_p_nom=None): if isinstance(n.opt, pypsa.opf.PersistentSolver): n.opt.update_var(n.model.link_p_nom) -def solve_network(n, config=None, gurobi_log=None): +def solve_network(n, config=None, gurobi_log=None, opts=None): if config is None: config = snakemake.config['solving'] solve_opts = config['options'] @@ -122,7 +122,7 @@ def solve_network(n, config=None, gurobi_log=None): if not hasattr(n, 'opt') or not isinstance(n.opt, pypsa.opf.PersistentSolver): pypsa.opf.network_lopf_build_model(n, formulation=solve_opts['formulation']) - add_opts_constraints(n) + add_opts_constraints(n, opts) add_lv_constraint(n) add_eps_storage_constraint(n) @@ -231,6 +231,11 @@ def solve_network(n, config=None, gurobi_log=None): logger.info("Starting last run with fixed extendable lines") + # Not really needed, could also be taken out + # if 'snakemake' in globals(): + # fn = os.path.basename(snakemake.output[0]) + # n.export_to_netcdf('/home/vres/data/jonas/playground/pypsa-eur/' + fn) + status, termination_condition = run_lopf(n, fix_ext_lines=True) # Drop zero lines from network diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 10be0160..983e197a 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -1,20 +1,27 @@ import pypsa +import numpy as np +import re import logging logger = logging.getLogger(__name__) -from solve_network import patch_pyomo_tmpdir, log_memory, solve_network, prepare_network +from vresutils.benchmark import memory_logger +from solve_network import patch_pyomo_tmpdir, solve_network, prepare_network def set_parameters_from_optimized(n, n_optim): lines_typed_i = n.lines.index[n.lines.type != ''] - n.lines.loc[lines_typed_i, 'num_parallel'] = n_optim.lines.loc[lines_typed_i, 'num_parallel'] + n.lines.loc[lines_typed_i, 'num_parallel'] = n_optim.lines['num_parallel'].reindex(lines_typed_i, fill_value=0.) + n.lines.loc[lines_typed_i, 's_nom'] = ( + np.sqrt(3) * n.lines['type'].map(n.line_types.i_nom) * + n.lines.bus0.map(n.buses.v_nom) * n.lines.num_parallel + ) lines_untyped_i = n.lines.index[n.lines.type == ''] for attr in ('s_nom', 'r', 'x'): - n.lines.loc[lines_untyped_i, attr] = n_optim.lines.loc[lines_untyped_i, attr] + n.lines.loc[lines_untyped_i, attr] = n_optim.lines[attr].reindex(lines_untyped_i, fill_value=0.) n.lines['s_nom_extendable'] = False - links_extend_i = n.links.index[n.links.p_nom_extendable] - n.links.loc[links_extend_i, 'p_nom'] = n_optim.links['p_nom_opt'].reindex(links_extend_i, fill_value=0.) - n.links.loc[links_extend_i, 'p_nom_extendable'] = False + links_dc_i = n.links.index[n.links.carrier == 'DC'] + n.links.loc[links_dc_i, 'p_nom'] = n_optim.links['p_nom_opt'].reindex(links_dc_i, fill_value=0.) + n.links.loc[links_dc_i, 'p_nom_extendable'] = False gen_extend_i = n.generators.index[n.generators.p_nom_extendable] n.generators.loc[gen_extend_i, 'p_nom'] = n_optim.generators['p_nom_opt'].reindex(gen_extend_i, fill_value=0.) @@ -29,10 +36,10 @@ def set_parameters_from_optimized(n, n_optim): if __name__ == "__main__": # Detect running outside of snakemake and mock snakemake for testing if 'snakemake' not in globals(): - from vresutils.snakemake import MockSnakemake, Dict + from vresutils.snakemake import MockSnakemake snakemake = MockSnakemake( wildcards=dict(network='elec', simpl='', clusters='45', lv='1.5', opts='Co2L-3H'), - input=Dict(unprepared="networks/{network}_s{simpl}_{clusters}.nc", + input=dict(unprepared="networks/{network}_s{simpl}_{clusters}.nc", optimized="results/networks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}.nc"), output=["results/networks/{network}_s{simpl}_{clusters}_lv{lv}_{opts}_op.nc"], log=dict(gurobi="logs/s{simpl}_{clusters}_lv{lv}_{opts}_op_gurobi.log", @@ -52,10 +59,13 @@ if __name__ == "__main__": n = set_parameters_from_optimized(n, n_optim) del n_optim - with log_memory(filename=getattr(snakemake.log, 'memory', None), interval=30., max_usage=True) as mem: - n = prepare_network(n, solve_opts=snakemake.config['solving']['options']) - n = solve_network(n, config=snakemake.config['solving'], gurobi_log=snakemake.log.gurobi) + opts = [o + for o in snakemake.wildcards.opts.split('-') + if not re.match(r'^\d+h$', o, re.IGNORECASE)] + with memory_logger(filename=getattr(snakemake.log, 'memory', None), interval=30.) as mem: + n = prepare_network(n, solve_opts=snakemake.config['solving']['options']) + n = solve_network(n, config=snakemake.config['solving'], gurobi_log=snakemake.log.gurobi, opts=opts) n.export_to_netcdf(snakemake.output[0]) - logger.info("Maximum memory usage: {}".format(mem.mem_usage[0])) + logger.info("Maximum memory usage: {}".format(mem.mem_usage))