From 708cdb8916f127d4c9e14db16c62860079848bae Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 22 Feb 2023 19:13:03 +0100 Subject: [PATCH 1/7] ci: reduce duplications of test/config.test1.yaml --- .github/workflows/ci.yaml | 4 +- test/config.test1.yaml | 281 -------------------------------------- 2 files changed, 1 insertion(+), 284 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 26111c94..e4dbbe24 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -95,6 +95,4 @@ jobs: run: | conda activate pypsa-eur conda list - cp test/config.test1.yaml config.yaml - snakemake --cores all solve_all_networks - rm -rf resources/*.nc resources/*.geojson resources/*.h5 networks results + snakemake -call solve_all_networks --configfile config.test1.yaml diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 399d2265..a7f52eaa 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -2,18 +2,9 @@ # # SPDX-License-Identifier: CC0-1.0 -version: 0.7.0 tutorial: true -logging: - level: INFO - format: '%(levelname)s:%(name)s:%(message)s' - -run: - name: "" scenario: - simpl: [''] - ll: ['copt'] clusters: [5] opts: [Co2L-24H] @@ -22,20 +13,8 @@ countries: ['BE'] snapshots: start: "2013-03-01" end: "2013-03-08" - inclusive: 'left' # include start, not end - -enable: - prepare_links_p_nom: false - retrieve_databundle: true - retrieve_cost_data: true - build_cutout: false - retrieve_cutout: true - build_natura_raster: false - retrieve_natura_raster: true - custom_busmap: false electricity: - voltages: [220., 300., 380.] co2limit: 100.e+6 extendable_carriers: @@ -44,38 +23,8 @@ electricity: Store: [H2] Link: [H2 pipeline] - max_hours: - battery: 6 - H2: 168 - - # use pandas query strings here, e.g. Country not in ['Germany'] - powerplants_filter: (DateOut >= 2022 or DateOut != DateOut) - # use pandas query strings here, e.g. Country in ['Germany'] - custom_powerplants: false - - conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] - renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro] - - estimate_renewable_capacities: - enable: true - # Add capacities from OPSD data - from_opsd: true - # Renewable capacities are based on existing capacities reported by IRENA - year: 2020 - # Artificially limit maximum capacities to factor * (IRENA capacities), - # i.e. 110% of 's capacities => expansion_limit: 1.1 - # false: Use estimated renewable potentials determine by the workflow - expansion_limit: false - technology_mapping: - # Wind is the Fueltype in powerplantmatching, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur - Offshore: [offwind-ac, offwind-dc] - Onshore: [onwind] - PV: [solar] - atlite: - nprocesses: 4 - show_progress: false # false saves time cutouts: be-03-2013-era5: module: era5 @@ -86,246 +35,16 @@ atlite: renewable: onwind: cutout: be-03-2013-era5 - resource: - method: wind - turbine: Vestas_V112_3MW - capacity_per_sqkm: 3 # ScholzPhd Tab 4.3.1: 10MW/km^2 - # correction_factor: 0.93 - corine: - # Scholz, Y. (2012). Renewable energy based electricity supply at low costs: - # development of the REMix model and application for Europe. ( p.42 / p.28) - grid_codes: [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32] - distance: 1000 - distance_grid_codes: [1, 2, 3, 4, 5, 6] - natura: true - excluder_resolution: 200 - potential: simple # or conservative - clip_p_max_pu: 1.e-2 offwind-ac: cutout: be-03-2013-era5 - resource: - method: wind - turbine: NREL_ReferenceTurbine_5MW_offshore - capacity_per_sqkm: 3 - # correction_factor: 0.93 - corine: [44, 255] - natura: true - ship_threshold: 400 - max_shore_distance: 30000 - excluder_resolution: 200 - potential: simple # or conservative - clip_p_max_pu: 1.e-2 offwind-dc: cutout: be-03-2013-era5 - resource: - method: wind - turbine: NREL_ReferenceTurbine_5MW_offshore - # ScholzPhd Tab 4.3.1: 10MW/km^2 - capacity_per_sqkm: 3 - # correction_factor: 0.93 - corine: [44, 255] - natura: true - ship_threshold: 400 - min_shore_distance: 30000 - excluder_resolution: 200 - potential: simple # or conservative - clip_p_max_pu: 1.e-2 solar: cutout: be-03-2013-era5 - resource: - method: pv - panel: CSi - orientation: - slope: 35. - azimuth: 180. - capacity_per_sqkm: 1.7 # ScholzPhd Tab 4.3.1: 170 MW/km^2 - # Determined by comparing uncorrected area-weighted full-load hours to those - # published in Supplementary Data to - # Pietzcker, Robert Carl, et al. "Using the sun to decarbonize the power - # sector: The economic potential of photovoltaics and concentrating solar - # power." Applied Energy 135 (2014): 704-720. - correction_factor: 0.854337 - corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32] - natura: true - excluder_resolution: 200 - potential: simple # or conservative - clip_p_max_pu: 1.e-2 - -lines: - types: - 220.: "Al/St 240/40 2-bundle 220.0" - 300.: "Al/St 240/40 3-bundle 300.0" - 380.: "Al/St 240/40 4-bundle 380.0" - s_max_pu: 0.7 - s_nom_max: .inf - length_factor: 1.25 - under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity - -links: - p_max_pu: 1.0 - p_nom_max: .inf - include_tyndp: true - under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity - -transformers: - x: 0.1 - s_nom: 2000. - type: '' - -load: - power_statistics: true # only for files from <2019; set false in order to get ENTSOE transparency data - interpolate_limit: 3 # data gaps up until this size are interpolated linearly - time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from - manual_adjustments: true # false - scaling_factor: 1.0 - -costs: - year: 2030 - version: v0.5.0 - rooftop_share: 0.14 - fill_values: - FOM: 0 - VOM: 0 - efficiency: 1 - fuel: 0 - investment: 0 - lifetime: 25 - "CO2 intensity": 0 - "discount rate": 0.07 - marginal_cost: - solar: 0.01 - onwind: 0.015 - offwind: 0.015 - H2: 0. - battery: 0. - emission_prices: # only used with the option Ep - co2: 0. clustering: - simplify_network: - to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) - algorithm: kmeans # choose from: [hac, kmeans] - feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. - cluster_network: - algorithm: kmeans - feature: solar+onwind-time exclude_carriers: ["OCGT", "offwind-ac", "coal"] - aggregation_strategies: - generators: - p_nom_max: sum # use "min" for more conservative assumptions - p_nom_min: sum - p_min_pu: mean - marginal_cost: mean - committable: any - ramp_limit_up: max - ramp_limit_down: max - efficiency: mean solving: - options: - formulation: kirchhoff - load_shedding: false - noisy_costs: true - min_iterations: 1 - max_iterations: 1 - clip_p_max_pu: 0.01 - #nhours: 10 solver: name: glpk - # solver: - # name: gurobi - # threads: 4 - # method: 2 # barrier - # crossover: 0 - # BarConvTol: 1.e-5 - # FeasibilityTol: 1.e-6 - # AggFill: 0 - # PreDual: 0 - # GURO_PAR_BARDENSETHRESH: 200 - # solver: - # name: cplex - # threads: 4 - # lpmethod: 4 # barrier - # solutiontype: 2 # non basic solution, ie no crossover - # barrier.convergetol: 1.e-5 - # feasopt.tolerance: 1.e-6 - -plotting: - map: - figsize: [7, 7] - boundaries: [-10.2, 29, 35, 72] - p_nom: - bus_size_factor: 5.e+4 - linewidth_factor: 3.e+3 - - costs_max: 800 - costs_threshold: 1 - - energy_max: 15000. - energy_min: -10000. - energy_threshold: 50. - - vre_techs: ["onwind", "offwind-ac", "offwind-dc", "solar", "ror"] - conv_techs: ["OCGT", "CCGT", "Nuclear", "Coal"] - storage_techs: ["hydro+PHS", "battery", "H2"] - load_carriers: ["AC load"] - AC_carriers: ["AC line", "AC transformer"] - link_carriers: ["DC line", "Converter AC-DC"] - tech_colors: - "onwind": "#235ebc" - "onshore wind": "#235ebc" - 'offwind': "#6895dd" - 'offwind-ac': "#6895dd" - 'offshore wind': "#6895dd" - 'offshore wind ac': "#6895dd" - 'offwind-dc': "#74c6f2" - 'offshore wind dc': "#74c6f2" - "hydro": "#08ad97" - "hydro+PHS": "#08ad97" - "PHS": "#08ad97" - "hydro reservoir": "#08ad97" - 'hydroelectricity': '#08ad97' - "ror": "#4adbc8" - "run of river": "#4adbc8" - 'solar': "#f9d002" - 'solar PV': "#f9d002" - 'solar thermal': '#ffef60' - 'biomass': '#0c6013' - 'solid biomass': '#06540d' - 'biogas': '#23932d' - 'waste': '#68896b' - 'geothermal': '#ba91b1' - "OCGT": "#d35050" - "gas": "#d35050" - "natural gas": "#d35050" - "CCGT": "#b20101" - "nuclear": "#ff9000" - "coal": "#707070" - "lignite": "#9e5a01" - "oil": "#262626" - "H2": "#ea048a" - "hydrogen storage": "#ea048a" - "battery": "#b8ea04" - "Electric load": "#f9d002" - "electricity": "#f9d002" - "lines": "#70af1d" - "transmission lines": "#70af1d" - "AC-AC": "#70af1d" - "AC line": "#70af1d" - "links": "#8a1caf" - "HVDC links": "#8a1caf" - "DC-DC": "#8a1caf" - "DC link": "#8a1caf" - nice_names: - OCGT: "Open-Cycle Gas" - CCGT: "Combined-Cycle Gas" - offwind-ac: "Offshore Wind (AC)" - offwind-dc: "Offshore Wind (DC)" - onwind: "Onshore Wind" - solar: "Solar" - PHS: "Pumped Hydro Storage" - hydro: "Reservoir & Dam" - battery: "Battery Storage" - H2: "Hydrogen Storage" - lines: "Transmission Lines" - ror: "Run of River" From 4e75d6c64432ae32b31c0e7bbacc70987ef65211 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 22 Feb 2023 19:20:28 +0100 Subject: [PATCH 2/7] fix path to test config --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e4dbbe24..eb995662 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -95,4 +95,4 @@ jobs: run: | conda activate pypsa-eur conda list - snakemake -call solve_all_networks --configfile config.test1.yaml + snakemake -call solve_all_networks --configfile test/config.test1.yaml From 67d878d1d7d0dccbdf43508b49d34ec5ec3c4aca Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 22 Feb 2023 19:28:07 +0100 Subject: [PATCH 3/7] allow disabling max_depth through config overwrite --- Snakefile | 2 +- test/config.test1.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 0ebfe89d..5c96c55d 100644 --- a/Snakefile +++ b/Snakefile @@ -343,7 +343,7 @@ rule build_renewable_profiles: ), gebco=lambda w: ( "data/bundle/GEBCO_2014_2D.nc" - if "max_depth" in config["renewable"][w.technology].keys() + if config["renewable"][w.technology].get("max_depth") else [] ), ship_density=lambda w: ( diff --git a/test/config.test1.yaml b/test/config.test1.yaml index a7f52eaa..5a44a394 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -37,8 +37,10 @@ renewable: cutout: be-03-2013-era5 offwind-ac: cutout: be-03-2013-era5 + max_depth: false offwind-dc: cutout: be-03-2013-era5 + max_depth: false solar: cutout: be-03-2013-era5 From 8837ba6a5f1f179bf892a03dc7f9874f6d5acd45 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 5 Mar 2023 15:12:20 +0100 Subject: [PATCH 4/7] allow false for max_depth --- scripts/build_renewable_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index a132115b..01e050a7 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -256,7 +256,7 @@ if __name__ == "__main__": snakemake.input.ship_density, codes=func, crs=4326, allow_no_overlap=True ) - if "max_depth" in config: + if config.get('max_depth'): # lambda not supported for atlite + multiprocessing # use named function np.greater with partially frozen argument instead # and exclude areas where: -max_depth > grid cell depth From 36003c96270f622e67288ed8179e04cbbaa42d84 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 5 Mar 2023 14:12:38 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_renewable_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 01e050a7..f74aa2aa 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -256,7 +256,7 @@ if __name__ == "__main__": snakemake.input.ship_density, codes=func, crs=4326, allow_no_overlap=True ) - if config.get('max_depth'): + if config.get("max_depth"): # lambda not supported for atlite + multiprocessing # use named function np.greater with partially frozen argument instead # and exclude areas where: -max_depth > grid cell depth From 3bbd321b8296a37d690f70281547185d228a4a99 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 6 Mar 2023 09:35:11 +0100 Subject: [PATCH 6/7] config: separate solver_options solve_network: adjust code to new solver_options config --- config.default.yaml | 91 +++++++++++++++++++++++++++++++--------- scripts/solve_network.py | 9 ++-- 2 files changed, 77 insertions(+), 23 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index f01474b2..9e98ea01 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -281,33 +281,86 @@ clustering: efficiency: mean solving: + #tmpdir: "path/to/tmp" options: formulation: kirchhoff + clip_p_max_pu: 1.e-2 load_shedding: false noisy_costs: true - min_iterations: 4 - max_iterations: 6 - clip_p_max_pu: 0.01 skip_iterations: true track_iterations: false - #nhours: 10 + min_iterations: 4 + max_iterations: 6 + keep_shadowprices: + - Bus + - Line + - Link + - Transformer + - GlobalConstraint + - Generator + - Store + - StorageUnit + solver: name: gurobi - threads: 4 - method: 2 # barrier - crossover: 0 - BarConvTol: 1.e-5 - FeasibilityTol: 1.e-6 - AggFill: 0 - PreDual: 0 - GURO_PAR_BARDENSETHRESH: 200 - # solver: - # name: cplex - # threads: 4 - # lpmethod: 4 # barrier - # solutiontype: 2 # non basic solution, ie no crossover - # barrier.convergetol: 1.e-5 - # feasopt.tolerance: 1.e-6 + options: gurobi-default + + solver_options: + highs-default: + # refer to https://ergo-code.github.io/HiGHS/options/definitions.html#solver + threads: 4 + solver: "ipm" + run_crossover: "off" + small_matrix_value: 1e-6 + large_matrix_value: 1e9 + primal_feasibility_tolerance: 1e-5 + dual_feasibility_tolerance: 1e-5 + ipm_optimality_tolerance: 1e-4 + parallel: "on" + random_seed: 123 + gurobi-default: + threads: 4 + method: 2 # barrier + crossover: 0 + BarConvTol: 1.e-6 + Seed: 123 + AggFill: 0 + PreDual: 0 + GURO_PAR_BARDENSETHRESH: 200 + seed: 10 # Consistent seed for all plattforms + gurobi-numeric-focus: + name: gurobi + NumericFocus: 3 # Favour numeric stability over speed + method: 2 # barrier + crossover: 0 # do not use crossover + BarHomogeneous: 1 # Use homogeneous barrier if standard does not converge + BarConvTol: 1.e-5 + FeasibilityTol: 1.e-4 + OptimalityTol: 1.e-4 + ObjScale: -0.5 + threads: 8 + Seed: 123 + gurobi-fallback: # Use gurobi defaults + name: gurobi + crossover: 0 + method: 2 # barrier + BarHomogeneous: 1 # Use homogeneous barrier if standard does not converge + BarConvTol: 1.e-5 + FeasibilityTol: 1.e-5 + OptimalityTol: 1.e-5 + Seed: 123 + threads: 8 + cplex-default: + threads: 4 + lpmethod: 4 # barrier + solutiontype: 2 # non basic solution, ie no crossover + barrier_convergetol: 1.e-5 + feasopt_tolerance: 1.e-6 + + cbc-default: {} # Used in CI + + mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 + plotting: map: diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 4804a23c..75e1c601 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -373,8 +373,9 @@ def extra_functionality(n, snapshots): def solve_network(n, config, opts="", **kwargs): - solver_options = config["solving"]["solver"].copy() - solver_name = solver_options.pop("name") + set_of_options = config['solving']['solver']['options'] + solver_options = config['solving']["solver_options"][set_of_options] if set_of_options else {} + solver_name = config['solving']['solver']['name'] cf_solving = config["solving"]["options"] track_iterations = cf_solving.get("track_iterations", False) min_iterations = cf_solving.get("min_iterations", 4) @@ -411,7 +412,7 @@ if __name__ == "__main__": from _helpers import mock_snakemake snakemake = mock_snakemake( - "solve_network", simpl="", clusters="5", ll="copt", opts="Co2L-BAU-CCL-24H" + "solve_network", simpl="", clusters="5", ll="v1.5", opts="" ) configure_logging(snakemake) @@ -419,7 +420,7 @@ if __name__ == "__main__": if tmpdir is not None: Path(tmpdir).mkdir(parents=True, exist_ok=True) opts = snakemake.wildcards.opts.split("-") - solve_opts = snakemake.config["solving"]["options"] + solve_opts = snakemake.config['solving']['options'] fn = getattr(snakemake.log, "memory", None) with memory_logger(filename=fn, interval=30.0) as mem: From 5ab10eae37cd71a4676478c79c79b520ff048d61 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 08:40:19 +0000 Subject: [PATCH 7/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- config.default.yaml | 16 ++++++++-------- scripts/solve_network.py | 10 ++++++---- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 9e98ea01..dde27bfb 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -292,14 +292,14 @@ solving: min_iterations: 4 max_iterations: 6 keep_shadowprices: - - Bus - - Line - - Link - - Transformer - - GlobalConstraint - - Generator - - Store - - StorageUnit + - Bus + - Line + - Link + - Transformer + - GlobalConstraint + - Generator + - Store + - StorageUnit solver: name: gurobi diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 75e1c601..24269cb2 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -373,9 +373,11 @@ def extra_functionality(n, snapshots): def solve_network(n, config, opts="", **kwargs): - set_of_options = config['solving']['solver']['options'] - solver_options = config['solving']["solver_options"][set_of_options] if set_of_options else {} - solver_name = config['solving']['solver']['name'] + set_of_options = config["solving"]["solver"]["options"] + solver_options = ( + config["solving"]["solver_options"][set_of_options] if set_of_options else {} + ) + solver_name = config["solving"]["solver"]["name"] cf_solving = config["solving"]["options"] track_iterations = cf_solving.get("track_iterations", False) min_iterations = cf_solving.get("min_iterations", 4) @@ -420,7 +422,7 @@ if __name__ == "__main__": if tmpdir is not None: Path(tmpdir).mkdir(parents=True, exist_ok=True) opts = snakemake.wildcards.opts.split("-") - solve_opts = snakemake.config['solving']['options'] + solve_opts = snakemake.config["solving"]["options"] fn = getattr(snakemake.log, "memory", None) with memory_logger(filename=fn, interval=30.0) as mem: