diff --git a/Snakefile b/Snakefile new file mode 100644 index 00000000..171be632 --- /dev/null +++ b/Snakefile @@ -0,0 +1,157 @@ +configfile: "config.yaml" + +localrules: all, prepare_links_p_nom, base_network, add_electricity, add_sectors, extract_summaries, plot_network, scenario_comparions + +wildcard_constraints: + resarea="[a-zA-Z0-9]+", + cost="[-a-zA-Z0-9]+", + sectors="[+a-zA-Z0-9]+", + opts="[-+a-zA-Z0-9]+" + +rule all: + input: "results/version-{version}/summaries/costs2-summary.csv".format(version=config['version']) + +rule prepare_links_p_nom: + output: 'data/links_p_nom.csv' + threads: 1 + resources: mem_mb=500 + script: 'scripts/prepare_links_p_nom.py' + +rule base_network: + input: + eg_buses='data/entsoegridkit/buses.csv', + eg_lines='data/entsoegridkit/lines.csv', + eg_links='data/entsoegridkit/links.csv', + eg_converters='data/entsoegridkit/converters.csv', + eg_transformers='data/entsoegridkit/transformers.csv', + parameter_corrections='data/parameter_corrections.yaml', + links_p_nom='data/links_p_nom.csv' + output: "networks/base_{opts}.h5" + benchmark: "benchmarks/base_network_{opts}" + threads: 1 + resources: mem_mb=500 + script: "scripts/base_network.py" + +rule landuse_remove_protected_and_conservation_areas: + input: + landuse = "data/Original_UTM35north/sa_lcov_2013-14_gti_utm35n_vs22b.tif", + protected_areas = "data/SAPAD_OR_2017_Q2/", + conservation_areas = "data/SACAD_OR_2017_Q2/" + output: "resources/landuse_without_protected_conservation.tiff" + benchmark: "benchmarks/landuse_remove_protected_and_conservation_areas" + threads: 1 + resources: mem_mb=10000 + script: "scripts/landuse_remove_protected_and_conservation_areas.py" + +rule landuse_map_to_tech_and_supply_region: + input: + landuse = "resources/landuse_without_protected_conservation.tiff", + supply_regions = "data/supply_regions/supply_regions.shp", + resarea = lambda w: config['data']['resarea'][w.resarea] + output: + raster = "resources/raster_{tech}_percent_{resarea}.tiff", + area = "resources/area_{tech}_{resarea}.csv" + benchmark: "benchmarks/landuse_map_to_tech_and_supply_region/{tech}_{resarea}" + threads: 1 + resources: mem_mb=17000 + script: "scripts/landuse_map_to_tech_and_supply_region.py" + +rule inflow_per_country: + input: EIA_hydro_gen="data/EIA_hydro_generation_2011_2014.csv" + output: "resources/hydro_inflow.nc" + benchmark: "benchmarks/inflow_per_country" + threads: 1 + resources: mem_mb=1000 + script: "scripts/inflow_per_country.py" + +rule add_electricity: + input: + base_network='networks/base_{opts}.h5', + supply_regions='data/supply_regions/supply_regions.shp', + load='data/SystemEnergy2009_13.csv', + wind_pv_profiles='data/Wind_PV_Normalised_Profiles.xlsx', + wind_area='resources/area_wind_{resarea}.csv', + solar_area='resources/area_solar_{resarea}.csv', + existing_generators="data/Existing Power Stations SA.xlsx", + hydro_inflow="resources/hydro_inflow.csv", + tech_costs="data/technology_costs.xlsx" + output: "networks/elec_{cost}_{resarea}_{opts}.h5" + benchmark: "benchmarks/add_electricity/elec_{resarea}_{opts}" + threads: 1 + resources: mem_mb=1000 + script: "scripts/add_electricity.py" + +rule add_sectors: + input: + network="networks/elec_{cost}_{resarea}_{opts}.h5", + emobility="data/emobility" + output: "networks/sector_{cost}_{resarea}_{sectors}_{opts}.h5" + benchmark: "benchmarks/add_sectors/sector_{resarea}_{sectors}_{opts}" + threads: 1 + resources: mem_mb=1000 + script: "scripts/add_sectors.py" + +rule solve_network: + input: network="networks/sector_{cost}_{resarea}_{sectors}_{opts}.h5" + output: "results/version-{version}/networks/{{cost}}_{{resarea}}_{{sectors}}_{{opts}}.h5".format(version=config['version']) + shadow: "shallow" + log: + gurobi="logs/{cost}_{resarea}_{sectors}_{opts}_gurobi.log", + python="logs/{cost}_{resarea}_{sectors}_{opts}_python.log" + benchmark: "benchmarks/solve_network/{cost}_{resarea}_{sectors}_{opts}" + threads: 4 + resources: mem_mb=19000 # for electricity only + script: "scripts/solve_network.py" + +rule plot_network: + input: + network='results/version-{version}/networks/{{cost}}_{{resarea}}_{{sectors}}_{{opts}}.h5'.format(version=config['version']), + supply_regions='data/supply_regions/supply_regions.shp', + resarea=lambda w: config['data']['resarea'][w.resarea] + output: + only_map=touch('results/version-{version}/plots/network_{{cost}}_{{resarea}}_{{sectors}}_{{opts}}_{{attr}}'.format(version=config['version'])), + ext=touch('results/version-{version}/plots/network_{{cost}}_{{resarea}}_{{sectors}}_{{opts}}_{{attr}}_ext'.format(version=config['version'])) + params: ext=['png', 'pdf'] + script: "scripts/plot_network.py" + +# rule plot_costs: +# input: 'results/summaries/costs2-summary.csv' +# output: +# expand('results/plots/costs_{cost}_{resarea}_{sectors}_{opt}', +# **dict(chain(config['scenario'].items(), (('{param}'))) +# touch('results/plots/scenario_plots') +# params: +# tmpl="results/plots/costs_[cost]_[resarea]_[sectors]_[opt]" +# exts=["pdf", "png"] +# scripts: "scripts/plot_costs.py" + +rule scenario_comparison: + input: + expand('results/version-{version}/plots/network_{cost}_{sectors}_{opts}_{attr}_ext', + version=config['version'], + attr=['p_nom'], + **config['scenario']) + output: + html='results/version-{version}/plots/scenario_{{param}}.html'.format(version=config['version']) + params: + tmpl="network_[cost]_[resarea]_[sectors]_[opts]_[attr]_ext", + plot_dir='results/version-{}/plots'.format(config['version']) + script: "scripts/scenario_comparison.py" + +rule extract_summaries: + input: + expand("results/version-{version}/networks/{cost}_{sectors}_{opts}.h5", + version=config['version'], + **config['scenario']) + output: + **{n: "results/version-{version}/summaries/{}-summary.csv".format(n, version=config['version']) + for n in ['costs', 'costs2', 'e_curtailed', 'e_nom_opt', 'e', 'p_nom_opt']} + params: + scenario_tmpl="[cost]_[resarea]_[sectors]_[opts]", + scenarios=config['scenario'] + script: "scripts/extract_summaries.py" + + +# Local Variables: +# mode: python +# End: diff --git a/cluster.yaml b/cluster.yaml new file mode 100644 index 00000000..023e266e --- /dev/null +++ b/cluster.yaml @@ -0,0 +1,5 @@ +__default__: + partition: x-men + name: "pypsa-za.{rule}.{wildcards}" + output: "logs/cluster/{rule}.{wildcards}.out" + error: "logs/cluster/{rule}.{wildcards}.err" diff --git a/config.yaml b/config.yaml new file mode 100644 index 00000000..48fd64e8 --- /dev/null +++ b/config.yaml @@ -0,0 +1,191 @@ +version: 0.1 +logging_level: INFO + +scenario: + sectors: [E] # ,E+EV,E+BEV,E+BEV+V2G] # [ E+EV, E+BEV, E+BEV+V2G ] + cost: [diw2030] + lv: [1., 1.125, 1.25, 1.5, 2.0, 3.0] + opts: [Co2L, Co2L-T] #, LC-FL, LC-T, Ep-T, Co2L-T] + +countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] + +historical_year: "2012" + +electricity: + voltages: [220., 300., 380.] + + co2limit: xxx + + extendable_carriers: + Generator: [OCGT] + StorageUnit: [Battery, H2] # [CAES] + + SAFE_reservemargin: 0.1 + + max_hours: + Battery: 3 + H2: 10 + +reanalysis: + cutout: europe_2011_2016 + + landusetype_percent: + wind: + - [[7, 8, 9, 41], 80] + # - [[5, 6], 50] + # - [[11, 12, 14, 15], 10] + solar: + - [[7, 8, 9, 41], 80] + # - [[11, 12, 14, 15], 50] + # - [[46, 47, 51, 56, 64, 68, 72], 10] + + capacity_per_sqm: + wind: 5 # half of 10 (IWES) + solar: 16.5 # half of 33 (IWES) + +lines: + types: + 220.: "Al/St 240/40 2-bundle 220.0" + 300.: "Al/St 240/40 3-bundle 300.0" + 380.: "Al/St 240/40 4-bundle 380.0" + + s_max_pu: 0.7 + length_factor: 1.25 + +links: + s_max_pu: 0.7 + +transformers: + x: 0.1 + s_nom: 2000. + type: '' + +costs: + discountrate: 0.07 + + # Marginal and capital costs can be overwritten + # capital_cost: + # Wind: Bla + marginal_cost: # + PV: 0.01 + Wind: 0.015 + EUR_to_ZAR: 15.63 + + emission_prices: # only used with the option Ep (emission prices) + # Externality costs from Integrated Energy Plan by the ZA DOE + co2: 0.27e+3 + sox: 7.6e+3 + nox: 4.5e+3 + hg: 41484.e-6 # is also part of the excel sheet + particulate: 11.3e+3 + +solving: + options: + formulation: kirchhoff + clip_p_max_pu: 1.e-2 + load_shedding: true + noisy_costs: true + min_iterations: 4 + max_iterations: 8 + # max_iterations: 1 + # nhours: 10 + solver: + name: gurobi_persistent + threads: 4 + method: 2 + crossover: 0 # -1 (Choose freely) + BarConvTol: 1.e-5 + FeasibilityTol: 1.e-6 + LogToConsole: 0 + OutputFlag: 1 + +plotting: + map: + figsize: [7, 7] + boundaries: [16, -35, 33, -22] + p_nom: + bus_size_factor: 5.e+4 + linewidth_factor: 3.e+3 # 1.e+3 #3.e+3 + + costs_max: 800 + + vre_techs: ["Wind", "PV"] + conv_techs: ["OCGT", "CCGT", "Nuclear", "Coal"] + storage_techs: ["Hydro", "CAES", "Battery", "Pumped storage", "Hydro+PS"] + store_techs: ["Li ion", "water tanks"] + load_carriers: ["AC load", "heat load", "Li ion load"] + AC_carriers: ["AC line", "AC transformer"] + link_carriers: ["DC line", "Converter AC-DC"] + heat_links: ["heat pump", "resistive heater", "CHP heat", "CHP electric", + "gas boiler", "central heat pump", "central resistive heater", "central CHP heat", + "central CHP electric", "central gas boiler"] + heat_generators: ["gas boiler", "central gas boiler", "solar thermal collector", "central solar thermal collector"] + tech_colors: + Wind: "xkcd:azure" + Hydro: "g" + ror: "g" + Hydro+PS: "g" + PV: "y" + OCGT: "brown" + OCGT marginal: "sandybrown" + OCGT-heat: "orange" + central gas boiler: "orange" + gas boiler: "orange" + gas boilers: "orange" + gas boiler marginal: "orange" + gas: "brown" + lines: "k" + AC line: "k" + AC-AC: "k" + transmission lines: "k" + H2: "m" + hydrogen storage: "m" + Battery: "slategray" + battery storage: "slategray" + CAES: "lightgray" + Nuclear: "r" + Nuclear marginal: "r" + Coal: "k" + Coal marginal: "k" + Lignite: "grey" + Lignite marginal: "grey" + CCGT: "orange" + CCGT marginal: "orange" + Diesel: "darkred" + Diesel marginal: "darkred" + heat pumps: "green" + heat pump: "green" + central heat pump: "green" + resistive heater: "pink" + central resistive heater: "pink" + Sabatier: "turquoise" + water tanks: "w" + CHP: "r" + CHP heat: "r" + CHP electric: "r" + central CHP heat: "r" + central CHP electric: "r" + Pumped storage: "g" + Ambient: "k" + AC load: "b" + Heat load: "r" + Li ion load: "grey" + heat: "r" + Li ion: "grey" + district heating: "#CC4E5C" + nice_names: + # OCGT: "Gas" + # OCGT marginal: "Gas (marginal)" + #Battery: "Battery storage" + lines: "Transmission lines" + AC line: "AC lines" + AC-AC: "DC lines" + ror: "Run of river" + nice_names_n: + offwind: "offshore\nwind" + onwind: "onshore\nwind" + # OCGT: "Gas" + H2: "Hydrogen\nstorage" + # OCGT marginal: "Gas (marginal)" + lines: "transmission\nlines" + ror: "run of river" diff --git a/scripts/prepare_links_p_nom.py b/scripts/prepare_links_p_nom.py new file mode 100644 index 00000000..504e16c9 --- /dev/null +++ b/scripts/prepare_links_p_nom.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +import pandas as pd +import numpy as np + +links_p_nom = pd.read_html('https://en.wikipedia.org/wiki/List_of_HVDC_projects', header=0, match="SwePol")[0] + +def extract_coordinates(s): + regex = (r"(\d{1,2})°(\d{1,2})′(\d{1,2})″(N|S) " + r"(\d{1,2})°(\d{1,2})′(\d{1,2})″(E|W)") + e = s.str.extract(regex, expand=True) + lat = (e[0].astype(float) + (e[1].astype(float) + e[2].astype(float)/60.)/60.)*e[3].map({'N': +1., 'S': -1.}) + lon = (e[4].astype(float) + (e[5].astype(float) + e[6].astype(float)/60.)/60.)*e[7].map({'E': +1., 'W': -1.}) + return lon, lat + +m_b = links_p_nom["Power (MW)"].str.contains('x').fillna(False) +def multiply(s): return s.str[0].astype(float) * s.str[1].astype(float) +links_p_nom.loc[m_b, "Power (MW)"] = links_p_nom.loc[m_b, "Power (MW)"].str.split('x').pipe(multiply) +links_p_nom["Power (MW)"] = links_p_nom["Power (MW)"].str.extract("[-/]?([\d.]+)", expand=False).astype(float) + +links_p_nom['x1'], links_p_nom['y1'] = extract_coordinates(links_p_nom['Converter station 1']) +links_p_nom['x2'], links_p_nom['y2'] = extract_coordinates(links_p_nom['Converter station 2']) + +links_p_nom.dropna(subset=['x1', 'y1', 'x2', 'y2']).to_csv(snakemake.output[0], index=False) + diff --git a/snakemake_cluster b/snakemake_cluster new file mode 100755 index 00000000..1851bc87 --- /dev/null +++ b/snakemake_cluster @@ -0,0 +1,4 @@ +#!/bin/bash + +snakemake --cluster-config cluster.yaml --cluster "sbatch --parsable -J '{cluster.name}' -p {cluster.partition} -n 1 --cpus-per-task {threads} -o '{cluster.output}' --mem {resources.mem_mb}" "$@" +