snakemake: Add skeletons and add prepare_links_p_nom rule
This commit is contained in:
Jonas Hoersch
parent
a19ff893e6
commit
6e1566ac32
157
Snakefile
Normal file
157
Snakefile
Normal file
@ -0,0 +1,157 @@
|
|||||||
|
configfile: "config.yaml"
|
||||||
|
|
||||||
|
localrules: all, prepare_links_p_nom, base_network, add_electricity, add_sectors, extract_summaries, plot_network, scenario_comparions
|
||||||
|
|
||||||
|
wildcard_constraints:
|
||||||
|
resarea="[a-zA-Z0-9]+",
|
||||||
|
cost="[-a-zA-Z0-9]+",
|
||||||
|
sectors="[+a-zA-Z0-9]+",
|
||||||
|
opts="[-+a-zA-Z0-9]+"
|
||||||
|
|
||||||
|
rule all:
|
||||||
|
input: "results/version-{version}/summaries/costs2-summary.csv".format(version=config['version'])
|
||||||
|
|
||||||
|
rule prepare_links_p_nom:
|
||||||
|
output: 'data/links_p_nom.csv'
|
||||||
|
threads: 1
|
||||||
|
resources: mem_mb=500
|
||||||
|
script: 'scripts/prepare_links_p_nom.py'
|
||||||
|
|
||||||
|
rule base_network:
|
||||||
|
input:
|
||||||
|
eg_buses='data/entsoegridkit/buses.csv',
|
||||||
|
eg_lines='data/entsoegridkit/lines.csv',
|
||||||
|
eg_links='data/entsoegridkit/links.csv',
|
||||||
|
eg_converters='data/entsoegridkit/converters.csv',
|
||||||
|
eg_transformers='data/entsoegridkit/transformers.csv',
|
||||||
|
parameter_corrections='data/parameter_corrections.yaml',
|
||||||
|
links_p_nom='data/links_p_nom.csv'
|
||||||
|
output: "networks/base_{opts}.h5"
|
||||||
|
benchmark: "benchmarks/base_network_{opts}"
|
||||||
|
threads: 1
|
||||||
|
resources: mem_mb=500
|
||||||
|
script: "scripts/base_network.py"
|
||||||
|
|
||||||
|
rule landuse_remove_protected_and_conservation_areas:
|
||||||
|
input:
|
||||||
|
landuse = "data/Original_UTM35north/sa_lcov_2013-14_gti_utm35n_vs22b.tif",
|
||||||
|
protected_areas = "data/SAPAD_OR_2017_Q2/",
|
||||||
|
conservation_areas = "data/SACAD_OR_2017_Q2/"
|
||||||
|
output: "resources/landuse_without_protected_conservation.tiff"
|
||||||
|
benchmark: "benchmarks/landuse_remove_protected_and_conservation_areas"
|
||||||
|
threads: 1
|
||||||
|
resources: mem_mb=10000
|
||||||
|
script: "scripts/landuse_remove_protected_and_conservation_areas.py"
|
||||||
|
|
||||||
|
rule landuse_map_to_tech_and_supply_region:
|
||||||
|
input:
|
||||||
|
landuse = "resources/landuse_without_protected_conservation.tiff",
|
||||||
|
supply_regions = "data/supply_regions/supply_regions.shp",
|
||||||
|
resarea = lambda w: config['data']['resarea'][w.resarea]
|
||||||
|
output:
|
||||||
|
raster = "resources/raster_{tech}_percent_{resarea}.tiff",
|
||||||
|
area = "resources/area_{tech}_{resarea}.csv"
|
||||||
|
benchmark: "benchmarks/landuse_map_to_tech_and_supply_region/{tech}_{resarea}"
|
||||||
|
threads: 1
|
||||||
|
resources: mem_mb=17000
|
||||||
|
script: "scripts/landuse_map_to_tech_and_supply_region.py"
|
||||||
|
|
||||||
|
rule inflow_per_country:
|
||||||
|
input: EIA_hydro_gen="data/EIA_hydro_generation_2011_2014.csv"
|
||||||
|
output: "resources/hydro_inflow.nc"
|
||||||
|
benchmark: "benchmarks/inflow_per_country"
|
||||||
|
threads: 1
|
||||||
|
resources: mem_mb=1000
|
||||||
|
script: "scripts/inflow_per_country.py"
|
||||||
|
|
||||||
|
rule add_electricity:
|
||||||
|
input:
|
||||||
|
base_network='networks/base_{opts}.h5',
|
||||||
|
supply_regions='data/supply_regions/supply_regions.shp',
|
||||||
|
load='data/SystemEnergy2009_13.csv',
|
||||||
|
wind_pv_profiles='data/Wind_PV_Normalised_Profiles.xlsx',
|
||||||
|
wind_area='resources/area_wind_{resarea}.csv',
|
||||||
|
solar_area='resources/area_solar_{resarea}.csv',
|
||||||
|
existing_generators="data/Existing Power Stations SA.xlsx",
|
||||||
|
hydro_inflow="resources/hydro_inflow.csv",
|
||||||
|
tech_costs="data/technology_costs.xlsx"
|
||||||
|
output: "networks/elec_{cost}_{resarea}_{opts}.h5"
|
||||||
|
benchmark: "benchmarks/add_electricity/elec_{resarea}_{opts}"
|
||||||
|
threads: 1
|
||||||
|
resources: mem_mb=1000
|
||||||
|
script: "scripts/add_electricity.py"
|
||||||
|
|
||||||
|
rule add_sectors:
|
||||||
|
input:
|
||||||
|
network="networks/elec_{cost}_{resarea}_{opts}.h5",
|
||||||
|
emobility="data/emobility"
|
||||||
|
output: "networks/sector_{cost}_{resarea}_{sectors}_{opts}.h5"
|
||||||
|
benchmark: "benchmarks/add_sectors/sector_{resarea}_{sectors}_{opts}"
|
||||||
|
threads: 1
|
||||||
|
resources: mem_mb=1000
|
||||||
|
script: "scripts/add_sectors.py"
|
||||||
|
|
||||||
|
rule solve_network:
|
||||||
|
input: network="networks/sector_{cost}_{resarea}_{sectors}_{opts}.h5"
|
||||||
|
output: "results/version-{version}/networks/{{cost}}_{{resarea}}_{{sectors}}_{{opts}}.h5".format(version=config['version'])
|
||||||
|
shadow: "shallow"
|
||||||
|
log:
|
||||||
|
gurobi="logs/{cost}_{resarea}_{sectors}_{opts}_gurobi.log",
|
||||||
|
python="logs/{cost}_{resarea}_{sectors}_{opts}_python.log"
|
||||||
|
benchmark: "benchmarks/solve_network/{cost}_{resarea}_{sectors}_{opts}"
|
||||||
|
threads: 4
|
||||||
|
resources: mem_mb=19000 # for electricity only
|
||||||
|
script: "scripts/solve_network.py"
|
||||||
|
|
||||||
|
rule plot_network:
|
||||||
|
input:
|
||||||
|
network='results/version-{version}/networks/{{cost}}_{{resarea}}_{{sectors}}_{{opts}}.h5'.format(version=config['version']),
|
||||||
|
supply_regions='data/supply_regions/supply_regions.shp',
|
||||||
|
resarea=lambda w: config['data']['resarea'][w.resarea]
|
||||||
|
output:
|
||||||
|
only_map=touch('results/version-{version}/plots/network_{{cost}}_{{resarea}}_{{sectors}}_{{opts}}_{{attr}}'.format(version=config['version'])),
|
||||||
|
ext=touch('results/version-{version}/plots/network_{{cost}}_{{resarea}}_{{sectors}}_{{opts}}_{{attr}}_ext'.format(version=config['version']))
|
||||||
|
params: ext=['png', 'pdf']
|
||||||
|
script: "scripts/plot_network.py"
|
||||||
|
|
||||||
|
# rule plot_costs:
|
||||||
|
# input: 'results/summaries/costs2-summary.csv'
|
||||||
|
# output:
|
||||||
|
# expand('results/plots/costs_{cost}_{resarea}_{sectors}_{opt}',
|
||||||
|
# **dict(chain(config['scenario'].items(), (('{param}')))
|
||||||
|
# touch('results/plots/scenario_plots')
|
||||||
|
# params:
|
||||||
|
# tmpl="results/plots/costs_[cost]_[resarea]_[sectors]_[opt]"
|
||||||
|
# exts=["pdf", "png"]
|
||||||
|
# scripts: "scripts/plot_costs.py"
|
||||||
|
|
||||||
|
rule scenario_comparison:
|
||||||
|
input:
|
||||||
|
expand('results/version-{version}/plots/network_{cost}_{sectors}_{opts}_{attr}_ext',
|
||||||
|
version=config['version'],
|
||||||
|
attr=['p_nom'],
|
||||||
|
**config['scenario'])
|
||||||
|
output:
|
||||||
|
html='results/version-{version}/plots/scenario_{{param}}.html'.format(version=config['version'])
|
||||||
|
params:
|
||||||
|
tmpl="network_[cost]_[resarea]_[sectors]_[opts]_[attr]_ext",
|
||||||
|
plot_dir='results/version-{}/plots'.format(config['version'])
|
||||||
|
script: "scripts/scenario_comparison.py"
|
||||||
|
|
||||||
|
rule extract_summaries:
|
||||||
|
input:
|
||||||
|
expand("results/version-{version}/networks/{cost}_{sectors}_{opts}.h5",
|
||||||
|
version=config['version'],
|
||||||
|
**config['scenario'])
|
||||||
|
output:
|
||||||
|
**{n: "results/version-{version}/summaries/{}-summary.csv".format(n, version=config['version'])
|
||||||
|
for n in ['costs', 'costs2', 'e_curtailed', 'e_nom_opt', 'e', 'p_nom_opt']}
|
||||||
|
params:
|
||||||
|
scenario_tmpl="[cost]_[resarea]_[sectors]_[opts]",
|
||||||
|
scenarios=config['scenario']
|
||||||
|
script: "scripts/extract_summaries.py"
|
||||||
|
|
||||||
|
|
||||||
|
# Local Variables:
|
||||||
|
# mode: python
|
||||||
|
# End:
|
||||||
5
cluster.yaml
Normal file
5
cluster.yaml
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
__default__:
|
||||||
|
partition: x-men
|
||||||
|
name: "pypsa-za.{rule}.{wildcards}"
|
||||||
|
output: "logs/cluster/{rule}.{wildcards}.out"
|
||||||
|
error: "logs/cluster/{rule}.{wildcards}.err"
|
||||||
191
config.yaml
Normal file
191
config.yaml
Normal file
@ -0,0 +1,191 @@
|
|||||||
|
version: 0.1
|
||||||
|
logging_level: INFO
|
||||||
|
|
||||||
|
scenario:
|
||||||
|
sectors: [E] # ,E+EV,E+BEV,E+BEV+V2G] # [ E+EV, E+BEV, E+BEV+V2G ]
|
||||||
|
cost: [diw2030]
|
||||||
|
lv: [1., 1.125, 1.25, 1.5, 2.0, 3.0]
|
||||||
|
opts: [Co2L, Co2L-T] #, LC-FL, LC-T, Ep-T, Co2L-T]
|
||||||
|
|
||||||
|
countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK']
|
||||||
|
|
||||||
|
historical_year: "2012"
|
||||||
|
|
||||||
|
electricity:
|
||||||
|
voltages: [220., 300., 380.]
|
||||||
|
|
||||||
|
co2limit: xxx
|
||||||
|
|
||||||
|
extendable_carriers:
|
||||||
|
Generator: [OCGT]
|
||||||
|
StorageUnit: [Battery, H2] # [CAES]
|
||||||
|
|
||||||
|
SAFE_reservemargin: 0.1
|
||||||
|
|
||||||
|
max_hours:
|
||||||
|
Battery: 3
|
||||||
|
H2: 10
|
||||||
|
|
||||||
|
reanalysis:
|
||||||
|
cutout: europe_2011_2016
|
||||||
|
|
||||||
|
landusetype_percent:
|
||||||
|
wind:
|
||||||
|
- [[7, 8, 9, 41], 80]
|
||||||
|
# - [[5, 6], 50]
|
||||||
|
# - [[11, 12, 14, 15], 10]
|
||||||
|
solar:
|
||||||
|
- [[7, 8, 9, 41], 80]
|
||||||
|
# - [[11, 12, 14, 15], 50]
|
||||||
|
# - [[46, 47, 51, 56, 64, 68, 72], 10]
|
||||||
|
|
||||||
|
capacity_per_sqm:
|
||||||
|
wind: 5 # half of 10 (IWES)
|
||||||
|
solar: 16.5 # half of 33 (IWES)
|
||||||
|
|
||||||
|
lines:
|
||||||
|
types:
|
||||||
|
220.: "Al/St 240/40 2-bundle 220.0"
|
||||||
|
300.: "Al/St 240/40 3-bundle 300.0"
|
||||||
|
380.: "Al/St 240/40 4-bundle 380.0"
|
||||||
|
|
||||||
|
s_max_pu: 0.7
|
||||||
|
length_factor: 1.25
|
||||||
|
|
||||||
|
links:
|
||||||
|
s_max_pu: 0.7
|
||||||
|
|
||||||
|
transformers:
|
||||||
|
x: 0.1
|
||||||
|
s_nom: 2000.
|
||||||
|
type: ''
|
||||||
|
|
||||||
|
costs:
|
||||||
|
discountrate: 0.07
|
||||||
|
|
||||||
|
# Marginal and capital costs can be overwritten
|
||||||
|
# capital_cost:
|
||||||
|
# Wind: Bla
|
||||||
|
marginal_cost: #
|
||||||
|
PV: 0.01
|
||||||
|
Wind: 0.015
|
||||||
|
EUR_to_ZAR: 15.63
|
||||||
|
|
||||||
|
emission_prices: # only used with the option Ep (emission prices)
|
||||||
|
# Externality costs from Integrated Energy Plan by the ZA DOE
|
||||||
|
co2: 0.27e+3
|
||||||
|
sox: 7.6e+3
|
||||||
|
nox: 4.5e+3
|
||||||
|
hg: 41484.e-6 # is also part of the excel sheet
|
||||||
|
particulate: 11.3e+3
|
||||||
|
|
||||||
|
solving:
|
||||||
|
options:
|
||||||
|
formulation: kirchhoff
|
||||||
|
clip_p_max_pu: 1.e-2
|
||||||
|
load_shedding: true
|
||||||
|
noisy_costs: true
|
||||||
|
min_iterations: 4
|
||||||
|
max_iterations: 8
|
||||||
|
# max_iterations: 1
|
||||||
|
# nhours: 10
|
||||||
|
solver:
|
||||||
|
name: gurobi_persistent
|
||||||
|
threads: 4
|
||||||
|
method: 2
|
||||||
|
crossover: 0 # -1 (Choose freely)
|
||||||
|
BarConvTol: 1.e-5
|
||||||
|
FeasibilityTol: 1.e-6
|
||||||
|
LogToConsole: 0
|
||||||
|
OutputFlag: 1
|
||||||
|
|
||||||
|
plotting:
|
||||||
|
map:
|
||||||
|
figsize: [7, 7]
|
||||||
|
boundaries: [16, -35, 33, -22]
|
||||||
|
p_nom:
|
||||||
|
bus_size_factor: 5.e+4
|
||||||
|
linewidth_factor: 3.e+3 # 1.e+3 #3.e+3
|
||||||
|
|
||||||
|
costs_max: 800
|
||||||
|
|
||||||
|
vre_techs: ["Wind", "PV"]
|
||||||
|
conv_techs: ["OCGT", "CCGT", "Nuclear", "Coal"]
|
||||||
|
storage_techs: ["Hydro", "CAES", "Battery", "Pumped storage", "Hydro+PS"]
|
||||||
|
store_techs: ["Li ion", "water tanks"]
|
||||||
|
load_carriers: ["AC load", "heat load", "Li ion load"]
|
||||||
|
AC_carriers: ["AC line", "AC transformer"]
|
||||||
|
link_carriers: ["DC line", "Converter AC-DC"]
|
||||||
|
heat_links: ["heat pump", "resistive heater", "CHP heat", "CHP electric",
|
||||||
|
"gas boiler", "central heat pump", "central resistive heater", "central CHP heat",
|
||||||
|
"central CHP electric", "central gas boiler"]
|
||||||
|
heat_generators: ["gas boiler", "central gas boiler", "solar thermal collector", "central solar thermal collector"]
|
||||||
|
tech_colors:
|
||||||
|
Wind: "xkcd:azure"
|
||||||
|
Hydro: "g"
|
||||||
|
ror: "g"
|
||||||
|
Hydro+PS: "g"
|
||||||
|
PV: "y"
|
||||||
|
OCGT: "brown"
|
||||||
|
OCGT marginal: "sandybrown"
|
||||||
|
OCGT-heat: "orange"
|
||||||
|
central gas boiler: "orange"
|
||||||
|
gas boiler: "orange"
|
||||||
|
gas boilers: "orange"
|
||||||
|
gas boiler marginal: "orange"
|
||||||
|
gas: "brown"
|
||||||
|
lines: "k"
|
||||||
|
AC line: "k"
|
||||||
|
AC-AC: "k"
|
||||||
|
transmission lines: "k"
|
||||||
|
H2: "m"
|
||||||
|
hydrogen storage: "m"
|
||||||
|
Battery: "slategray"
|
||||||
|
battery storage: "slategray"
|
||||||
|
CAES: "lightgray"
|
||||||
|
Nuclear: "r"
|
||||||
|
Nuclear marginal: "r"
|
||||||
|
Coal: "k"
|
||||||
|
Coal marginal: "k"
|
||||||
|
Lignite: "grey"
|
||||||
|
Lignite marginal: "grey"
|
||||||
|
CCGT: "orange"
|
||||||
|
CCGT marginal: "orange"
|
||||||
|
Diesel: "darkred"
|
||||||
|
Diesel marginal: "darkred"
|
||||||
|
heat pumps: "green"
|
||||||
|
heat pump: "green"
|
||||||
|
central heat pump: "green"
|
||||||
|
resistive heater: "pink"
|
||||||
|
central resistive heater: "pink"
|
||||||
|
Sabatier: "turquoise"
|
||||||
|
water tanks: "w"
|
||||||
|
CHP: "r"
|
||||||
|
CHP heat: "r"
|
||||||
|
CHP electric: "r"
|
||||||
|
central CHP heat: "r"
|
||||||
|
central CHP electric: "r"
|
||||||
|
Pumped storage: "g"
|
||||||
|
Ambient: "k"
|
||||||
|
AC load: "b"
|
||||||
|
Heat load: "r"
|
||||||
|
Li ion load: "grey"
|
||||||
|
heat: "r"
|
||||||
|
Li ion: "grey"
|
||||||
|
district heating: "#CC4E5C"
|
||||||
|
nice_names:
|
||||||
|
# OCGT: "Gas"
|
||||||
|
# OCGT marginal: "Gas (marginal)"
|
||||||
|
#Battery: "Battery storage"
|
||||||
|
lines: "Transmission lines"
|
||||||
|
AC line: "AC lines"
|
||||||
|
AC-AC: "DC lines"
|
||||||
|
ror: "Run of river"
|
||||||
|
nice_names_n:
|
||||||
|
offwind: "offshore\nwind"
|
||||||
|
onwind: "onshore\nwind"
|
||||||
|
# OCGT: "Gas"
|
||||||
|
H2: "Hydrogen\nstorage"
|
||||||
|
# OCGT marginal: "Gas (marginal)"
|
||||||
|
lines: "transmission\nlines"
|
||||||
|
ror: "run of river"
|
||||||
25
scripts/prepare_links_p_nom.py
Normal file
25
scripts/prepare_links_p_nom.py
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
links_p_nom = pd.read_html('https://en.wikipedia.org/wiki/List_of_HVDC_projects', header=0, match="SwePol")[0]
|
||||||
|
|
||||||
|
def extract_coordinates(s):
|
||||||
|
regex = (r"(\d{1,2})°(\d{1,2})′(\d{1,2})″(N|S) "
|
||||||
|
r"(\d{1,2})°(\d{1,2})′(\d{1,2})″(E|W)")
|
||||||
|
e = s.str.extract(regex, expand=True)
|
||||||
|
lat = (e[0].astype(float) + (e[1].astype(float) + e[2].astype(float)/60.)/60.)*e[3].map({'N': +1., 'S': -1.})
|
||||||
|
lon = (e[4].astype(float) + (e[5].astype(float) + e[6].astype(float)/60.)/60.)*e[7].map({'E': +1., 'W': -1.})
|
||||||
|
return lon, lat
|
||||||
|
|
||||||
|
m_b = links_p_nom["Power (MW)"].str.contains('x').fillna(False)
|
||||||
|
def multiply(s): return s.str[0].astype(float) * s.str[1].astype(float)
|
||||||
|
links_p_nom.loc[m_b, "Power (MW)"] = links_p_nom.loc[m_b, "Power (MW)"].str.split('x').pipe(multiply)
|
||||||
|
links_p_nom["Power (MW)"] = links_p_nom["Power (MW)"].str.extract("[-/]?([\d.]+)", expand=False).astype(float)
|
||||||
|
|
||||||
|
links_p_nom['x1'], links_p_nom['y1'] = extract_coordinates(links_p_nom['Converter station 1'])
|
||||||
|
links_p_nom['x2'], links_p_nom['y2'] = extract_coordinates(links_p_nom['Converter station 2'])
|
||||||
|
|
||||||
|
links_p_nom.dropna(subset=['x1', 'y1', 'x2', 'y2']).to_csv(snakemake.output[0], index=False)
|
||||||
|
|
||||||
4
snakemake_cluster
Executable file
4
snakemake_cluster
Executable file
@ -0,0 +1,4 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
snakemake --cluster-config cluster.yaml --cluster "sbatch --parsable -J '{cluster.name}' -p {cluster.partition} -n 1 --cpus-per-task {threads} -o '{cluster.output}' --mem {resources.mem_mb}" "$@"
|
||||||
|
|
||||||
Loading…
Reference in New Issue
Block a user