diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f0cdfca1..c2be3909 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,6 @@ on: - cron: "0 5 * * TUE" env: - CONDA_CACHE_NUMBER: 1 # Change this value to manually reset the environment cache DATA_CACHE_NUMBER: 2 jobs: @@ -27,22 +26,12 @@ jobs: strategy: fail-fast: false + max-parallel: 3 matrix: - include: - # Matrix required to handle caching with Mambaforge - - os: ubuntu-latest - label: ubuntu-latest - prefix: /usr/share/miniconda3/envs/pypsa-eur - - - os: macos-latest - label: macos-latest - prefix: /Users/runner/miniconda3/envs/pypsa-eur - - - os: windows-latest - label: windows-latest - prefix: C:\Miniconda3\envs\pypsa-eur - - name: ${{ matrix.label }} + os: + - ubuntu-latest + - macos-latest + - windows-latest runs-on: ${{ matrix.os }} @@ -60,24 +49,25 @@ jobs: - name: Add solver to environment run: | echo -e "- glpk\n- ipopt<3.13.3" >> envs/environment.yaml - if: ${{ matrix.label }} == 'windows-latest' + if: ${{ matrix.os }} == 'windows-latest' - name: Add solver to environment run: | echo -e "- glpk\n- ipopt" >> envs/environment.yaml - if: ${{ matrix.label }} != 'windows-latest' + if: ${{ matrix.os }} != 'windows-latest' - - name: Setup Mambaforge - uses: conda-incubator/setup-miniconda@v2 + - name: Setup micromamba + uses: mamba-org/setup-micromamba@v1 with: - miniforge-variant: Mambaforge - miniforge-version: latest - activate-environment: pypsa-eur - use-mamba: true + micromamba-version: latest + environment-file: envs/environment.yaml + log-level: debug + init-shell: bash + cache-environment: true + cache-downloads: true - name: Set cache dates run: | - echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV - name: Cache data and cutouts folders @@ -88,21 +78,8 @@ jobs: cutouts key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }} - - name: Create environment cache - uses: actions/cache@v3 - id: cache - with: - path: ${{ matrix.prefix }} - key: ${{ matrix.label }}-conda-${{ env.DATE }}-${{ env.CONDA_CACHE_NUMBER }} - - - name: Update environment due to outdated or unavailable cache - run: mamba env update -n pypsa-eur -f envs/environment.yaml - if: steps.cache.outputs.cache-hit != 'true' - - name: Test snakemake workflow run: | - conda activate pypsa-eur - conda list snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6fb0dbf0..006673b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -87,6 +87,6 @@ repos: # Check for FSFE REUSE compliance (licensing) - repo: https://github.com/fsfe/reuse-tool - rev: v2.0.0 + rev: v2.1.0 hooks: - id: reuse diff --git a/.sync-send b/.sync-send new file mode 100644 index 00000000..72252956 --- /dev/null +++ b/.sync-send @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + +rules +scripts +config +config/test +envs +matplotlibrc +Snakefile diff --git a/.syncignore-receive b/.syncignore-receive deleted file mode 100644 index d2e9b76d..00000000 --- a/.syncignore-receive +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors -# -# SPDX-License-Identifier: CC0-1.0 - -.snakemake -.git -.pytest_cache -.ipynb_checkpoints -.vscode -.DS_Store -__pycache__ -*.pyc -*.pyo -*.ipynb -notebooks -doc -cutouts -data -benchmarks -*.nc -configs diff --git a/.syncignore-send b/.syncignore-send deleted file mode 100644 index 4e6e9a8c..00000000 --- a/.syncignore-send +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors -# -# SPDX-License-Identifier: CC0-1.0 - -.snakemake -.git -.pytest_cache -.ipynb_checkpoints -.vscode -.DS_Store -__pycache__ -*.pyc -*.pyo -*.ipynb -notebooks -benchmarks -logs -resources* -results -networks* -cutouts -data/bundle -doc diff --git a/CITATION.cff b/CITATION.cff index 712a02f3..c80b73ef 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -6,7 +6,7 @@ cff-version: 1.1.0 message: "If you use this package, please cite it in the following way." title: "PyPSA-Eur: An open sector-coupled optimisation model of the European energy system" repository: https://github.com/pypsa/pypsa-eur -version: 0.8.0 +version: 0.8.1 license: MIT authors: - family-names: Brown diff --git a/README.md b/README.md index 79cdfa65..9691abc4 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ SPDX-License-Identifier: CC-BY-4.0 ![Size](https://img.shields.io/github/repo-size/pypsa/pypsa-eur) [![Zenodo PyPSA-Eur](https://zenodo.org/badge/DOI/10.5281/zenodo.3520874.svg)](https://doi.org/10.5281/zenodo.3520874) [![Zenodo PyPSA-Eur-Sec](https://zenodo.org/badge/DOI/10.5281/zenodo.3938042.svg)](https://doi.org/10.5281/zenodo.3938042) -[![Snakemake](https://img.shields.io/badge/snakemake-≥5.0.0-brightgreen.svg?style=flat)](https://snakemake.readthedocs.io) +[![Snakemake](https://img.shields.io/badge/snakemake-≥7.7.0-brightgreen.svg?style=flat)](https://snakemake.readthedocs.io) [![REUSE status](https://api.reuse.software/badge/github.com/pypsa/pypsa-eur)](https://api.reuse.software/info/github.com/pypsa/pypsa-eur) [![Stack Exchange questions](https://img.shields.io/stackexchange/stackoverflow/t/pypsa)](https://stackoverflow.com/questions/tagged/pypsa) @@ -35,17 +35,18 @@ The model is designed to be imported into the open toolbox [PyPSA](https://github.com/PyPSA/PyPSA). **WARNING**: PyPSA-Eur is under active development and has several -[limitations](https://pypsa-eur.readthedocs.io/en/latest/limitations.html) -which you should understand before using the model. The github repository +[limitations](https://pypsa-eur.readthedocs.io/en/latest/limitations.html) which +you should understand before using the model. The github repository [issues](https://github.com/PyPSA/pypsa-eur/issues) collect known topics we are working on (please feel free to help or make suggestions). The [documentation](https://pypsa-eur.readthedocs.io/) remains somewhat patchy. You -can find showcases of the model's capabilities in the preprint [Benefits of a -Hydrogen Network in Europe](https://arxiv.org/abs/2207.05816), a [paper in Joule -with a description of the industry sector](https://arxiv.org/abs/2109.09563), or -in [a 2021 presentation at EMP-E](https://nworbmot.org/energy/brown-empe.pdf). -We cannot support this model if you choose to use it. We do not recommend to use -the full resolution network model for simulations. At high granularity the +can find showcases of the model's capabilities in the Joule paper [The potential +role of a hydrogen network in +Europe](https://doi.org/10.1016/j.joule.2023.06.016), another [paper in Joule +with a description of the industry +sector](https://doi.org/10.1016/j.joule.2022.04.016), or in [a 2021 presentation +at EMP-E](https://nworbmot.org/energy/brown-empe.pdf). We do not recommend to +use the full resolution network model for simulations. At high granularity the assignment of loads and generators to the nearest network node may not be a correct assumption, depending on the topology of the underlying distribution grid, and local grid bottlenecks may cause unrealistic load-shedding or diff --git a/Snakefile b/Snakefile index 6d694aea..2770f0cf 100644 --- a/Snakefile +++ b/Snakefile @@ -40,7 +40,7 @@ localrules: wildcard_constraints: simpl="[a-zA-Z0-9]*", - clusters="[0-9]+m?|all", + clusters="[0-9]+(m|c)?|all", ll="(v|c)([0-9\.]+|opt)", opts="[-+a-zA-Z0-9\.]*", sector_opts="[-+a-zA-Z0-9\.\s]*", @@ -53,6 +53,7 @@ include: "rules/build_electricity.smk" include: "rules/build_sector.smk" include: "rules/solve_electricity.smk" include: "rules/postprocess.smk" +include: "rules/validate.smk" if config["foresight"] == "overnight": @@ -103,3 +104,14 @@ rule doc: directory("doc/_build"), shell: "make -C doc html" + + +rule sync: + params: + cluster=f"{config['remote']['ssh']}:{config['remote']['path']}", + shell: + """ + rsync -uvarh --ignore-missing-args --files-from=.sync-send . {params.cluster} + rsync -uvarh --no-g {params.cluster}/results . || echo "No results directory, skipping rsync" + rsync -uvarh --no-g {params.cluster}/logs . || echo "No logs directory, skipping rsync" + """ diff --git a/config/config.default.yaml b/config/config.default.yaml index bf5156b9..8b88db93 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -3,13 +3,21 @@ # SPDX-License-Identifier: CC0-1.0 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#top-level-configuration -version: 0.8.0 +version: 0.8.1 tutorial: false logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' +private: + keys: + entsoe_api: + +remote: + ssh: "" + path: "" + # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run run: name: "" @@ -18,7 +26,7 @@ run: shared_cutouts: true # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight -foresight: overnight +foresight: perfect # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#scenario # Wildcard docs in https://pypsa-eur.readthedocs.io/en/latest/wildcards.html @@ -26,21 +34,22 @@ scenario: simpl: - '' ll: + - v1.0 - v1.5 clusters: - 37 - 128 - - 256 - - 512 - - 1024 +# - 256 +# - 512 + #- 1024 opts: - '' sector_opts: - Co2L0-3H-T-H-B-I-A-solar+p3-dist1 planning_horizons: - # - 2020 - # - 2030 - # - 2040 + - 2020 + - 2030 + - 2040 - 2050 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#countries @@ -209,10 +218,14 @@ renewable: carriers: [ror, PHS, hydro] PHS_max_hours: 6 hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float + flatten_dispatch: false + flatten_dispatch_buffer: 0.2 clip_min_inflow: 1.0 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#conventional conventional: + unit_commitment: false + dynamic_fuel_price: false nuclear: p_max_pu: "data/nuclear_p_max_pu.csv" # float of file name @@ -227,6 +240,12 @@ lines: max_extension: .inf length_factor: 1.25 under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity + dynamic_line_rating: + activate: false + cutout: europe-2013-era5 + correction_factor: 0.95 + max_voltage_difference: false + max_line_rating: false # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#links links: @@ -528,7 +547,7 @@ industry: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#costs costs: year: 2030 - version: v0.5.0 + version: v0.6.0 rooftop_share: 0.14 # based on the potentials, assuming (0.1 kW/m2 and 10 m2/person) fill_values: FOM: 0 @@ -568,16 +587,12 @@ clustering: algorithm: kmeans feature: solar+onwind-time exclude_carriers: [] + consider_efficiency_classes: false aggregation_strategies: generators: - p_nom_max: sum - p_nom_min: sum - p_min_pu: mean - marginal_cost: mean committable: any ramp_limit_up: max ramp_limit_down: max - efficiency: mean # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#solving solving: @@ -585,13 +600,17 @@ solving: options: clip_p_max_pu: 1.e-2 load_shedding: false - transmission_losses: 0 noisy_costs: true skip_iterations: true + rolling_horizon: false + seed: 123 + # options that go into the optimize function track_iterations: false min_iterations: 4 max_iterations: 6 - seed: 123 + transmission_losses: 0 + linearized_unit_commitment: true + horizon: 365 solver: name: gurobi @@ -619,7 +638,6 @@ solving: AggFill: 0 PreDual: 0 GURO_PAR_BARDENSETHRESH: 200 - seed: 10 # Consistent seed for all plattforms gurobi-numeric-focus: name: gurobi NumericFocus: 3 # Favour numeric stability over speed @@ -652,6 +670,7 @@ solving: glpk-default: {} # Used in CI mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 + walltime: "12:00:00" # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#plotting plotting: @@ -682,6 +701,8 @@ plotting: H2: "Hydrogen Storage" lines: "Transmission Lines" ror: "Run of River" + ac: "AC" + dc: "DC" tech_colors: # wind diff --git a/config/config.validation.yaml b/config/config.validation.yaml new file mode 100644 index 00000000..5bcd5c31 --- /dev/null +++ b/config/config.validation.yaml @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 +run: + name: "validation" + +scenario: + ll: + - v1.0 + clusters: + - 37 + opts: + - 'Ept' + +snapshots: + start: "2019-01-01" + end: "2020-01-01" + inclusive: 'left' + +enable: + retrieve_cutout: false + +electricity: + co2limit: 1e9 + + extendable_carriers: + Generator: [] + StorageUnit: [] + Store: [] + Link: [] + + powerplants_filter: not (DateOut < 2019) + + conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] + renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro] + + estimate_renewable_capacities: + year: 2019 + +atlite: + default_cutout: europe-2019-era5 + cutouts: + europe-2019-era5: + module: era5 + x: [-12., 35.] + y: [33., 72] + dx: 0.3 + dy: 0.3 + time: ['2019', '2019'] + +renewable: + onwind: + cutout: europe-2019-era5 + offwind-ac: + cutout: europe-2019-era5 + offwind-dc: + cutout: europe-2019-era5 + solar: + cutout: europe-2019-era5 + hydro: + cutout: europe-2019-era5 + flatten_dispatch: 0.01 + +conventional: + unit_commitment: false + dynamic_fuel_price: true + nuclear: + p_max_pu: "data/nuclear_p_max_pu.csv" + biomass: + p_max_pu: 0.65 + +load: + power_statistics: false + +lines: + s_max_pu: 0.23 + under_construction: 'remove' + +links: + include_tyndp: false + +costs: + year: 2020 + emission_prices: + co2: 25 + +clustering: + simplify_network: + exclude_carriers: [oil, coal, lignite, OCGT, CCGT] + cluster_network: + consider_efficiency_classes: true + +solving: + options: + load_shedding: true + rolling_horizon: false + horizon: 1000 + overlap: 48 diff --git a/config/test/config.electricity.yaml b/config/test/config.electricity.yaml index 6798e38c..b750bf62 100644 --- a/config/test/config.electricity.yaml +++ b/config/test/config.electricity.yaml @@ -60,6 +60,12 @@ renewable: clustering: exclude_carriers: ["OCGT", "offwind-ac", "coal"] +lines: + dynamic_line_rating: + activate: true + cutout: be-03-2013-era5 + max_line_rating: 1.3 + solving: solver: diff --git a/data/costs.csv b/data/costs.csv deleted file mode 100644 index 8953eb8a..00000000 --- a/data/costs.csv +++ /dev/null @@ -1,195 +0,0 @@ -technology,year,parameter,value,unit,source -solar-rooftop,2030,discount rate,0.04,per unit,standard for decentral -onwind,2030,lifetime,30,years,DEA https://ens.dk/en/our-services/projections-and-models/technology-data -offwind,2030,lifetime,30,years,DEA https://ens.dk/en/our-services/projections-and-models/technology-data -solar,2030,lifetime,25,years,IEA2010 -solar-rooftop,2030,lifetime,25,years,IEA2010 -solar-utility,2030,lifetime,25,years,IEA2010 -PHS,2030,lifetime,80,years,IEA2010 -hydro,2030,lifetime,80,years,IEA2010 -ror,2030,lifetime,80,years,IEA2010 -OCGT,2030,lifetime,30,years,IEA2010 -nuclear,2030,lifetime,45,years,ECF2010 in DIW DataDoc http://hdl.handle.net/10419/80348 -CCGT,2030,lifetime,30,years,IEA2010 -coal,2030,lifetime,40,years,IEA2010 -lignite,2030,lifetime,40,years,IEA2010 -geothermal,2030,lifetime,40,years,IEA2010 -biomass,2030,lifetime,30,years,ECF2010 in DIW DataDoc http://hdl.handle.net/10419/80348 -oil,2030,lifetime,30,years,ECF2010 in DIW DataDoc http://hdl.handle.net/10419/80348 -onwind,2030,investment,1040,EUR/kWel,DEA https://ens.dk/en/our-services/projections-and-models/technology-data -offwind,2030,investment,1640,EUR/kWel,DEA https://ens.dk/en/our-services/projections-and-models/technology-data -offwind-ac-station,2030,investment,250,EUR/kWel,DEA https://ens.dk/en/our-services/projections-and-models/technology-data -offwind-ac-connection-submarine,2030,investment,2685,EUR/MW/km,DEA https://ens.dk/en/our-services/projections-and-models/technology-data -offwind-ac-connection-underground,2030,investment,1342,EUR/MW/km,DEA https://ens.dk/en/our-services/projections-and-models/technology-data -offwind-dc-station,2030,investment,400,EUR/kWel,Haertel 2017; assuming one onshore and one offshore node + 13% learning reduction -offwind-dc-connection-submarine,2030,investment,2000,EUR/MW/km,DTU report based on Fig 34 of https://ec.europa.eu/energy/sites/ener/files/documents/2014_nsog_report.pdf -offwind-dc-connection-underground,2030,investment,1000,EUR/MW/km,Haertel 2017; average + 13% learning reduction -solar,2030,investment,600,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348 -biomass,2030,investment,2209,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348 -geothermal,2030,investment,3392,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348 -coal,2030,investment,1300,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348 PC (Advanced/SuperC) -lignite,2030,investment,1500,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348 -solar-rooftop,2030,investment,725,EUR/kWel,ETIP PV -solar-utility,2030,investment,425,EUR/kWel,ETIP PV -PHS,2030,investment,2000,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348 -hydro,2030,investment,2000,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348 -ror,2030,investment,3000,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348 -OCGT,2030,investment,400,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348 -nuclear,2030,investment,6000,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348 -CCGT,2030,investment,800,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348 -oil,2030,investment,400,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348 -onwind,2030,FOM,2.450549,%/year,DEA https://ens.dk/en/our-services/projections-and-models/technology-data -offwind,2030,FOM,2.304878,%/year,DEA https://ens.dk/en/our-services/projections-and-models/technology-data -solar,2030,FOM,4.166667,%/year,DIW DataDoc http://hdl.handle.net/10419/80348 -solar-rooftop,2030,FOM,2,%/year,ETIP PV -solar-utility,2030,FOM,3,%/year,ETIP PV -biomass,2030,FOM,4.526935,%/year,DIW DataDoc http://hdl.handle.net/10419/80348 -geothermal,2030,FOM,2.358491,%/year,DIW DataDoc http://hdl.handle.net/10419/80348 -coal,2030,FOM,1.923076,%/year,DIW DataDoc http://hdl.handle.net/10419/80348 PC (Advanced/SuperC) -lignite,2030,FOM,2.0,%/year,DIW DataDoc http://hdl.handle.net/10419/80348 PC (Advanced/SuperC) -oil,2030,FOM,1.5,%/year,DIW DataDoc http://hdl.handle.net/10419/80348 -PHS,2030,FOM,1,%/year,DIW DataDoc http://hdl.handle.net/10419/80348 -hydro,2030,FOM,1,%/year,DIW DataDoc http://hdl.handle.net/10419/80348 -ror,2030,FOM,2,%/year,DIW DataDoc http://hdl.handle.net/10419/80348 -CCGT,2030,FOM,2.5,%/year,DIW DataDoc http://hdl.handle.net/10419/80348 -OCGT,2030,FOM,3.75,%/year,DIW DataDoc http://hdl.handle.net/10419/80348 -onwind,2030,VOM,2.3,EUR/MWhel,DEA https://ens.dk/en/our-services/projections-and-models/technology-data -offwind,2030,VOM,2.7,EUR/MWhel,DEA https://ens.dk/en/our-services/projections-and-models/technology-data -solar,2030,VOM,0.01,EUR/MWhel,RES costs made up to fix curtailment order -coal,2030,VOM,6,EUR/MWhel,DIW DataDoc http://hdl.handle.net/10419/80348 PC (Advanced/SuperC) -lignite,2030,VOM,7,EUR/MWhel,DIW DataDoc http://hdl.handle.net/10419/80348 -CCGT,2030,VOM,4,EUR/MWhel,DIW DataDoc http://hdl.handle.net/10419/80348 -OCGT,2030,VOM,3,EUR/MWhel,DIW DataDoc http://hdl.handle.net/10419/80348 -nuclear,2030,VOM,8,EUR/MWhel,DIW DataDoc http://hdl.handle.net/10419/80348 -gas,2030,fuel,21.6,EUR/MWhth,IEA2011b -uranium,2030,fuel,3,EUR/MWhth,DIW DataDoc http://hdl.handle.net/10419/80348 -oil,2030,VOM,3,EUR/MWhel,DIW DataDoc http://hdl.handle.net/10419/80348 -nuclear,2030,fuel,3,EUR/MWhth,IEA2011b -biomass,2030,fuel,7,EUR/MWhth,IEA2011b -coal,2030,fuel,8.4,EUR/MWhth,IEA2011b -lignite,2030,fuel,2.9,EUR/MWhth,IEA2011b -oil,2030,fuel,50,EUR/MWhth,IEA WEM2017 97USD/boe = http://www.iea.org/media/weowebsite/2017/WEM_Documentation_WEO2017.pdf -PHS,2030,efficiency,0.75,per unit,DIW DataDoc http://hdl.handle.net/10419/80348 -hydro,2030,efficiency,0.9,per unit,DIW DataDoc http://hdl.handle.net/10419/80348 -ror,2030,efficiency,0.9,per unit,DIW DataDoc http://hdl.handle.net/10419/80348 -OCGT,2030,efficiency,0.39,per unit,DIW DataDoc http://hdl.handle.net/10419/80348 -CCGT,2030,efficiency,0.5,per unit,DIW DataDoc http://hdl.handle.net/10419/80348 -biomass,2030,efficiency,0.468,per unit,DIW DataDoc http://hdl.handle.net/10419/80348 -geothermal,2030,efficiency,0.239,per unit,DIW DataDoc http://hdl.handle.net/10419/80348 -nuclear,2030,efficiency,0.337,per unit,DIW DataDoc http://hdl.handle.net/10419/80348 -gas,2030,CO2 intensity,0.187,tCO2/MWth,https://www.eia.gov/environment/emissions/co2_vol_mass.php -coal,2030,efficiency,0.464,per unit,DIW DataDoc http://hdl.handle.net/10419/80348 PC (Advanced/SuperC) -lignite,2030,efficiency,0.447,per unit,DIW DataDoc http://hdl.handle.net/10419/80348 -oil,2030,efficiency,0.393,per unit,DIW DataDoc http://hdl.handle.net/10419/80348 CT -coal,2030,CO2 intensity,0.354,tCO2/MWth,https://www.eia.gov/environment/emissions/co2_vol_mass.php -lignite,2030,CO2 intensity,0.334,tCO2/MWth,https://www.eia.gov/environment/emissions/co2_vol_mass.php -oil,2030,CO2 intensity,0.248,tCO2/MWth,https://www.eia.gov/environment/emissions/co2_vol_mass.php -geothermal,2030,CO2 intensity,0.026,tCO2/MWth,https://www.eia.gov/environment/emissions/co2_vol_mass.php -electrolysis,2030,investment,350,EUR/kWel,Palzer Thesis -electrolysis,2030,FOM,4,%/year,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013 -electrolysis,2030,lifetime,18,years,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013 -electrolysis,2030,efficiency,0.8,per unit,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013 -fuel cell,2030,investment,339,EUR/kWel,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013 -fuel cell,2030,FOM,3,%/year,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013 -fuel cell,2030,lifetime,20,years,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013 -fuel cell,2030,efficiency,0.58,per unit,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013 conservative 2020 -hydrogen storage,2030,investment,11.2,USD/kWh,budischak2013 -hydrogen storage,2030,lifetime,20,years,budischak2013 -hydrogen underground storage,2030,investment,0.5,EUR/kWh,maximum from https://www.nrel.gov/docs/fy10osti/46719.pdf -hydrogen underground storage,2030,lifetime,40,years,http://www.acatech.de/fileadmin/user_upload/Baumstruktur_nach_Website/Acatech/root/de/Publikationen/Materialien/ESYS_Technologiesteckbrief_Energiespeicher.pdf -H2 pipeline,2030,investment,267,EUR/MW/km,Welder et al https://doi.org/10.1016/j.ijhydene.2018.12.156 -H2 pipeline,2030,lifetime,40,years,Krieg2012 http://juser.fz-juelich.de/record/136392/files/Energie%26Umwelt_144.pdf -H2 pipeline,2030,FOM,5,%/year,Krieg2012 http://juser.fz-juelich.de/record/136392/files/Energie%26Umwelt_144.pdf -H2 pipeline,2030,efficiency,0.98,per unit,Krieg2012 http://juser.fz-juelich.de/record/136392/files/Energie%26Umwelt_144.pdf -methanation,2030,investment,1000,EUR/kWH2,Schaber thesis -methanation,2030,lifetime,25,years,Schaber thesis -methanation,2030,FOM,3,%/year,Schaber thesis -methanation,2030,efficiency,0.6,per unit,Palzer; Breyer for DAC -helmeth,2030,investment,1000,EUR/kW,no source -helmeth,2030,lifetime,25,years,no source -helmeth,2030,FOM,3,%/year,no source -helmeth,2030,efficiency,0.8,per unit,HELMETH press release -DAC,2030,investment,250,EUR/(tCO2/a),Fasihi/Climeworks -DAC,2030,lifetime,30,years,Fasihi -DAC,2030,FOM,4,%/year,Fasihi -battery inverter,2030,investment,411,USD/kWel,budischak2013 -battery inverter,2030,lifetime,20,years,budischak2013 -battery inverter,2030,efficiency,0.9,per unit charge/discharge,budischak2013; Lund and Kempton (2008) http://dx.doi.org/10.1016/j.enpol.2008.06.007 -battery inverter,2030,FOM,3,%/year,budischak2013 -battery storage,2030,investment,192,USD/kWh,budischak2013 -battery storage,2030,lifetime,15,years,budischak2013 -decentral air-sourced heat pump,2030,investment,1050,EUR/kWth,HP; Palzer thesis -decentral air-sourced heat pump,2030,lifetime,20,years,HP; Palzer thesis -decentral air-sourced heat pump,2030,FOM,3.5,%/year,Palzer thesis -decentral air-sourced heat pump,2030,efficiency,3,per unit,default for costs -decentral air-sourced heat pump,2030,discount rate,0.04,per unit,Palzer thesis -decentral ground-sourced heat pump,2030,investment,1400,EUR/kWth,Palzer thesis -decentral ground-sourced heat pump,2030,lifetime,20,years,Palzer thesis -decentral ground-sourced heat pump,2030,FOM,3.5,%/year,Palzer thesis -decentral ground-sourced heat pump,2030,efficiency,4,per unit,default for costs -decentral ground-sourced heat pump,2030,discount rate,0.04,per unit,Palzer thesis -central air-sourced heat pump,2030,investment,700,EUR/kWth,Palzer thesis -central air-sourced heat pump,2030,lifetime,20,years,Palzer thesis -central air-sourced heat pump,2030,FOM,3.5,%/year,Palzer thesis -central air-sourced heat pump,2030,efficiency,3,per unit,default for costs -retrofitting I,2030,discount rate,0.04,per unit,Palzer thesis -retrofitting I,2030,lifetime,50,years,Palzer thesis -retrofitting I,2030,FOM,1,%/year,Palzer thesis -retrofitting I,2030,investment,50,EUR/m2/fraction reduction,Palzer thesis -retrofitting II,2030,discount rate,0.04,per unit,Palzer thesis -retrofitting II,2030,lifetime,50,years,Palzer thesis -retrofitting II,2030,FOM,1,%/year,Palzer thesis -retrofitting II,2030,investment,250,EUR/m2/fraction reduction,Palzer thesis -water tank charger,2030,efficiency,0.9,per unit,HP -water tank discharger,2030,efficiency,0.9,per unit,HP -decentral water tank storage,2030,investment,860,EUR/m3,IWES Interaktion -decentral water tank storage,2030,FOM,1,%/year,HP -decentral water tank storage,2030,lifetime,20,years,HP -decentral water tank storage,2030,discount rate,0.04,per unit,Palzer thesis -central water tank storage,2030,investment,30,EUR/m3,IWES Interaktion -central water tank storage,2030,FOM,1,%/year,HP -central water tank storage,2030,lifetime,40,years,HP -decentral resistive heater,2030,investment,100,EUR/kWhth,Schaber thesis -decentral resistive heater,2030,lifetime,20,years,Schaber thesis -decentral resistive heater,2030,FOM,2,%/year,Schaber thesis -decentral resistive heater,2030,efficiency,0.9,per unit,Schaber thesis -decentral resistive heater,2030,discount rate,0.04,per unit,Palzer thesis -central resistive heater,2030,investment,100,EUR/kWhth,Schaber thesis -central resistive heater,2030,lifetime,20,years,Schaber thesis -central resistive heater,2030,FOM,2,%/year,Schaber thesis -central resistive heater,2030,efficiency,0.9,per unit,Schaber thesis -decentral gas boiler,2030,investment,175,EUR/kWhth,Palzer thesis -decentral gas boiler,2030,lifetime,20,years,Palzer thesis -decentral gas boiler,2030,FOM,2,%/year,Palzer thesis -decentral gas boiler,2030,efficiency,0.9,per unit,Palzer thesis -decentral gas boiler,2030,discount rate,0.04,per unit,Palzer thesis -central gas boiler,2030,investment,63,EUR/kWhth,Palzer thesis -central gas boiler,2030,lifetime,22,years,Palzer thesis -central gas boiler,2030,FOM,1,%/year,Palzer thesis -central gas boiler,2030,efficiency,0.9,per unit,Palzer thesis -decentral CHP,2030,lifetime,25,years,HP -decentral CHP,2030,investment,1400,EUR/kWel,HP -decentral CHP,2030,FOM,3,%/year,HP -decentral CHP,2030,discount rate,0.04,per unit,Palzer thesis -central CHP,2030,lifetime,25,years,HP -central CHP,2030,investment,650,EUR/kWel,HP -central CHP,2030,FOM,3,%/year,HP -decentral solar thermal,2030,discount rate,0.04,per unit,Palzer thesis -decentral solar thermal,2030,FOM,1.3,%/year,HP -decentral solar thermal,2030,investment,270000,EUR/1000m2,HP -decentral solar thermal,2030,lifetime,20,years,HP -central solar thermal,2030,FOM,1.4,%/year,HP -central solar thermal,2030,investment,140000,EUR/1000m2,HP -central solar thermal,2030,lifetime,20,years,HP -HVAC overhead,2030,investment,400,EUR/MW/km,Hagspiel -HVAC overhead,2030,lifetime,40,years,Hagspiel -HVAC overhead,2030,FOM,2,%/year,Hagspiel -HVDC overhead,2030,investment,400,EUR/MW/km,Hagspiel -HVDC overhead,2030,lifetime,40,years,Hagspiel -HVDC overhead,2030,FOM,2,%/year,Hagspiel -HVDC submarine,2030,investment,2000,EUR/MW/km,DTU report based on Fig 34 of https://ec.europa.eu/energy/sites/ener/files/documents/2014_nsog_report.pdf -HVDC submarine,2030,lifetime,40,years,Hagspiel -HVDC submarine,2030,FOM,2,%/year,Hagspiel -HVDC inverter pair,2030,investment,150000,EUR/MW,Hagspiel -HVDC inverter pair,2030,lifetime,40,years,Hagspiel -HVDC inverter pair,2030,FOM,2,%/year,Hagspiel diff --git a/data/nuclear_p_max_pu.csv b/data/nuclear_p_max_pu.csv index 7bc54455..0fdb5e5b 100644 --- a/data/nuclear_p_max_pu.csv +++ b/data/nuclear_p_max_pu.csv @@ -1,16 +1,16 @@ country,factor -BE,0.65 -BG,0.89 -CZ,0.82 -FI,0.92 -FR,0.70 -DE,0.88 -HU,0.90 -NL,0.86 -RO,0.92 -SK,0.89 -SI,0.94 -ES,0.89 -SE,0.82 -CH,0.86 -GB,0.67 +BE,0.796 +BG,0.894 +CZ,0.827 +FI,0.936 +FR,0.71 +DE,0.871 +HU,0.913 +NL,0.868 +RO,0.909 +SK,0.9 +SI,0.913 +ES,0.897 +SE,0.851 +CH,0.87 +GB,0.656 diff --git a/data/override_component_attrs/buses.csv b/data/override_component_attrs/buses.csv deleted file mode 100644 index 7581e328..00000000 --- a/data/override_component_attrs/buses.csv +++ /dev/null @@ -1,3 +0,0 @@ -attribute,type,unit,default,description,status -location,string,n/a,n/a,Reference to original electricity bus,Input (optional) -unit,string,n/a,MWh,Unit of the bus (descriptive only), Input (optional) diff --git a/data/override_component_attrs/generators.csv b/data/override_component_attrs/generators.csv deleted file mode 100644 index 4f214396..00000000 --- a/data/override_component_attrs/generators.csv +++ /dev/null @@ -1,4 +0,0 @@ -attribute,type,unit,default,description,status -carrier,string,n/a,n/a,carrier,Input (optional) -lifetime,float,years,inf,lifetime,Input (optional) -build_year,int,year ,0,build year,Input (optional) diff --git a/data/override_component_attrs/links.csv b/data/override_component_attrs/links.csv deleted file mode 100644 index 0fc2747a..00000000 --- a/data/override_component_attrs/links.csv +++ /dev/null @@ -1,13 +0,0 @@ -attribute,type,unit,default,description,status -bus2,string,n/a,n/a,2nd bus,Input (optional) -bus3,string,n/a,n/a,3rd bus,Input (optional) -bus4,string,n/a,n/a,4th bus,Input (optional) -efficiency2,static or series,per unit,1,2nd bus efficiency,Input (optional) -efficiency3,static or series,per unit,1,3rd bus efficiency,Input (optional) -efficiency4,static or series,per unit,1,4th bus efficiency,Input (optional) -p2,series,MW,0,2nd bus output,Output -p3,series,MW,0,3rd bus output,Output -p4,series,MW,0,4th bus output,Output -carrier,string,n/a,n/a,carrier,Input (optional) -lifetime,float,years,inf,lifetime,Input (optional) -build_year,int,year ,0,build year,Input (optional) diff --git a/data/override_component_attrs/loads.csv b/data/override_component_attrs/loads.csv deleted file mode 100644 index 10bb5b4f..00000000 --- a/data/override_component_attrs/loads.csv +++ /dev/null @@ -1,2 +0,0 @@ -attribute,type,unit,default,description,status -carrier,string,n/a,n/a,carrier,Input (optional) diff --git a/data/override_component_attrs/stores.csv b/data/override_component_attrs/stores.csv deleted file mode 100644 index 4f214396..00000000 --- a/data/override_component_attrs/stores.csv +++ /dev/null @@ -1,4 +0,0 @@ -attribute,type,unit,default,description,status -carrier,string,n/a,n/a,carrier,Input (optional) -lifetime,float,years,inf,lifetime,Input (optional) -build_year,int,year ,0,build year,Input (optional) diff --git a/data/unit_commitment.csv b/data/unit_commitment.csv new file mode 100644 index 00000000..e93b1a90 --- /dev/null +++ b/data/unit_commitment.csv @@ -0,0 +1,8 @@ +attribute,OCGT,CCGT,coal,lignite,nuclear +ramp_limit_up,1,1,1,1,0.3 +ramp_limit_start_up,0.2,0.45,0.38,0.4,0.5 +ramp_limit_shut_down,0.2,0.45,0.38,0.4,0.5 +p_min_pu,0.2,0.45,0.325,0.4,0.5 +min_up_time,,3,5,7,6 +min_down_time,,2,6,6,10 +start_up_cost,9.6,34.2,35.64,19.14,16.5 diff --git a/doc/conf.py b/doc/conf.py index 8111c86c..1ddae466 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -82,7 +82,7 @@ author = "Tom Brown (KIT, TUB, FIAS), Jonas Hoersch (KIT, FIAS), Fabian Hofmann # The short X.Y version. version = "0.8" # The full version, including alpha/beta/rc tags. -release = "0.8.0" +release = "0.8.1" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/configtables/clustering.csv b/doc/configtables/clustering.csv index f13d8cbd..5f52c222 100644 --- a/doc/configtables/clustering.csv +++ b/doc/configtables/clustering.csv @@ -1,17 +1,18 @@ -,Unit,Values,Description -simplify_network,,, --- to_substations,bool,"{'true','false'}","Aggregates all nodes without power injection (positive or negative, i.e. demand or generation) to electrically closest ones" --- algorithm,str,"One of {‘kmeans’, ‘hac’, ‘modularity‘}", --- feature,str,"Str in the format ‘carrier1+carrier2+...+carrierN-X’, where CarrierI can be from {‘solar’, ‘onwind’, ‘offwind’, ‘ror’} and X is one of {‘cap’, ‘time’}.", --- exclude_carriers,list,"List of Str like [ 'solar', 'onwind'] or empy list []","List of carriers which will not be aggregated. If empty, all carriers will be aggregated." --- remove stubs,bool,"true/false","Controls whether radial parts of the network should be recursively aggregated. Defaults to true." --- remove_stubs_across_borders,bool,"true/false","Controls whether radial parts of the network should be recursively aggregated across borders. Defaults to true." -cluster_network,,, --- algorithm,str,"One of {‘kmeans’, ‘hac’}", --- feature,str,"Str in the format ‘carrier1+carrier2+...+carrierN-X’, where CarrierI can be from {‘solar’, ‘onwind’, ‘offwind’, ‘ror’} and X is one of {‘cap’, ‘time’}.", --- exclude_carriers,list,"List of Str like [ 'solar', 'onwind'] or empy list []","List of carriers which will not be aggregated. If empty, all carriers will be aggregated." -aggregation_strategies,,, --- generators,,, --- -- {key},str,"{key} can be any of the component of the generator (str). It’s value can be any that can be converted to pandas.Series using getattr(). For example one of {min, max, sum}.","Aggregates the component according to the given strategy. For example, if sum, then all values within each cluster are summed to represent the new generator." --- buses,,, --- -- {key},str,"{key} can be any of the component of the bus (str). It’s value can be any that can be converted to pandas.Series using getattr(). For example one of {min, max, sum}.","Aggregates the component according to the given strategy. For example, if sum, then all values within each cluster are summed to represent the new bus." +,Unit,Values,Description +simplify_network,,, +-- to_substations,bool,"{'true','false'}","Aggregates all nodes without power injection (positive or negative, i.e. demand or generation) to electrically closest ones" +-- algorithm,str,"One of {‘kmeans’, ‘hac’, ‘modularity‘}", +-- feature,str,"Str in the format ‘carrier1+carrier2+...+carrierN-X’, where CarrierI can be from {‘solar’, ‘onwind’, ‘offwind’, ‘ror’} and X is one of {‘cap’, ‘time’}.", +-- exclude_carriers,list,"List of Str like [ 'solar', 'onwind'] or empy list []","List of carriers which will not be aggregated. If empty, all carriers will be aggregated." +-- remove stubs,bool,"{'true','false'}",Controls whether radial parts of the network should be recursively aggregated. Defaults to true. +-- remove_stubs_across_borders,bool,"{'true','false'}",Controls whether radial parts of the network should be recursively aggregated across borders. Defaults to true. +cluster_network,,, +-- algorithm,str,"One of {‘kmeans’, ‘hac’}", +-- feature,str,"Str in the format ‘carrier1+carrier2+...+carrierN-X’, where CarrierI can be from {‘solar’, ‘onwind’, ‘offwind’, ‘ror’} and X is one of {‘cap’, ‘time’}.", +-- exclude_carriers,list,"List of Str like [ 'solar', 'onwind'] or empy list []","List of carriers which will not be aggregated. If empty, all carriers will be aggregated." +-- consider_efficiency_classes,bool,"{'true','false'}","Aggregated each carriers into the top 10-quantile (high), the bottom 90-quantile (low), and everything in between (medium)." +aggregation_strategies,,, +-- generators,,, +-- -- {key},str,"{key} can be any of the component of the generator (str). It’s value can be any that can be converted to pandas.Series using getattr(). For example one of {min, max, sum}.","Aggregates the component according to the given strategy. For example, if sum, then all values within each cluster are summed to represent the new generator." +-- buses,,, +-- -- {key},str,"{key} can be any of the component of the bus (str). It’s value can be any that can be converted to pandas.Series using getattr(). For example one of {min, max, sum}.","Aggregates the component according to the given strategy. For example, if sum, then all values within each cluster are summed to represent the new bus." diff --git a/doc/configtables/conventional.csv b/doc/configtables/conventional.csv index fb031197..12902f53 100644 --- a/doc/configtables/conventional.csv +++ b/doc/configtables/conventional.csv @@ -1,3 +1,5 @@ -,Unit,Values,Description -{name},--,"string","For any carrier/technology overwrite attributes as listed below." --- {attribute},--,"string or float","For any attribute, can specify a float or reference to a file path to a CSV file giving floats for each country (2-letter code)." +,Unit,Values,Description +unit_commitment ,bool,"{true, false}","Allow the overwrite of ramp_limit_up, ramp_limit_start_up, ramp_limit_shut_down, p_min_pu, min_up_time, min_down_time, and start_up_cost of conventional generators. Refer to the CSV file „unit_commitment.csv“." +dynamic_fuel_price ,bool,"{true, false}","Consider the monthly fluctuating fuel prices for each conventional generator. Refer to the CSV file ""data/validation/monthly_fuel_price.csv""." +{name},--,string,For any carrier/technology overwrite attributes as listed below. +-- {attribute},--,string or float,"For any attribute, can specify a float or reference to a file path to a CSV file giving floats for each country (2-letter code)." diff --git a/doc/configtables/hydro.csv b/doc/configtables/hydro.csv index fc53334e..4544d110 100644 --- a/doc/configtables/hydro.csv +++ b/doc/configtables/hydro.csv @@ -1,6 +1,8 @@ -,Unit,Values,Description -cutout,--,"Must be 'europe-2013-era5'","Specifies the directory where the relevant weather data ist stored." -carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydro power plants to build per-unit availability time series for. 'ror' stands for run-of-river plants, 'PHS' represents pumped-hydro storage, and 'hydro' stands for hydroelectric dams." -PHS_max_hours,h,float,"Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_." -hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}","Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation `_." -clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero." +,Unit,Values,Description +cutout,--,Must be 'europe-2013-era5',Specifies the directory where the relevant weather data ist stored. +carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydro power plants to build per-unit availability time series for. 'ror' stands for run-of-river plants, 'PHS' represents pumped-hydro storage, and 'hydro' stands for hydroelectric dams." +PHS_max_hours,h,float,Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. +hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}",Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation `_. +flatten_dispatch,bool,"{true, false}",Consider an upper limit for the hydro dispatch. The limit is given by the average capacity factor plus the buffer given in ``flatten_dispatch_buffer`` +flatten_dispatch_buffer,--,float,"If ``flatten_dispatch`` is true, specify the value added above the average capacity factor." +clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero." diff --git a/doc/configtables/lines.csv b/doc/configtables/lines.csv index ad5dd690..ec9ec007 100644 --- a/doc/configtables/lines.csv +++ b/doc/configtables/lines.csv @@ -5,3 +5,9 @@ s_nom_max,MW,"float","Global upper limit for the maximum capacity of each extend max_extension,MW,"float","Upper limit for the extended capacity of each extendable line." length_factor,--,float,"Correction factor to account for the fact that buses are *not* connected by lines through air-line distance." under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}","Specifies how to handle lines which are currently under construction." +dynamic_line_rating,,, +-- activate,bool,"true or false","Whether to take dynamic line rating into account" +-- cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored." +-- correction_factor,--,"float","Factor to compensate for overestimation of wind speeds in hourly averaged wind data" +-- max_voltage_difference,deg,"float","Maximum voltage angle difference in degrees or 'false' to disable" +-- max_line_rating,--,"float","Maximum line rating relative to nominal capacity without DLR, e.g. 1.3 or 'false' to disable" diff --git a/doc/configtables/opts.csv b/doc/configtables/opts.csv index b468be6e..8c8a706f 100644 --- a/doc/configtables/opts.csv +++ b/doc/configtables/opts.csv @@ -3,6 +3,7 @@ Trigger, Description, Definition, Status ``nSEG``; e.g. ``4380SEG``, "Apply time series segmentation with `tsam `_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use ``Co2L``, Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() `_ and its `caller `__, In active use ``Ep``, Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() `_ and its `caller `__, In active use +``Ept``, Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``, In active use ``CCL``, Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use ``EQ``, "Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.", ``solve_network``, In active use ``ATK``, "Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links.", ``prepare_network``, In active use diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv index c252ff32..45d50d84 100644 --- a/doc/configtables/solving.csv +++ b/doc/configtables/solving.csv @@ -1,17 +1,19 @@ -,Unit,Values,Description -options,,, --- load_shedding,bool/float,"{'true','false', float}","Add generators with very high marginal cost to simulate load shedding and avoid problem infeasibilities. If load shedding is a float, it denotes the marginal cost in EUR/kWh." --- transmission_losses,int,"[0-9]","Add piecewise linear approximation of transmission losses based on n tangents. Defaults to 0, which means losses are ignored." --- noisy_costs,bool,"{'true','false'}","Add random noise to marginal cost of generators by :math:`\mathcal{U}(0.009,0,011)` and capital cost of lines and links by :math:`\mathcal{U}(0.09,0,11)`." --- min_iterations,--,int,"Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run." --- max_iterations,--,int,"Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run." --- nhours,--,int,"Specifies the :math:`n` first snapshots to take into account. Must be less than the total number of snapshots. Rather recommended only for debugging." --- clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero." --- skip_iterations,bool,"{'true','false'}","Skip iterating, do not update impedances of branches. Defaults to true." --- track_iterations,bool,"{'true','false'}","Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration)" --- seed,--,int,"Random seed for increased deterministic behaviour." -solver,,, --- name,--,"One of {'gurobi', 'cplex', 'cbc', 'glpk', 'ipopt'}; potentially more possible","Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow." --- options,--,"Key listed under ``solver_options``.","Link to specific parameter settings." -solver_options,,"dict","Dictionaries with solver-specific parameter settings." -mem,MB,"int","Estimated maximum memory requirement for solving networks." +,Unit,Values,Description +options,,, +-- clip_p_max_pu,p.u.,float,To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero. +-- load_shedding,bool/float,"{'true','false', float}","Add generators with very high marginal cost to simulate load shedding and avoid problem infeasibilities. If load shedding is a float, it denotes the marginal cost in EUR/kWh." +-- noisy_costs,bool,"{'true','false'}","Add random noise to marginal cost of generators by :math:`\mathcal{U}(0.009,0,011)` and capital cost of lines and links by :math:`\mathcal{U}(0.09,0,11)`." +-- skip_iterations,bool,"{'true','false'}","Skip iterating, do not update impedances of branches. Defaults to true." +-- rolling_horizon,bool,"{'true','false'}","Whether to optimize the network in a rolling horizon manner, where the snapshot range is split into slices of size `horizon` which are solved consecutively." +-- seed,--,int,Random seed for increased deterministic behaviour. +-- track_iterations,bool,"{'true','false'}",Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration) +-- min_iterations,--,int,Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run. +-- max_iterations,--,int,Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run. +-- transmission_losses,int,[0-9],"Add piecewise linear approximation of transmission losses based on n tangents. Defaults to 0, which means losses are ignored." +-- linearized_unit_commitment,bool,"{'true','false'}",Whether to optimise using the linearized unit commitment formulation. +-- horizon,--,int,Number of snapshots to consider in each iteration. Defaults to 100. +solver,,, +-- name,--,"One of {'gurobi', 'cplex', 'cbc', 'glpk', 'ipopt'}; potentially more possible",Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow. +-- options,--,Key listed under ``solver_options``.,Link to specific parameter settings. +solver_options,,dict,Dictionaries with solver-specific parameter settings. +mem,MB,int,Estimated maximum memory requirement for solving networks. diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv index dcf0f29a..67954389 100644 --- a/doc/configtables/toplevel.csv +++ b/doc/configtables/toplevel.csv @@ -1,6 +1,12 @@ -,Unit,Values,Description -version,--,0.x.x,"Version of PyPSA-Eur. Descriptive only." -tutorial,bool,"{true, false}","Switch to retrieve the tutorial data set instead of the full data set." -logging,,, --- level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only" --- format,--,"","Custom format for log messages. See `LogRecord `_ attributes." +,Unit,Values,Description +version,--,0.x.x,Version of PyPSA-Eur. Descriptive only. +tutorial,bool,"{true, false}",Switch to retrieve the tutorial data set instead of the full data set. +logging,,, +-- level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only" +-- format,--,,Custom format for log messages. See `LogRecord `_ attributes. +private,,, +-- keys,,, +-- -- entsoe_api,--,,Optionally specify the ENTSO-E API key. See the guidelines to get `ENTSO-E API key `_ +remote,,, +-- ssh,--,,Optionally specify the SSH of a remote cluster to be synchronized. +-- path,--,,Optionally specify the file path within the remote cluster to be synchronized. diff --git a/doc/configuration.rst b/doc/configuration.rst index 8f15faa7..ceda1141 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -16,12 +16,13 @@ PyPSA-Eur has several configuration options which are documented in this section Top-level configuration ======================= +"Private" refers to local, machine-specific settings or data meant for personal use, not to be shared. "Remote" indicates the address of a server used for data exchange, often for clusters and data pushing/pulling. + .. literalinclude:: ../config/config.default.yaml :language: yaml :start-at: version: :end-before: # docs - .. csv-table:: :header-rows: 1 :widths: 22,7,22,33 diff --git a/doc/index.rst b/doc/index.rst index 314900f7..1552729c 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -78,10 +78,10 @@ them: .. note:: You can find showcases of the model's capabilities in the Supplementary Materials of the - preprint `Benefits of a Hydrogen Network in Europe - `_, the Supplementary Materials of the `paper in Joule with a + Joule paper `The potential role of a hydrogen network in Europe + `_, the Supplementary Materials of another `paper in Joule with a description of the industry sector - `_, or in `a 2021 presentation + `_, or in `a 2021 presentation at EMP-E `_. The sector-coupled extension of PyPSA-Eur was initially described in the paper `Synergies of sector coupling and transmission @@ -179,10 +179,13 @@ For sector-coupling studies: :: @misc{PyPSAEurSec, author = "Fabian Neumann and Elisabeth Zeyen and Marta Victoria and Tom Brown", - title = "The Potential Role of a Hydrogen Network in Europe", - year = "2022", + title = "The potential role of a hydrogen network in Europe", + journal "Joule", + volume = "7", + pages = "1--25" + year = "2023", eprint = "2207.05816", - url = "https://arxiv.org/abs/2207.05816", + doi = "10.1016/j.joule.2022.04.016", } For sector-coupling studies with pathway optimisation: :: @@ -224,7 +227,10 @@ The included ``.nc`` files are PyPSA network files which can be imported with Py n = pypsa.Network(filename) +Operating Systems +================= +The PyPSA-Eur workflow is continuously tested for Linux, macOS and Windows (WSL only). .. toctree:: @@ -274,6 +280,7 @@ The included ``.nc`` files are PyPSA network files which can be imported with Py release_notes licenses + validation limitations contributing support diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 4c7219e7..b6db1b90 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,39 +10,131 @@ Release Notes Upcoming Release ================ -* ``param:`` section in rule definition are added to track changed settings in ``config.yaml``. The goal is to automatically re-execute rules whose parameters have changed. See `Non-file parameters for rules `_ in the snakemake documentation. +* Updated Global Energy Monitor LNG terminal data to March 2023 version. -* **Important:** The configuration files are now located in the ``config`` directory. This counts for ``config.default.yaml``, ``config.yaml`` as well as the test configuration files which are now located in ``config/test``. Config files that are still in the root directory will be ignored. +* For industry distribution, use EPRTR as fallback if ETS data is not available. -* Bugfix: Correct typo in the CPLEX solver configuration in ``config.default.yaml``. +PyPSA-Eur 0.8.1 (27th July 2023) +================================ -* Bugfix: Error in ``add_electricity`` where carriers were added multiple times to the network, resulting in a non-unique carriers error. +**New Features** -* Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand`` and ``retrieve_load_data`` to ``retrieve_electricity_demand``. +* Add option to consider dynamic line rating based on wind speeds and + temperature according to `Glaum and Hofmann (2022) + `_. See configuration section ``lines: + dynamic_line_rating:`` for more details. (https://github.com/PyPSA/pypsa-eur/pull/675) -* Fix docs readthedocs built +* Add option to include a piecewise linear approximation of transmission losses, + e.g. by setting ``solving: options: transmission_losses: 2`` for an + approximation with two tangents. (https://github.com/PyPSA/pypsa-eur/pull/664) * Add plain hydrogen turbine as additional re-electrification option besides hydrogen fuel cell. Add switches for both re-electrification options under ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``. - -* A new function named ``sanitize_carrier`` ensures that all unique carrier names are present in the network's carriers attribute, and adds nice names and colors for each carrier according to the provided configuration dictionary. - -* Additional tech_color are added to include previously unlisted carriers. - -* Remove ``vresutils`` dependency. + (https://github.com/PyPSA/pypsa-eur/pull/647) * Added configuration option ``lines: max_extension:`` and ``links: max_extension:``` to control the maximum capacity addition per line or link in - MW. + MW. (https://github.com/PyPSA/pypsa-eur/pull/665) -* Add option to include a piecewise linear approximation of transmission losses, - e.g. by setting ``solving: options: transmission_losses: 2`` for an - approximation with two tangents. +* A ``param:`` section in the snakemake rule definitions was added to track + changed settings in ``config.yaml``. The goal is to automatically re-execute + rules where parameters have changed. See `Non-file parameters for rules + `_ + in the snakemake documentation. (https://github.com/PyPSA/pypsa-eur/pull/663) + +* A new function named ``sanitize_carrier`` ensures that all unique carrier + names are present in the network's carriers attribute, and adds nice names and + colors for each carrier according to the provided configuration dictionary. + (https://github.com/PyPSA/pypsa-eur/pull/653, + https://github.com/PyPSA/pypsa-eur/pull/690) + +* The configuration settings have been documented in more detail. + (https://github.com/PyPSA/pypsa-eur/pull/685) + +**Breaking Changes** + +* The configuration files are now located in the ``config`` directory. This + includes the ``config.default.yaml``, ``config.yaml`` as well as the test + configuration files which are now located in the ``config/test`` directory. + Config files that are still in the root directory will be ignored. + (https://github.com/PyPSA/pypsa-eur/pull/640) + +* Renamed script and rule name from ``build_load_data`` to + ``build_electricity_demand`` and ``retrieve_load_data`` to + ``retrieve_electricity_demand``. (https://github.com/PyPSA/pypsa-eur/pull/642, + https://github.com/PyPSA/pypsa-eur/pull/652) + +* Updated to new spatial clustering module introduced in PyPSA v0.25. + (https://github.com/PyPSA/pypsa-eur/pull/696) + +**Changes** + +* Handling networks with links with multiple inputs/outputs no longer requires + to override component attributes. + (https://github.com/PyPSA/pypsa-eur/pull/695) * Added configuration option ``enable: retrieve:`` to control whether data retrieval rules from snakemake are enabled or not. Th default setting ``auto`` - will automatically detect and enable/disable the rules based on internet connectivity. + will automatically detect and enable/disable the rules based on internet + connectivity. (https://github.com/PyPSA/pypsa-eur/pull/694) + +* Update to ``technology-data`` v0.6.0. + (https://github.com/PyPSA/pypsa-eur/pull/704) + +* Handle data bundle extraction paths via ``snakemake.output``. + +* Additional technologies are added to ``tech_color`` in the configuration files + to include previously unlisted carriers. + +* Doc: Added note that Windows is only tested in CI with WSL. + (https://github.com/PyPSA/pypsa-eur/issues/697) + +* Doc: Add support section. (https://github.com/PyPSA/pypsa-eur/pull/656) + +* Open ``rasterio`` files with ``rioxarray``. + (https://github.com/PyPSA/pypsa-eur/pull/474) + +* Migrate CI to ``micromamba``. (https://github.com/PyPSA/pypsa-eur/pull/700) + +**Bugs and Compatibility** + +* The new minimum PyPSA version is v0.25.1. + +* Removed ``vresutils`` dependency. + (https://github.com/PyPSA/pypsa-eur/pull/662) + +* Adapt to new ``powerplantmatching`` version. + (https://github.com/PyPSA/pypsa-eur/pull/687, + https://github.com/PyPSA/pypsa-eur/pull/701) + +* Bugfix: Correct typo in the CPLEX solver configuration in + ``config.default.yaml``. (https://github.com/PyPSA/pypsa-eur/pull/630) + +* Bugfix: Error in ``add_electricity`` where carriers were added multiple times + to the network, resulting in a non-unique carriers error. + +* Bugfix of optional reserve constraint. + (https://github.com/PyPSA/pypsa-eur/pull/645) + +* Fix broken equity constraints logic. + (https://github.com/PyPSA/pypsa-eur/pull/679) + +* Fix addition of load shedding generators. + (https://github.com/PyPSA/pypsa-eur/pull/649) + +* Fix automatic building of documentation on readthedocs.org. + (https://github.com/PyPSA/pypsa-eur/pull/658) + +* Bugfix: Update network clustering to avoid adding deleted links in clustered + network. (https://github.com/PyPSA/pypsa-eur/pull/678) + +* Address ``geopandas`` deprecations. + (https://github.com/PyPSA/pypsa-eur/pull/678) + +* Fix bug with underground hydrogen storage creation, where for some small model + regions no cavern storage is available. + (https://github.com/PyPSA/pypsa-eur/pull/672) PyPSA-Eur 0.8.0 (18th March 2023) diff --git a/doc/retrieve.rst b/doc/retrieve.rst index cc93c3d9..4786581e 100644 --- a/doc/retrieve.rst +++ b/doc/retrieve.rst @@ -83,7 +83,7 @@ This rule, as a substitute for :mod:`build_natura_raster`, downloads an already Rule ``retrieve_electricity_demand`` ==================================== -This rule downloads hourly electric load data for each country from the `OPSD platform `_. +This rule downloads hourly electric load data for each country from the `OPSD platform `_. **Relevant Settings** diff --git a/doc/validation.rst b/doc/validation.rst new file mode 100644 index 00000000..7049e3de --- /dev/null +++ b/doc/validation.rst @@ -0,0 +1,53 @@ +.. + SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors + + SPDX-License-Identifier: CC-BY-4.0 + +########################################## +Validation +########################################## + +The PyPSA-Eur model workflow provides a built-in mechanism for validation. This allows users to contrast the outcomes of network optimization against the historical behaviour of the European power system. The snakemake rule ``validate_elec_networks`` enables this by generating comparative figures that encapsulate key data points such as dispatch carrier, cross-border flows, and market prices per price zone. + +These comparisons utilize data from the 2019 ENTSO-E Transparency Platform. To enable this, an ENTSO-E API key must be inserted into the ``config.yaml`` file. Detailed steps for this process can be found in the user guide `here `_. + +Once the API key is set, the validation workflow can be triggered by running the following command: + + snakemake validate_elec_networks --configfile config/config.validation.yaml -c8 + + +The configuration file `config/config.validation.yaml` contains the following parameters: + +.. literalinclude:: ../config/config.validation.yaml + :language: yaml + +The setup uses monthly varying fuel prices for gas, lignite, coal and oil as well as CO2 prices, which are created by the script ``build_monthly_prices``. Upon completion of the validation process, the resulting network and generated figures will be stored in the ``results/validation`` directory for further analysis. + + +Results +======= + +By the time of writing the comparison with the historical data shows partially accurate, partially improvable results. The following figures show the comparison of the dispatch of the different carriers. + +.. image:: ../graphics/validation_seasonal_operation_area_elec_s_37_ec_lv1.0_Ept.png + :width: 100% + :align: center + +.. image:: ../graphics/validation_production_bar_elec_s_37_ec_lv1.0_Ept.png + :width: 100% + :align: center + + + +Issues and possible improvements +-------------------------------- + +**Overestimated dispatch of wind and solar:** Renewable potentials of wind and solar are slightly overestimated in the model. This leads to a higher dispatch of these carriers than in the historical data. In particular, the solar dispatch during winter is overestimated. + +**Coal - Lignite fuel switch:** The model has a fuel switch from coal to lignite. This might result from non-captured subsidies for lignite and coal in the model. In order to fix the fuel switch from coal to lignite, a manual cost correction was added to the script ``build_monthly_prices``. + +**Planned outages of nuclear power plants:** Planned outages of nuclear power plants are not captured in the model. This leads to a underestimated dispatch of nuclear power plants in winter and a overestimated dispatch in summer. This point is hard to fix, since the planned outages are not published in the ENTSO-E Transparency Platform. + +**False classification of run-of-river power plants:** Some run-of-river power plants are classified as hydro power plants in the model. This leads to a general overestimation of the hydro power dispatch. In particular, Swedish hydro power plants are overestimated. + +**Load shedding:** Due to constraint NTC's (crossborder capacities), the model has to shed load in some regions. This leads to a high market prices in the regions which drive the average market price up. Further fine-tuning of the NTC's is needed to avoid load shedding. diff --git a/envs/environment.fixed.yaml b/envs/environment.fixed.yaml index 1ff9313d..ca2ae848 100644 --- a/envs/environment.fixed.yaml +++ b/envs/environment.fixed.yaml @@ -12,74 +12,93 @@ dependencies: - _libgcc_mutex=0.1 - _openmp_mutex=4.5 - affine=2.4.0 -- alsa-lib=1.2.8 +- alsa-lib=1.2.9 - ampl-mp=3.1.0 -- amply=0.1.5 +- amply=0.1.6 +- anyio=3.7.1 - appdirs=1.4.4 +- argon2-cffi=21.3.0 +- argon2-cffi-bindings=21.2.0 - asttokens=2.2.1 -- atlite=0.2.10 +- async-lru=2.0.3 +- atk-1.0=2.38.0 +- atlite=0.2.11 - attr=2.5.1 -- attrs=22.2.0 +- attrs=23.1.0 +- aws-c-auth=0.7.0 +- aws-c-cal=0.6.0 +- aws-c-common=0.8.23 +- aws-c-compression=0.2.17 +- aws-c-event-stream=0.3.1 +- aws-c-http=0.7.11 +- aws-c-io=0.13.28 +- aws-c-mqtt=0.8.14 +- aws-c-s3=0.3.13 +- aws-c-sdkutils=0.1.11 +- aws-checksums=0.1.16 +- aws-crt-cpp=0.20.3 +- aws-sdk-cpp=1.10.57 +- babel=2.12.1 - backcall=0.2.0 - backports=1.0 -- backports.functools_lru_cache=1.6.4 -- beautifulsoup4=4.11.2 -- blosc=1.21.3 -- bokeh=2.4.3 +- backports.functools_lru_cache=1.6.5 +- beautifulsoup4=4.12.2 +- bleach=6.0.0 +- blosc=1.21.4 +- bokeh=3.2.1 - boost-cpp=1.78.0 -- bottleneck=1.3.6 +- bottleneck=1.3.7 - branca=0.6.0 - brotli=1.0.9 - brotli-bin=1.0.9 -- brotlipy=0.7.0 +- brotli-python=1.0.9 - bzip2=1.0.8 -- c-ares=1.18.1 -- ca-certificates=2022.12.7 +- c-ares=1.19.1 +- c-blosc2=2.10.0 +- ca-certificates=2023.7.22 - cairo=1.16.0 - cartopy=0.21.1 -- cdsapi=0.5.1 -- certifi=2022.12.7 +- cdsapi=0.6.1 +- certifi=2023.7.22 - cffi=1.15.1 - cfitsio=4.2.0 - cftime=1.6.2 -- charset-normalizer=2.1.1 -- click=8.1.3 +- charset-normalizer=3.2.0 +- click=8.1.6 - click-plugins=1.1.1 - cligj=0.7.2 - cloudpickle=2.2.1 -- coin-or-cbc=2.10.8 -- coin-or-cgl=0.60.6 -- coin-or-clp=1.17.7 -- coin-or-osi=0.108.7 -- coin-or-utils=2.11.6 -- coincbc=2.10.8 - colorama=0.4.6 -- configargparse=1.5.3 +- comm=0.1.3 +- configargparse=1.7 - connection_pool=0.0.3 -- country_converter=0.8.0 -- cryptography=39.0.1 -- curl=7.88.0 +- contourpy=1.1.0 +- country_converter=1.0.0 +- curl=8.2.0 - cycler=0.11.0 -- cytoolz=0.12.0 -- dask=2023.2.0 -- dask-core=2023.2.0 +- cytoolz=0.12.2 +- dask=2023.7.1 +- dask-core=2023.7.1 - datrie=0.8.2 - dbus=1.13.6 +- debugpy=1.6.7 - decorator=5.1.1 +- defusedxml=0.7.1 - deprecation=2.1.0 - descartes=1.1.0 -- distributed=2023.2.0 +- distributed=2023.7.1 - distro=1.8.0 -- docutils=0.19 -- dpath=2.1.4 -- entsoe-py=0.5.8 +- docutils=0.20.1 +- dpath=2.1.6 +- entrypoints=0.4 +- entsoe-py=0.5.10 - et_xmlfile=1.1.0 -- exceptiongroup=1.1.0 +- exceptiongroup=1.1.2 - executing=1.2.0 - expat=2.5.0 -- fftw=3.3.10 -- filelock=3.9.0 -- fiona=1.9.1 +- filelock=3.12.2 +- fiona=1.9.4 +- flit-core=3.9.0 - folium=0.14.0 - font-ttf-dejavu-sans-mono=2.37 - font-ttf-inconsolata=3.000 @@ -88,293 +107,366 @@ dependencies: - fontconfig=2.14.2 - fonts-conda-ecosystem=1 - fonts-conda-forge=1 -- fonttools=4.38.0 +- fonttools=4.41.1 - freetype=2.12.1 - freexl=1.0.6 -- fsspec=2023.1.0 -- gdal=3.6.2 +- fribidi=1.0.10 +- fsspec=2023.6.0 +- gdal=3.7.0 +- gdk-pixbuf=2.42.10 - geographiclib=1.52 - geojson-rewind=1.0.2 -- geopandas=0.12.2 -- geopandas-base=0.12.2 +- geopandas=0.13.2 +- geopandas-base=0.13.2 - geopy=2.3.0 -- geos=3.11.1 +- geos=3.11.2 - geotiff=1.7.1 - gettext=0.21.1 +- gflags=2.2.2 - giflib=5.2.1 - gitdb=4.0.10 -- gitpython=3.1.30 -- glib=2.74.1 -- glib-tools=2.74.1 +- gitpython=3.1.32 +- glib=2.76.4 +- glib-tools=2.76.4 +- glog=0.6.0 +- gmp=6.2.1 - graphite2=1.3.13 -- gst-plugins-base=1.22.0 -- gstreamer=1.22.0 -- gstreamer-orc=0.4.33 -- harfbuzz=6.0.0 +- graphviz=8.1.0 +- gst-plugins-base=1.22.5 +- gstreamer=1.22.5 +- gtk2=2.24.33 +- gts=0.7.6 +- harfbuzz=7.3.0 - hdf4=4.2.15 -- hdf5=1.12.2 -- heapdict=1.0.1 +- hdf5=1.14.1 - humanfriendly=10.0 -- icu=70.1 +- icu=72.1 - idna=3.4 -- importlib-metadata=6.0.0 -- importlib_resources=5.10.2 +- importlib-metadata=6.8.0 +- importlib_metadata=6.8.0 +- importlib_resources=6.0.0 - iniconfig=2.0.0 -- ipopt=3.14.11 -- ipython=8.10.0 -- jack=1.9.22 +- ipopt=3.14.12 +- ipykernel=6.24.0 +- ipython=8.14.0 +- ipython_genutils=0.2.0 +- ipywidgets=8.0.7 - jedi=0.18.2 - jinja2=3.1.2 -- joblib=1.2.0 -- jpeg=9e +- joblib=1.3.0 - json-c=0.16 -- jsonschema=4.17.3 -- jupyter_core=5.2.0 -- kealib=1.5.0 +- json5=0.9.14 +- jsonschema=4.18.4 +- jsonschema-specifications=2023.7.1 +- jupyter=1.0.0 +- jupyter-lsp=2.2.0 +- jupyter_client=8.3.0 +- jupyter_console=6.6.3 +- jupyter_core=5.3.1 +- jupyter_events=0.6.3 +- jupyter_server=2.7.0 +- jupyter_server_terminals=0.4.4 +- jupyterlab=4.0.3 +- jupyterlab_pygments=0.2.2 +- jupyterlab_server=2.24.0 +- jupyterlab_widgets=3.0.8 +- kealib=1.5.1 - keyutils=1.6.1 - kiwisolver=1.4.4 -- krb5=1.20.1 +- krb5=1.21.1 - lame=3.100 -- lcms2=2.14 +- lcms2=2.15 - ld_impl_linux-64=2.40 - lerc=4.0.0 +- libabseil=20230125.3 - libaec=1.0.6 +- libarchive=3.6.2 +- libarrow=12.0.1 - libblas=3.9.0 - libbrotlicommon=1.0.9 - libbrotlidec=1.0.9 - libbrotlienc=1.0.9 -- libcap=2.66 +- libcap=2.67 - libcblas=3.9.0 - libclang=15.0.7 - libclang13=15.0.7 +- libcrc32c=1.1.2 - libcups=2.3.3 -- libcurl=7.88.0 -- libdb=6.2.32 -- libdeflate=1.17 +- libcurl=8.2.0 +- libdeflate=1.18 - libedit=3.1.20191231 - libev=4.33 -- libevent=2.1.10 +- libevent=2.1.12 +- libexpat=2.5.0 - libffi=3.4.2 -- libflac=1.4.2 -- libgcc-ng=12.2.0 +- libflac=1.4.3 +- libgcc-ng=13.1.0 - libgcrypt=1.10.1 -- libgdal=3.6.2 -- libgfortran-ng=12.2.0 -- libgfortran5=12.2.0 -- libglib=2.74.1 -- libgomp=12.2.0 -- libgpg-error=1.46 +- libgd=2.3.3 +- libgdal=3.7.0 +- libgfortran-ng=13.1.0 +- libgfortran5=13.1.0 +- libglib=2.76.4 +- libgomp=13.1.0 +- libgoogle-cloud=2.12.0 +- libgpg-error=1.47 +- libgrpc=1.56.2 - libiconv=1.17 +- libjpeg-turbo=2.1.5.1 - libkml=1.3.0 - liblapack=3.9.0 - liblapacke=3.9.0 - libllvm15=15.0.7 -- libnetcdf=4.8.1 -- libnghttp2=1.51.0 +- libnetcdf=4.9.2 +- libnghttp2=1.52.0 - libnsl=2.0.0 +- libnuma=2.0.16 - libogg=1.3.4 -- libopenblas=0.3.21 +- libopenblas=0.3.23 - libopus=1.3.1 - libpng=1.6.39 -- libpq=15.2 +- libpq=15.3 +- libprotobuf=4.23.3 +- librsvg=2.56.1 - librttopo=1.1.0 - libsndfile=1.2.0 +- libsodium=1.0.18 - libspatialindex=1.9.3 - libspatialite=5.0.1 -- libsqlite=3.40.0 -- libssh2=1.10.0 -- libstdcxx-ng=12.2.0 -- libsystemd0=252 -- libtiff=4.5.0 +- libsqlite=3.42.0 +- libssh2=1.11.0 +- libstdcxx-ng=13.1.0 +- libsystemd0=253 +- libthrift=0.18.1 +- libtiff=4.5.1 - libtool=2.4.7 -- libudev1=252 -- libuuid=2.32.1 +- libutf8proc=2.8.0 +- libuuid=2.38.1 - libvorbis=1.3.7 -- libwebp-base=1.2.4 -- libxcb=1.13 +- libwebp=1.3.1 +- libwebp-base=1.3.1 +- libxcb=1.15 - libxkbcommon=1.5.0 -- libxml2=2.10.3 +- libxml2=2.11.4 - libxslt=1.1.37 - libzip=1.9.2 - libzlib=1.2.13 -- linopy=0.1.3 - locket=1.0.0 -- lxml=4.9.2 +- lxml=4.9.3 - lz4=4.3.2 - lz4-c=1.9.4 - lzo=2.10 - mapclassify=2.5.0 -- markupsafe=2.1.2 +- markupsafe=2.1.3 - matplotlib=3.5.3 - matplotlib-base=3.5.3 - matplotlib-inline=0.1.6 - memory_profiler=0.61.0 -- metis=5.1.0 -- mpg123=1.31.2 -- msgpack-python=1.0.4 +- metis=5.1.1 +- mistune=3.0.0 +- mpg123=1.31.3 +- msgpack-python=1.0.5 - mumps-include=5.2.1 - mumps-seq=5.2.1 -- munch=2.5.0 +- munch=4.0.0 - munkres=1.1.4 -- mysql-common=8.0.32 -- mysql-libs=8.0.32 -- nbformat=5.7.3 -- ncurses=6.3 -- netcdf4=1.6.2 -- networkx=3.0 +- mysql-common=8.0.33 +- mysql-libs=8.0.33 +- nbclient=0.8.0 +- nbconvert=7.7.2 +- nbconvert-core=7.7.2 +- nbconvert-pandoc=7.7.2 +- nbformat=5.9.1 +- ncurses=6.4 +- nest-asyncio=1.5.6 +- netcdf4=1.6.4 +- networkx=3.1 - nomkl=1.0 +- notebook=7.0.0 +- notebook-shim=0.2.3 - nspr=4.35 -- nss=3.88 -- numexpr=2.8.3 -- numpy=1.24 +- nss=3.89 +- numexpr=2.8.4 +- numpy=1.25.1 - openjdk=17.0.3 - openjpeg=2.5.0 -- openpyxl=3.1.0 -- openssl=3.0.8 -- packaging=23.0 -- pandas=1.5.3 +- openpyxl=3.1.2 +- openssl=3.1.1 +- orc=1.9.0 +- overrides=7.3.1 +- packaging=23.1 +- pandas=2.0.3 +- pandoc=3.1.3 +- pandocfilters=1.5.0 +- pango=1.50.14 - parso=0.8.3 -- partd=1.3.0 +- partd=1.4.0 - patsy=0.5.3 - pcre2=10.40 - pexpect=4.8.0 - pickleshare=0.7.5 -- pillow=9.4.0 -- pip=23.0 +- pillow=10.0.0 +- pip=23.2.1 - pixman=0.40.0 - pkgutil-resolve-name=1.3.10 - plac=1.3.5 -- platformdirs=3.0.0 -- pluggy=1.0.0 +- platformdirs=3.9.1 +- pluggy=1.2.0 - ply=3.11 -- pooch=1.6.0 -- poppler=22.12.0 +- pooch=1.7.0 +- poppler=23.05.0 - poppler-data=0.4.12 -- postgresql=15.2 -- powerplantmatching=0.5.6 +- postgresql=15.3 +- powerplantmatching=0.5.7 - progressbar2=4.2.0 -- proj=9.1.0 -- prompt-toolkit=3.0.36 -- psutil=5.9.4 +- proj=9.2.1 +- prometheus_client=0.17.1 +- prompt-toolkit=3.0.39 +- prompt_toolkit=3.0.39 +- psutil=5.9.5 - pthread-stubs=0.4 - ptyprocess=0.7.0 - pulp=2.7.0 -- pulseaudio=16.1 +- pulseaudio-client=16.1 - pure_eval=0.2.2 +- py-cpuinfo=9.0.0 +- pyarrow=12.0.1 - pycountry=22.3.5 - pycparser=2.21 -- pygments=2.14.0 -- pyomo=6.4.4 -- pyopenssl=23.0.0 -- pyparsing=3.0.9 -- pyproj=3.4.1 -- pypsa=0.22.1 +- pygments=2.15.1 +- pyomo=6.6.1 +- pyparsing=3.1.0 +- pyproj=3.6.0 - pyqt=5.15.7 - pyqt5-sip=12.11.0 -- pyrsistent=0.19.3 - pyshp=2.3.1 - pysocks=1.7.1 -- pytables=3.7.0 -- pytest=7.2.1 -- python=3.10.9 +- pytables=3.8.0 +- pytest=7.4.0 +- python=3.10.12 - python-dateutil=2.8.2 -- python-fastjsonschema=2.16.2 -- python-utils=3.5.2 +- python-fastjsonschema=2.18.0 +- python-json-logger=2.0.7 +- python-tzdata=2023.3 +- python-utils=3.7.0 - python_abi=3.10 -- pytz=2022.7.1 +- pytz=2023.3 - pyxlsb=1.0.10 - pyyaml=6.0 +- pyzmq=25.1.0 - qt-main=5.15.8 -- rasterio=1.3.4 -- readline=8.1.2 -- requests=2.28.1 -- retry=0.9.2 -- rich=12.5.1 -- rioxarray=0.13.3 -- rtree=1.0.0 -- s2n=1.0.10 -- scikit-learn=1.1.1 -- scipy=1.8.1 +- qtconsole=5.4.3 +- qtconsole-base=5.4.3 +- qtpy=2.3.1 +- rasterio=1.3.8 +- rdma-core=28.9 +- re2=2023.03.02 +- readline=8.2 +- referencing=0.30.0 +- requests=2.31.0 +- reretry=0.11.8 +- rfc3339-validator=0.1.4 +- rfc3986-validator=0.1.1 +- rioxarray=0.14.1 +- rpds-py=0.9.2 +- rtree=1.0.1 +- s2n=1.3.46 +- scikit-learn=1.3.0 +- scipy=1.11.1 - scotch=6.0.9 - seaborn=0.12.2 - seaborn-base=0.12.2 -- setuptools=67.3.2 +- send2trash=1.8.2 +- setuptools=68.0.0 - setuptools-scm=7.1.0 - setuptools_scm=7.1.0 - shapely=2.0.1 -- sip=6.7.7 +- sip=6.7.10 - six=1.16.0 - smart_open=6.3.0 - smmap=3.0.5 -- snakemake-minimal=7.22.0 -- snappy=1.1.9 +- snakemake-minimal=7.30.2 +- snappy=1.1.10 +- sniffio=1.3.0 - snuggs=1.4.7 - sortedcontainers=2.4.0 - soupsieve=2.3.2.post1 -- sqlite=3.40.0 +- sqlite=3.42.0 - stack_data=0.6.2 -- statsmodels=0.13.5 +- statsmodels=0.14.0 - stopit=1.1.2 - tabula-py=2.6.0 - tabulate=0.9.0 - tblib=1.7.0 -- threadpoolctl=3.1.0 +- terminado=0.17.1 +- threadpoolctl=3.2.0 - throttler=1.2.1 - tiledb=2.13.2 +- tinycss2=1.2.1 - tk=8.6.12 - toml=0.10.2 - tomli=2.0.1 - toolz=0.12.0 -- toposort=1.9 -- tornado=6.2 -- tqdm=4.64.1 +- toposort=1.10 +- tornado=6.3.2 +- tqdm=4.65.0 - traitlets=5.9.0 -- typing-extensions=4.4.0 -- typing_extensions=4.4.0 -- tzcode=2022g -- tzdata=2022g +- typing-extensions=4.7.1 +- typing_extensions=4.7.1 +- typing_utils=0.1.0 +- tzcode=2023c +- tzdata=2023c +- ucx=1.14.1 - unicodedata2=15.0.0 - unidecode=1.3.6 - unixodbc=2.3.10 -- urllib3=1.26.14 +- urllib3=2.0.4 - wcwidth=0.2.6 -- wheel=0.38.4 -- wrapt=1.14.1 -- xarray=2023.2.0 +- webencodings=0.5.1 +- websocket-client=1.6.1 +- wheel=0.41.0 +- widgetsnbextension=4.0.8 +- wrapt=1.15.0 +- xarray=2023.7.0 - xcb-util=0.4.0 - xcb-util-image=0.4.0 - xcb-util-keysyms=0.4.0 - xcb-util-renderutil=0.3.9 - xcb-util-wm=0.4.1 - xerces-c=3.2.4 +- xkeyboard-config=2.39 - xlrd=2.0.1 - xorg-fixesproto=5.0 - xorg-inputproto=2.3.2 - xorg-kbproto=1.0.7 -- xorg-libice=1.0.10 -- xorg-libsm=1.2.3 -- xorg-libx11=1.7.2 -- xorg-libxau=1.0.9 +- xorg-libice=1.1.1 +- xorg-libsm=1.2.4 +- xorg-libx11=1.8.6 +- xorg-libxau=1.0.11 - xorg-libxdmcp=1.1.3 - xorg-libxext=1.3.4 - xorg-libxfixes=5.0.3 - xorg-libxi=1.7.10 -- xorg-libxrender=0.9.10 +- xorg-libxrender=0.9.11 - xorg-libxtst=1.2.3 - xorg-recordproto=1.14.2 - xorg-renderproto=0.11.1 - xorg-xextproto=7.3.0 +- xorg-xf86vidmodeproto=2.3.1 - xorg-xproto=7.0.31 -- xyzservices=2022.9.0 +- xyzservices=2023.7.0 - xz=5.2.6 - yaml=0.2.5 - yte=1.5.1 -- zict=2.2.0 -- zipp=3.13.0 +- zeromq=4.3.4 +- zict=3.0.0 +- zipp=3.16.2 - zlib=1.2.13 +- zlib-ng=2.0.7 - zstd=1.5.2 - pip: - - countrycode==0.2 - - highspy==1.5.0.dev0 - - pybind11==2.10.3 - - tsam==2.2.2 + - gurobipy==10.0.2 + - linopy==0.2.2 + - pypsa==0.25.1 + - tsam==2.3.0 + - validators==0.20.0 diff --git a/envs/environment.yaml b/envs/environment.yaml index c8e469ac..c3af36bb 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -10,7 +10,6 @@ dependencies: - python>=3.8 - pip -# - pypsa>=0.23 - atlite>=0.2.9 - dask @@ -54,6 +53,7 @@ dependencies: - descartes - rasterio!=1.2.10 + - pip: - tsam>=1.1.0 - - git+https://github.com/PyPSA/PyPSA.git@master + - pypsa>=0.25.1 diff --git a/graphics/validation_production_bar_elec_s_37_ec_lv1.0_Ept.png b/graphics/validation_production_bar_elec_s_37_ec_lv1.0_Ept.png new file mode 100644 index 00000000..1954b715 Binary files /dev/null and b/graphics/validation_production_bar_elec_s_37_ec_lv1.0_Ept.png differ diff --git a/graphics/validation_seasonal_operation_area_elec_s_37_ec_lv1.0_Ept.png b/graphics/validation_seasonal_operation_area_elec_s_37_ec_lv1.0_Ept.png new file mode 100644 index 00000000..19c3ed98 Binary files /dev/null and b/graphics/validation_seasonal_operation_area_elec_s_37_ec_lv1.0_Ept.png differ diff --git a/matplotlibrc b/matplotlibrc index 2cc4c229..f00ed5cd 100644 --- a/matplotlibrc +++ b/matplotlibrc @@ -4,3 +4,4 @@ font.family: sans-serif font.sans-serif: Ubuntu, DejaVu Sans image.cmap: viridis +figure.autolayout : True diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index d01ee0e2..383951bd 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -62,6 +62,9 @@ rule base_network: params: countries=config["countries"], snapshots=config["snapshots"], + lines=config["lines"], + links=config["links"], + transformers=config["transformers"], input: eg_buses="data/entsoegridkit/buses.csv", eg_lines="data/entsoegridkit/lines.csv", @@ -203,75 +206,117 @@ rule build_ship_raster: "../scripts/build_ship_raster.py" -# rule build_renewable_profiles: -# params: -# renewable=config["renewable"], -# input: -# base_network=RESOURCES + "networks/base.nc", -# corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"), -# natura=lambda w: ( -# RESOURCES + "natura.tiff" -# if config["renewable"][w.technology]["natura"] -# else [] -# ), -# gebco=ancient( -# lambda w: ( -# "data/bundle/GEBCO_2014_2D.nc" -# if config["renewable"][w.technology].get("max_depth") -# else [] -# ) -# ), -# ship_density=lambda w: ( -# RESOURCES + "shipdensity_raster.tif" -# if "ship_threshold" in config["renewable"][w.technology].keys() -# else [] -# ), -# country_shapes=RESOURCES + "country_shapes.geojson", -# offshore_shapes=RESOURCES + "offshore_shapes.geojson", -# regions=lambda w: ( -# RESOURCES + "regions_onshore.geojson" -# if w.technology in ("onwind", "solar") -# else RESOURCES + "regions_offshore.geojson" -# ), -# cutout=lambda w: "cutouts/" -# + CDIR -# + config["renewable"][w.technology]["cutout"] -# + ".nc", -# output: -# profile=RESOURCES + "profile_{technology}.nc", -# log: -# LOGS + "build_renewable_profile_{technology}.log", -# benchmark: -# BENCHMARKS + "build_renewable_profiles_{technology}" -# threads: ATLITE_NPROCESSES -# resources: -# mem_mb=ATLITE_NPROCESSES * 5000, -# wildcard_constraints: -# technology="(?!hydro).*", # Any technology other than hydro -# conda: -# "../envs/environment.yaml" -# script: -# "../scripts/build_renewable_profiles.py" +rule build_renewable_profiles: + params: + renewable=config["renewable"], + input: + base_network=RESOURCES + "networks/base.nc", + corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"), + natura=lambda w: ( + RESOURCES + "natura.tiff" + if config["renewable"][w.technology]["natura"] + else [] + ), + gebco=ancient( + lambda w: ( + "data/bundle/GEBCO_2014_2D.nc" + if config["renewable"][w.technology].get("max_depth") + else [] + ) + ), + ship_density=lambda w: ( + RESOURCES + "shipdensity_raster.tif" + if "ship_threshold" in config["renewable"][w.technology].keys() + else [] + ), + country_shapes=RESOURCES + "country_shapes.geojson", + offshore_shapes=RESOURCES + "offshore_shapes.geojson", + regions=lambda w: ( + RESOURCES + "regions_onshore.geojson" + if w.technology in ("onwind", "solar") + else RESOURCES + "regions_offshore.geojson" + ), + cutout=lambda w: "cutouts/" + + CDIR + + config["renewable"][w.technology]["cutout"] + + ".nc", + output: + profile=RESOURCES + "profile_{technology}.nc", + log: + LOGS + "build_renewable_profile_{technology}.log", + benchmark: + BENCHMARKS + "build_renewable_profiles_{technology}" + threads: ATLITE_NPROCESSES + resources: + mem_mb=ATLITE_NPROCESSES * 5000, + wildcard_constraints: + technology="(?!hydro).*", # Any technology other than hydro + conda: + "../envs/environment.yaml" + script: + "../scripts/build_renewable_profiles.py" -# rule build_hydro_profile: -# params: -# hydro=config["renewable"]["hydro"], -# countries=config["countries"], -# input: -# country_shapes=RESOURCES + "country_shapes.geojson", -# eia_hydro_generation="data/eia_hydro_annual_generation.csv", -# cutout=f"cutouts/" + CDIR + config["renewable"]["hydro"]["cutout"] + ".nc", -# output: -# RESOURCES + "profile_hydro.nc", -# log: -# LOGS + "build_hydro_profile.log", -# resources: -# mem_mb=5000, -# conda: -# "../envs/environment.yaml" -# script: -# "../scripts/build_hydro_profile.py" +rule build_monthly_prices: + input: + co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls", + fuel_price_raw="data/validation/energy-price-trends-xlsx-5619002.xlsx", + output: + co2_price=RESOURCES + "co2_price.csv", + fuel_price=RESOURCES + "monthly_fuel_price.csv", + log: + LOGS + "build_monthly_prices.log", + threads: 1 + resources: + mem_mb=5000, + conda: + "../envs/environment.yaml" + script: + "../scripts/build_monthly_prices.py" + + +rule build_hydro_profile: + params: + hydro=config["renewable"]["hydro"], + countries=config["countries"], + input: + country_shapes=RESOURCES + "country_shapes.geojson", + eia_hydro_generation="data/eia_hydro_annual_generation.csv", + cutout=f"cutouts/" + CDIR + config["renewable"]["hydro"]["cutout"] + ".nc", + output: + RESOURCES + "profile_hydro.nc", + log: + LOGS + "build_hydro_profile.log", + resources: + mem_mb=5000, + conda: + "../envs/environment.yaml" + script: + "../scripts/build_hydro_profile.py" + + +if config["lines"]["dynamic_line_rating"]["activate"]: + + rule build_line_rating: + input: + base_network=RESOURCES + "networks/base.nc", + cutout="cutouts/" + + CDIR + + config["lines"]["dynamic_line_rating"]["cutout"] + + ".nc", + output: + output=RESOURCES + "networks/line_rating.nc", + log: + LOGS + "build_line_rating.log", + benchmark: + BENCHMARKS + "build_line_rating" + threads: ATLITE_NPROCESSES + resources: + mem_mb=ATLITE_NPROCESSES * 1000, + conda: + "../envs/environment.yaml" + script: + "../scripts/build_line_rating.py" rule add_electricity: @@ -281,7 +326,7 @@ rule add_electricity: countries=config["countries"], renewable=config["renewable"], electricity=config["electricity"], - conventional=config.get("conventional", {}), + conventional=config["conventional"], costs=config["costs"], input: **{ @@ -291,15 +336,21 @@ rule add_electricity: **{ f"conventional_{carrier}_{attr}": fn for carrier, d in config.get("conventional", {None: {}}).items() + if carrier in config["electricity"]["conventional_carriers"] for attr, fn in d.items() if str(fn).startswith("data/") }, base_network=RESOURCES + "networks/base.nc", + line_rating=RESOURCES + "networks/line_rating.nc" + if config["lines"]["dynamic_line_rating"]["activate"] + else RESOURCES + "networks/base.nc", tech_costs=COSTS, regions=RESOURCES + "regions_onshore.geojson", powerplants=RESOURCES + "powerplants.csv", hydro_capacities=ancient("data/bundle/hydro_capacities.csv"), geth_hydro_capacities="data/geth2015_hydro_capacities.csv", + unit_commitment="data/unit_commitment.csv", + fuel_price=RESOURCES + "monthly_fuel_price.csv", load=RESOURCES + "load.csv", nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", output: @@ -310,7 +361,7 @@ rule add_electricity: BENCHMARKS + "add_electricity" threads: 1 resources: - mem_mb=5000, + mem_mb=10000, conda: "../envs/environment.yaml" script: @@ -344,7 +395,7 @@ rule simplify_network: BENCHMARKS + "simplify_network/elec_s{simpl}" threads: 1 resources: - mem_mb=4000, + mem_mb=12000, conda: "../envs/environment.yaml" script: @@ -385,7 +436,7 @@ rule cluster_network: BENCHMARKS + "cluster_network/elec_s{simpl}_{clusters}" threads: 1 resources: - mem_mb=6000, + mem_mb=10000, conda: "../envs/environment.yaml" script: @@ -408,7 +459,7 @@ rule add_extra_components: BENCHMARKS + "add_extra_components/elec_s{simpl}_{clusters}_ec" threads: 1 resources: - mem_mb=3000, + mem_mb=4000, conda: "../envs/environment.yaml" script: @@ -427,6 +478,7 @@ rule prepare_network: input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", tech_costs=COSTS, + co2_price=RESOURCES + "co2_price.csv", output: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", log: diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 1c3507fd..356abdc5 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -86,7 +86,7 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule build_gas_input_locations: input: lng=HTTP.remote( - "https://globalenergymonitor.org/wp-content/uploads/2022/09/Europe-Gas-Tracker-August-2022.xlsx", + "https://globalenergymonitor.org/wp-content/uploads/2023/07/Europe-Gas-Tracker-2023-03-v3.xlsx", keep_local=True, ), entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson", @@ -295,7 +295,7 @@ rule build_biomass_potentials: "../scripts/build_biomass_potentials.py" -if config["sector"]["biomass_transport"]: +if config["sector"]["biomass_transport"] or config["sector"]["biomass_spatial"]: rule build_biomass_transport_costs: input: @@ -320,9 +320,8 @@ if config["sector"]["biomass_transport"]: build_biomass_transport_costs_output = rules.build_biomass_transport_costs.output -if not config["sector"]["biomass_transport"]: +if not (config["sector"]["biomass_transport"] or config["sector"]["biomass_spatial"]): # this is effecively an `else` statement which is however not liked by snakefmt - build_biomass_transport_costs_output = {} @@ -724,7 +723,6 @@ rule prepare_sector_network: **build_biomass_transport_costs_output, **gas_infrastructure, **build_sequestration_potentials_output, - overrides="data/override_component_attrs", network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", energy_totals_name=RESOURCES + "energy_totals.csv", eurostat=input_eurostat, diff --git a/rules/collect.smk b/rules/collect.smk index 1d7e21e5..14a04480 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -82,3 +82,18 @@ rule plot_networks: + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", **config["scenario"] ), + + +rule validate_elec_networks: + input: + expand( + RESULTS + + "figures/.statistics_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", + **config["scenario"] + ), + expand( + RESULTS + + "figures/.validation_{kind}_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", + **config["scenario"], + kind=["production", "prices", "cross_border"] + ), diff --git a/rules/common.smk b/rules/common.smk index 3fc2c721..ec5be355 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -15,8 +15,8 @@ def memory(w): if m is not None: factor *= int(m.group(1)) / 8760 break - if w.clusters.endswith("m"): - return int(factor * (18000 + 180 * int(w.clusters[:-1]))) + if w.clusters.endswith("m") or w.clusters.endswith("c"): + return int(factor * (55000 + 600 * int(w.clusters[:-1]))) elif w.clusters == "all": return int(factor * (18000 + 180 * 4000)) else: diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 3f2a3fcc..ae95b201 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -13,7 +13,6 @@ rule plot_network: foresight=config["foresight"], plotting=config["plotting"], input: - overrides="data/override_component_attrs", network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -35,12 +34,38 @@ rule plot_network: script: "../scripts/plot_network.py" +rule plot_network_perfect: + params: + foresight=config["foresight"], + plotting=config["plotting"], + input: + network=RESULTS + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", + regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + output: + map=RESULTS + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{year}.pdf", + today=RESULTS + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{year}-today.pdf", + threads: 2 + resources: + mem_mb=10000, + benchmark: + BENCHMARKS + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{year}_brownfield_all_years_benchmark.txt", + conda: + "../envs/environment.yaml" + script: + "../scripts/plot_network.py" + + + rule copy_config: params: RDIR=RDIR, output: - RESULTS + "config/config.yaml", + RESULTS + "config.yaml", threads: 1 resources: mem_mb=1000, @@ -52,22 +77,6 @@ rule copy_config: "../scripts/copy_config.py" -rule copy_conda_env: - output: - RESULTS + "config/environment.yaml", - threads: 1 - resources: - mem_mb=500, - log: - LOGS + "copy_conda_env.log", - benchmark: - BENCHMARKS + "copy_conda_env" - conda: - "../envs/environment.yaml" - shell: - "conda env export -f {output} --no-builds" - - rule make_summary: params: foresight=config["foresight"], @@ -76,7 +85,6 @@ rule make_summary: scenario=config["scenario"], RDIR=RDIR, input: - overrides="data/override_component_attrs", networks=expand( RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -147,3 +155,34 @@ rule plot_summary: "../envs/environment.yaml" script: "../scripts/plot_summary.py" + + +STATISTICS_BARPLOTS = [ + "capacity_factor", + "installed_capacity", + "optimal_capacity", + "capital_expenditure", + "operational_expenditure", + "curtailment", + "supply", + "withdrawal", + "market_value", +] + + +rule plot_elec_statistics: + params: + plotting=config["plotting"], + barplots=STATISTICS_BARPLOTS, + input: + network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + output: + **{ + f"{plot}_bar": RESULTS + + f"figures/statistics_{plot}_bar_elec_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" + for plot in STATISTICS_BARPLOTS + }, + barplots_touch=RESULTS + + "figures/.statistics_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", + script: + "../scripts/plot_statistics.py" diff --git a/rules/retrieve.smk b/rules/retrieve.smk index a253ec0c..0b60ee2e 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -158,7 +158,11 @@ if config["enable"]["retrieve"]: rule retrieve_electricity_demand: input: HTTP.remote( - "data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv", + "data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format( + version="2019-06-05" + if config["snapshots"]["end"] < "2019" + else "2020-10-06" + ), keep_local=True, static=True, ), @@ -191,3 +195,39 @@ if config["enable"]["retrieve"]: retries: 2 run: move(input[0], output[0]) + + +if config["enable"]["retrieve"]: + + rule retrieve_monthly_co2_prices: + input: + HTTP.remote( + "https://www.eex.com/fileadmin/EEX/Downloads/EUA_Emission_Spot_Primary_Market_Auction_Report/Archive_Reports/emission-spot-primary-market-auction-report-2019-data.xls", + keep_local=True, + static=True, + ), + output: + "data/validation/emission-spot-primary-market-auction-report-2019-data.xls", + log: + LOGS + "retrieve_monthly_co2_prices.log", + resources: + mem_mb=5000, + retries: 2 + run: + move(input[0], output[0]) + + +if config["enable"]["retrieve"]: + + rule retrieve_monthly_fuel_prices: + output: + "data/validation/energy-price-trends-xlsx-5619002.xlsx", + log: + LOGS + "retrieve_monthly_fuel_prices.log", + resources: + mem_mb=5000, + retries: 2 + conda: + "../envs/environment.yaml" + script: + "../scripts/retrieve_monthly_fuel_prices.py" diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index f73a5b0c..c396ebd5 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -13,6 +13,7 @@ rule solve_network: ), input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + config=RESULTS + "config.yaml", output: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", log: @@ -26,6 +27,7 @@ rule solve_network: threads: 4 resources: mem_mb=memory, + walltime=config["solving"].get("walltime", "12:00:00"), shadow: "minimal" conda: @@ -55,7 +57,8 @@ rule solve_operations_network: ) threads: 4 resources: - mem_mb=(lambda w: 5000 + 372 * int(w.clusters)), + mem_mb=(lambda w: 10000 + 372 * int(w.clusters)), + walltime=config["solving"].get("walltime", "12:00:00"), shadow: "minimal" conda: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 086375c2..8a93d24a 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -10,7 +10,6 @@ rule add_existing_baseyear: existing_capacities=config["existing_capacities"], costs=config["costs"], input: - overrides="data/override_component_attrs", network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", powerplants=RESOURCES + "powerplants.csv", @@ -52,7 +51,6 @@ rule add_brownfield: H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"], threshold_capacity=config["existing_capacities"]["threshold_capacity"], input: - overrides="data/override_component_attrs", network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", network_p=solved_previous_horizon, #solved network at previous time step @@ -91,11 +89,10 @@ rule solve_sector_network_myopic: "co2_sequestration_potential", 200 ), input: - overrides="data/override_component_attrs", network=RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", costs="data/costs_{planning_horizons}.csv", - config=RESULTS + "config/config.yaml", + config=RESULTS + "config.yaml", output: RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -109,6 +106,7 @@ rule solve_sector_network_myopic: threads: 4 resources: mem_mb=config["solving"]["mem"], + walltime=config["solving"].get("walltime", "12:00:00"), benchmark: ( BENCHMARKS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index fc430f4d..c7700760 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -12,12 +12,9 @@ rule solve_sector_network: "co2_sequestration_potential", 200 ), input: - overrides="data/override_component_attrs", network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - costs="data/costs_{}.csv".format(config["costs"]["year"]), - config=RESULTS + "config/config.yaml", - #env=RDIR + 'config/environment.yaml', + config=RESULTS + "config.yaml", output: RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -31,6 +28,7 @@ rule solve_sector_network: threads: config["solving"]["solver"].get("threads", 4) resources: mem_mb=config["solving"]["mem"], + walltime=config["solving"].get("walltime", "12:00:00"), benchmark: ( RESULTS diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 49561c36..162921e2 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -3,7 +3,6 @@ # SPDX-License-Identifier: MIT rule add_existing_baseyear: input: - overrides="data/override_component_attrs", network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", powerplants=RESOURCES + "powerplants.csv", @@ -41,7 +40,6 @@ rule add_existing_baseyear: rule add_brownfield: input: - overrides="data/override_component_attrs", network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", network_p=solved_previous_horizon, #solved network at previous time step @@ -82,7 +80,6 @@ rule prepare_perfect_foresight: + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" + "{}.nc".format(str(config["scenario"]["planning_horizons"][0])) ), - overrides="data/override_component_attrs", output: RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", @@ -105,7 +102,6 @@ rule prepare_perfect_foresight: rule solve_sector_network_perfect: input: - overrides="data/override_component_attrs", network=RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", costs="data/costs_2030.csv", @@ -148,7 +144,6 @@ rule make_summary_perfect: for ll in config["scenario"]["ll"] }, costs="data/costs_2020.csv", - overrides="data/override_component_attrs", output: nodal_costs=RESULTS + "csvs/nodal_costs.csv", nodal_capacities=RESULTS + "csvs/nodal_capacities.csv", diff --git a/rules/validate.smk b/rules/validate.smk new file mode 100644 index 00000000..cfb8c959 --- /dev/null +++ b/rules/validate.smk @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +PRODUCTION_PLOTS = [ + "production_bar", + "production_deviation_bar", + "seasonal_operation_area", +] +CROSS_BORDER_PLOTS = ["trade_time_series", "cross_border_bar"] +PRICES_PLOTS = ["price_bar", "price_line"] + + +rule build_electricity_production: + """ + This rule builds the electricity production for each country and technology from ENTSO-E data. + The data is used for validation of the optimization results. + """ + params: + snapshots=config["snapshots"], + countries=config["countries"], + output: + RESOURCES + "historical_electricity_production.csv", + log: + LOGS + "build_electricity_production.log", + resources: + mem_mb=5000, + script: + "../scripts/build_electricity_production.py" + + +rule build_cross_border_flows: + """ + This rule builds the cross-border flows from ENTSO-E data. + The data is used for validation of the optimization results. + """ + params: + snapshots=config["snapshots"], + countries=config["countries"], + input: + network=RESOURCES + "networks/base.nc", + output: + RESOURCES + "historical_cross_border_flows.csv", + log: + LOGS + "build_cross_border_flows.log", + resources: + mem_mb=5000, + script: + "../scripts/build_cross_border_flows.py" + + +rule build_electricity_prices: + """ + This rule builds the electricity prices from ENTSO-E data. + The data is used for validation of the optimization results. + """ + params: + snapshots=config["snapshots"], + countries=config["countries"], + output: + RESOURCES + "historical_electricity_prices.csv", + log: + LOGS + "build_electricity_prices.log", + resources: + mem_mb=5000, + script: + "../scripts/build_electricity_prices.py" + + +rule plot_validation_electricity_production: + input: + network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + electricity_production=RESOURCES + "historical_electricity_production.csv", + output: + **{ + plot: RESULTS + + f"figures/validation_{plot}_elec_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" + for plot in PRODUCTION_PLOTS + }, + plots_touch=RESULTS + + "figures/.validation_production_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", + script: + "../scripts/plot_validation_electricity_production.py" + + +rule plot_validation_cross_border_flows: + params: + countries=config["countries"], + input: + network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + cross_border_flows=RESOURCES + "historical_cross_border_flows.csv", + output: + **{ + plot: RESULTS + + f"figures/validation_{plot}_elec_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" + for plot in CROSS_BORDER_PLOTS + }, + plots_touch=RESULTS + + "figures/.validation_cross_border_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", + script: + "../scripts/plot_validation_cross_border_flows.py" + + +rule plot_validation_electricity_prices: + input: + network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + electricity_prices=RESOURCES + "historical_electricity_prices.csv", + output: + **{ + plot: RESULTS + + f"figures/validation_{plot}_elec_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf" + for plot in PRICES_PLOTS + }, + plots_touch=RESULTS + + "figures/.validation_prices_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", + script: + "../scripts/plot_validation_electricity_prices.py" diff --git a/scripts/_helpers.py b/scripts/_helpers.py index cd702950..fc7bc9e0 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -72,92 +72,6 @@ def configure_logging(snakemake, skip_handlers=False): logging.basicConfig(**kwargs) -def load_network(import_name=None, custom_components=None): - """ - Helper for importing a pypsa.Network with additional custom components. - - Parameters - ---------- - import_name : str - As in pypsa.Network(import_name) - custom_components : dict - Dictionary listing custom components. - For using ``snakemake.params['override_components']`` - in ``config/config.yaml`` define: - - .. code:: yaml - - override_components: - ShadowPrice: - component: ["shadow_prices","Shadow price for a global constraint.",np.nan] - attributes: - name: ["string","n/a","n/a","Unique name","Input (required)"] - value: ["float","n/a",0.,"shadow value","Output"] - - Returns - ------- - pypsa.Network - """ - import pypsa - from pypsa.descriptors import Dict - - override_components = None - override_component_attrs = None - - if custom_components is not None: - override_components = pypsa.components.components.copy() - override_component_attrs = Dict( - {k: v.copy() for k, v in pypsa.components.component_attrs.items()} - ) - for k, v in custom_components.items(): - override_components.loc[k] = v["component"] - override_component_attrs[k] = pd.DataFrame( - columns=["type", "unit", "default", "description", "status"] - ) - for attr, val in v["attributes"].items(): - override_component_attrs[k].loc[attr] = val - - return pypsa.Network( - import_name=import_name, - override_components=override_components, - override_component_attrs=override_component_attrs, - ) - - -def load_network_for_plots(fn, tech_costs, config, combine_hydro_ps=True): - import pypsa - from add_electricity import load_costs, update_transmission_costs - - n = pypsa.Network(fn) - - n.loads["carrier"] = n.loads.bus.map(n.buses.carrier) + " load" - n.stores["carrier"] = n.stores.bus.map(n.buses.carrier) - - n.links["carrier"] = ( - n.links.bus0.map(n.buses.carrier) + "-" + n.links.bus1.map(n.buses.carrier) - ) - n.lines["carrier"] = "AC line" - n.transformers["carrier"] = "AC transformer" - - n.lines["s_nom"] = n.lines["s_nom_min"] - n.links["p_nom"] = n.links["p_nom_min"] - - if combine_hydro_ps: - n.storage_units.loc[ - n.storage_units.carrier.isin({"PHS", "hydro"}), "carrier" - ] = "hydro+PHS" - - # if the carrier was not set on the heat storage units - # bus_carrier = n.storage_units.bus.map(n.buses.carrier) - # n.storage_units.loc[bus_carrier == "heat","carrier"] = "water tanks" - - Nyears = n.snapshot_weightings.objective.sum() / 8760.0 - costs = load_costs(tech_costs, config["costs"], config["electricity"], Nyears) - update_transmission_costs(n, costs) - - return n - - def update_p_nom_max(n): # if extendable carriers (solar/onwind/...) have capacity >= 0, # e.g. existing assets from the OPSD project are included to the network, @@ -367,34 +281,6 @@ def mock_snakemake(rulename, configfiles=[], **wildcards): return snakemake -def override_component_attrs(directory): - """ - Tell PyPSA that links can have multiple outputs by overriding the - component_attrs. This can be done for as many buses as you need with format - busi for i = 2,3,4,5,.... See https://pypsa.org/doc/components.html#link- - with-multiple-outputs-or-inputs. - - Parameters - ---------- - directory : string - Folder where component attributes to override are stored - analogous to ``pypsa/component_attrs``, e.g. `links.csv`. - - Returns - ------- - Dictionary of overridden component attributes. - """ - attrs = Dict({k: v.copy() for k, v in component_attrs.items()}) - - for component, list_name in components.list_name.items(): - fn = f"{directory}/{list_name}.csv" - if os.path.isfile(fn): - overrides = pd.read_csv(fn, index_col=0, na_values="n/a") - attrs[component] = overrides.combine_first(attrs[component]) - - return attrs - - def generate_periodic_profiles(dt_index, nodes, weekly_profile, localize=None): """ Give a 24*7 long list of weekly hourly profiles, generate this for each diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index b33d2f19..597792c0 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -16,7 +16,7 @@ idx = pd.IndexSlice import numpy as np import pypsa -from _helpers import override_component_attrs, update_config_with_sector_opts +from _helpers import update_config_with_sector_opts from add_existing_baseyear import add_build_year_to_new_assets @@ -147,12 +147,11 @@ if __name__ == "__main__": year = int(snakemake.wildcards.planning_horizons) - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) + n = pypsa.Network(snakemake.input.network) add_build_year_to_new_assets(n, year) - n_p = pypsa.Network(snakemake.input.network_p, override_component_attrs=overrides) + n_p = pypsa.Network(snakemake.input.network_p) add_brownfield(n, n_p, year) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 7cea165b..56375800 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -2,8 +2,6 @@ # SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors # # SPDX-License-Identifier: MIT - -# coding: utf-8 """ Adds electrical generators and existing hydro storage units to a base network. @@ -206,7 +204,6 @@ def load_costs(tech_costs, config, max_hours, Nyears=1.0): * costs["investment"] * Nyears ) - costs.at["OCGT", "fuel"] = costs.at["gas", "fuel"] costs.at["CCGT", "fuel"] = costs.at["gas", "fuel"] @@ -362,7 +359,6 @@ def attach_wind_and_solar( n, costs, input_profiles, carriers, extendable_carriers, line_length_factor=1 ): add_missing_carriers(n, carriers) - for car in carriers: if car == "hydro": continue @@ -421,26 +417,54 @@ def attach_conventional_generators( extendable_carriers, conventional_params, conventional_inputs, + unit_commitment=None, + fuel_price=None, ): carriers = list(set(conventional_carriers) | set(extendable_carriers["Generator"])) add_missing_carriers(n, carriers) add_co2_emissions(n, costs, carriers) + # Replace carrier "natural gas" with the respective technology (OCGT or + # CCGT) to align with PyPSA names of "carriers" and avoid filtering "natural + # gas" powerplants in ppl.query("carrier in @carriers") + ppl.loc[ppl["carrier"] == "natural gas", "carrier"] = ppl.loc[ + ppl["carrier"] == "natural gas", "technology" + ] + ppl = ( ppl.query("carrier in @carriers") .join(costs, on="carrier", rsuffix="_r") .rename(index=lambda s: "C" + str(s)) ) ppl["efficiency"] = ppl.efficiency.fillna(ppl.efficiency_r) - ppl["marginal_cost"] = ( - ppl.carrier.map(costs.VOM) + ppl.carrier.map(costs.fuel) / ppl.efficiency - ) - logger.info( - "Adding {} generators with capacities [GW] \n{}".format( - len(ppl), ppl.groupby("carrier").p_nom.sum().div(1e3).round(2) + if unit_commitment is not None: + committable_attrs = ppl.carrier.isin(unit_commitment).to_frame("committable") + for attr in unit_commitment.index: + default = pypsa.components.component_attrs["Generator"].default[attr] + committable_attrs[attr] = ppl.carrier.map(unit_commitment.loc[attr]).fillna( + default + ) + else: + committable_attrs = {} + + if fuel_price is not None: + fuel_price = fuel_price.assign( + OCGT=fuel_price["gas"], CCGT=fuel_price["gas"] + ).drop("gas", axis=1) + missing_carriers = list(set(carriers) - set(fuel_price)) + fuel_price = fuel_price.assign(**costs.fuel[missing_carriers]) + fuel_price = fuel_price.reindex(ppl.carrier, axis=1) + fuel_price.columns = ppl.index + marginal_cost = fuel_price.div(ppl.efficiency).add(ppl.carrier.map(costs.VOM)) + else: + marginal_cost = ( + ppl.carrier.map(costs.VOM) + ppl.carrier.map(costs.fuel) / ppl.efficiency ) - ) + + # Define generators using modified ppl DataFrame + caps = ppl.groupby("carrier").p_nom.sum().div(1e3).round(2) + logger.info(f"Adding {len(ppl)} generators with capacities [GW] \n{caps}") n.madd( "Generator", @@ -451,13 +475,14 @@ def attach_conventional_generators( p_nom=ppl.p_nom.where(ppl.carrier.isin(conventional_carriers), 0), p_nom_extendable=ppl.carrier.isin(extendable_carriers["Generator"]), efficiency=ppl.efficiency, - marginal_cost=ppl.marginal_cost, + marginal_cost=marginal_cost, capital_cost=ppl.capital_cost, build_year=ppl.datein.fillna(0).astype(int), lifetime=(ppl.dateout - ppl.datein).fillna(np.inf), + **committable_attrs, ) - for carrier in conventional_params: + for carrier in set(conventional_params) & set(carriers): # Generators with technology affected idx = n.generators.query("carrier == @carrier").index @@ -591,6 +616,14 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **par hydro.max_hours > 0, hydro.country.map(max_hours_country) ).fillna(6) + flatten_dispatch = params.get("flatten_dispatch", False) + if flatten_dispatch: + buffer = params.get("flatten_dispatch_buffer", 0.2) + average_capacity_factor = inflow_t[hydro.index].mean() / hydro["p_nom"] + p_max_pu = (average_capacity_factor + buffer).clip(upper=1) + else: + p_max_pu = 1 + n.madd( "StorageUnit", hydro.index, @@ -600,7 +633,7 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **par max_hours=hydro_max_hours, capital_cost=costs.at["hydro", "capital_cost"], marginal_cost=costs.at["hydro", "marginal_cost"], - p_max_pu=1.0, # dispatch + p_max_pu=p_max_pu, # dispatch p_min_pu=0.0, # store efficiency_dispatch=costs.at["hydro", "efficiency"], efficiency_store=0.0, @@ -690,13 +723,14 @@ def attach_OPSD_renewables(n, tech_map): {"Solar": "PV"} ) df = df.query("Fueltype in @tech_map").powerplant.convert_country_to_alpha2() + df = df.dropna(subset=["lat", "lon"]) for fueltype, carriers in tech_map.items(): gens = n.generators[lambda df: df.carrier.isin(carriers)] buses = n.buses.loc[gens.bus.unique()] gens_per_bus = gens.groupby("bus").p_nom.count() - caps = map_country_bus(df.query("Fueltype == @fueltype and lat == lat"), buses) + caps = map_country_bus(df.query("Fueltype == @fueltype"), buses) caps = caps.groupby(["bus"]).Capacity.sum() caps = caps / gens_per_bus.reindex(caps.index, fill_value=1) @@ -745,6 +779,30 @@ def estimate_renewable_capacities(n, year, tech_map, expansion_limit, countries) ) +def attach_line_rating( + n, rating, s_max_pu, correction_factor, max_voltage_difference, max_line_rating +): + # TODO: Only considers overhead lines + n.lines_t.s_max_pu = (rating / n.lines.s_nom[rating.columns]) * correction_factor + if max_voltage_difference: + x_pu = ( + n.lines.type.map(n.line_types["x_per_length"]) + * n.lines.length + / (n.lines.v_nom**2) + ) + # need to clip here as cap values might be below 1 + # -> would mean the line cannot be operated at actual given pessimistic ampacity + s_max_pu_cap = ( + np.deg2rad(max_voltage_difference) / (x_pu * n.lines.s_nom) + ).clip(lower=1) + n.lines_t.s_max_pu = n.lines_t.s_max_pu.clip( + lower=1, upper=s_max_pu_cap, axis=1 + ) + if max_line_rating: + n.lines_t.s_max_pu = n.lines_t.s_max_pu.clip(upper=max_line_rating) + n.lines_t.s_max_pu *= s_max_pu + + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -782,6 +840,20 @@ if __name__ == "__main__": conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") } + + if params.conventional["unit_commitment"]: + unit_commitment = pd.read_csv(snakemake.input.unit_commitment, index_col=0) + else: + unit_commitment = None + + if params.conventional["dynamic_fuel_price"]: + fuel_price = pd.read_csv( + snakemake.input.fuel_price, index_col=0, header=0, parse_dates=True + ) + fuel_price = fuel_price.reindex(n.snapshots).fillna(method="ffill") + else: + fuel_price = None + attach_conventional_generators( n, costs, @@ -790,6 +862,8 @@ if __name__ == "__main__": extendable_carriers, params.conventional, conventional_inputs, + unit_commitment=unit_commitment, + fuel_price=fuel_price, ) attach_wind_and_solar( @@ -802,15 +876,16 @@ if __name__ == "__main__": ) if "hydro" in renewable_carriers: - para = params.renewable["hydro"] + p = params.renewable["hydro"] + carriers = p.pop("carriers", []) attach_hydro( n, costs, ppl, snakemake.input.profile_hydro, snakemake.input.hydro_capacities, - para.pop("carriers", []), - **para, + carriers, + **p, ) estimate_renewable_caps = params.electricity["estimate_renewable_capacities"] @@ -827,6 +902,23 @@ if __name__ == "__main__": update_p_nom_max(n) + line_rating_config = snakemake.config["lines"]["dynamic_line_rating"] + if line_rating_config["activate"]: + rating = xr.open_dataarray(snakemake.input.line_rating).to_pandas().transpose() + s_max_pu = snakemake.config["lines"]["s_max_pu"] + correction_factor = line_rating_config["correction_factor"] + max_voltage_difference = line_rating_config["max_voltage_difference"] + max_line_rating = line_rating_config["max_line_rating"] + + attach_line_rating( + n, + rating, + s_max_pu, + correction_factor, + max_voltage_difference, + max_line_rating, + ) + sanitize_carriers(n, snakemake.config) n.meta = snakemake.config diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index f80bffb8..b2c534e6 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -21,7 +21,7 @@ import country_converter as coco import numpy as np import pypsa import xarray as xr -from _helpers import override_component_attrs, update_config_with_sector_opts +from _helpers import update_config_with_sector_opts from add_electricity import sanitize_carriers from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs @@ -129,10 +129,14 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas "Oil": "oil", "OCGT": "OCGT", "CCGT": "CCGT", - "Natural Gas": "gas", "Bioenergy": "urban central solid biomass CHP", } + # Replace Fueltype "Natural Gas" with the respective technology (OCGT or CCGT) + df_agg.loc[df_agg["Fueltype"] == "Natural Gas", "Fueltype"] = df_agg.loc[ + df_agg["Fueltype"] == "Natural Gas", "Technology" + ] + fueltype_to_drop = [ "Hydro", "Wind", @@ -601,12 +605,13 @@ if __name__ == "__main__": snakemake = mock_snakemake( "add_existing_baseyear", + configfiles="config/test/config.myopic.yaml", simpl="", - clusters="45", - ll="v1.0", + clusters="5", + ll="v1.5", opts="", - sector_opts="8760H-T-H-B-I-A-solar+p3-dist1", - planning_horizons=2020, + sector_opts="24H-T-H-B-I-A-solar+p3-dist1", + planning_horizons=2030, ) logging.basicConfig(level=snakemake.config["logging"]["level"]) @@ -618,8 +623,8 @@ if __name__ == "__main__": baseyear = snakemake.params.baseyear - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) + n = pypsa.Network(snakemake.input.network) + # define spatial resolution of carriers spatial = define_spatial(n.buses[n.buses.carrier == "AC"].index, options) add_build_year_to_new_assets(n, baseyear) diff --git a/scripts/base_network.py b/scripts/base_network.py index 87504ce7..b4ac1d8c 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -337,7 +337,7 @@ def _load_lines_from_eg(buses, eg_lines): ) lines["length"] /= 1e3 - + lines["carrier"] = "AC" lines = _remove_dangling_branches(lines, buses) return lines diff --git a/scripts/build_cross_border_flows.py b/scripts/build_cross_border_flows.py new file mode 100644 index 00000000..b9fc3fe8 --- /dev/null +++ b/scripts/build_cross_border_flows.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +import logging + +import pandas as pd +import pypsa +from _helpers import configure_logging +from entsoe import EntsoePandasClient +from entsoe.exceptions import InvalidBusinessParameterError, NoMatchingDataError +from requests import HTTPError + +logger = logging.getLogger(__name__) + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("build_cross_border_flows") + configure_logging(snakemake) + + api_key = snakemake.config["private"]["keys"]["entsoe_api"] + client = EntsoePandasClient(api_key=api_key) + + n = pypsa.Network(snakemake.input.network) + start = pd.Timestamp(snakemake.params.snapshots["start"], tz="Europe/Brussels") + end = pd.Timestamp(snakemake.params.snapshots["end"], tz="Europe/Brussels") + + branches = n.branches().query("carrier in ['AC', 'DC']") + c = n.buses.country + branch_countries = pd.concat([branches.bus0.map(c), branches.bus1.map(c)], axis=1) + branch_countries = branch_countries.query("bus0 != bus1") + branch_countries = branch_countries.apply(sorted, axis=1, result_type="broadcast") + country_pairs = branch_countries.drop_duplicates().reset_index(drop=True) + + flows = [] + unavailable_borders = [] + for from_country, to_country in country_pairs.values: + try: + flow_directed = client.query_crossborder_flows( + from_country, to_country, start=start, end=end + ) + flow_reverse = client.query_crossborder_flows( + to_country, from_country, start=start, end=end + ) + flow = (flow_directed - flow_reverse).rename( + f"{from_country} - {to_country}" + ) + flow = flow.tz_localize(None).resample("1h").mean() + flow = flow.loc[start.tz_localize(None) : end.tz_localize(None)] + flows.append(flow) + except (HTTPError, NoMatchingDataError, InvalidBusinessParameterError): + unavailable_borders.append(f"{from_country}-{to_country}") + + if unavailable_borders: + logger.warning( + "Historical electricity cross-border flows for countries" + f" {', '.join(unavailable_borders)} not available." + ) + + flows = pd.concat(flows, axis=1) + flows.to_csv(snakemake.output[0]) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index ba1fb881..38c75544 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -80,11 +80,9 @@ def load_timeseries(fn, years, countries, powerstatistics=True): def rename(s): return s[: -len(pattern)] - def date_parser(x): - return dateutil.parser.parse(x, ignoretz=True) - return ( - pd.read_csv(fn, index_col=0, parse_dates=[0], date_parser=date_parser) + pd.read_csv(fn, index_col=0, parse_dates=[0]) + .tz_localize(None) .filter(like=pattern) .rename(columns=rename) .dropna(how="all", axis=0) @@ -168,6 +166,7 @@ def manual_adjustment(load, fn_load, powerstatistics): by the corresponding ratio of total energy consumptions reported by IEA Data browser [0] for the year 2013. + 2. For the ENTSOE transparency load data (if powerstatistics is False) Albania (AL) and Macedonia (MK) do not exist in the data set. Both get the @@ -176,6 +175,9 @@ def manual_adjustment(load, fn_load, powerstatistics): [0] https://www.iea.org/data-and-statistics?country=WORLD&fuel=Electricity%20and%20heat&indicator=TotElecCons + Bosnia and Herzegovina (BA) does not exist in the data set for 2019. It gets the + electricity consumption data from Croatia (HR) for the year 2019, scaled by the + factors derived from https://energy.at-site.be/eurostat-2021/ Parameters ---------- @@ -264,9 +266,17 @@ def manual_adjustment(load, fn_load, powerstatistics): load["AL"] = load.ME * (5.7 / 2.9) if "MK" not in load and "MK" in countries: load["MK"] = load.ME * (6.7 / 2.9) + if "BA" not in load and "BA" in countries: + load["BA"] = load.HR * (11.0 / 16.2) copy_timeslice( load, "BG", "2018-10-27 21:00", "2018-10-28 22:00", Delta(weeks=1) ) + copy_timeslice( + load, "LU", "2019-01-02 11:00", "2019-01-05 05:00", Delta(weeks=-1) + ) + copy_timeslice( + load, "LU", "2019-02-05 20:00", "2019-02-06 19:00", Delta(weeks=-1) + ) return load @@ -291,6 +301,9 @@ if __name__ == "__main__": if snakemake.params.load["manual_adjustments"]: load = manual_adjustment(load, snakemake.input[0], powerstatistics) + if load.empty: + logger.warning("Build electricity demand time series is empty.") + logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.") load = load.interpolate(method="linear", limit=interpolate_limit) diff --git a/scripts/build_electricity_prices.py b/scripts/build_electricity_prices.py new file mode 100644 index 00000000..353ea7e3 --- /dev/null +++ b/scripts/build_electricity_prices.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +import logging + +import pandas as pd +from _helpers import configure_logging +from entsoe import EntsoePandasClient +from entsoe.exceptions import NoMatchingDataError + +logger = logging.getLogger(__name__) + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("build_cross_border_flows") + configure_logging(snakemake) + + api_key = snakemake.config["private"]["keys"]["entsoe_api"] + client = EntsoePandasClient(api_key=api_key) + + start = pd.Timestamp(snakemake.params.snapshots["start"], tz="Europe/Brussels") + end = pd.Timestamp(snakemake.params.snapshots["end"], tz="Europe/Brussels") + + countries = snakemake.params.countries + + prices = [] + unavailable_countries = [] + + for country in countries: + country_code = country + + try: + gen = client.query_day_ahead_prices(country, start=start, end=end) + gen = gen.tz_localize(None).resample("1h").mean() + gen = gen.loc[start.tz_localize(None) : end.tz_localize(None)] + prices.append(gen) + except NoMatchingDataError: + unavailable_countries.append(country) + + if unavailable_countries: + logger.warning( + f"Historical electricity prices for countries {', '.join(unavailable_countries)} not available." + ) + + keys = [c for c in countries if c not in unavailable_countries] + prices = pd.concat(prices, keys=keys, axis=1) + prices.to_csv(snakemake.output[0]) diff --git a/scripts/build_electricity_production.py b/scripts/build_electricity_production.py new file mode 100644 index 00000000..beb859bd --- /dev/null +++ b/scripts/build_electricity_production.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +import logging + +import pandas as pd +from _helpers import configure_logging +from entsoe import EntsoePandasClient +from entsoe.exceptions import NoMatchingDataError + +logger = logging.getLogger(__name__) + + +carrier_grouper = { + "Waste": "Biomass", + "Hydro Pumped Storage": "Hydro", + "Hydro Water Reservoir": "Hydro", + "Hydro Run-of-river and poundage": "Run of River", + "Fossil Coal-derived gas": "Gas", + "Fossil Gas": "Gas", + "Fossil Oil": "Oil", + "Fossil Oil shale": "Oil", + "Fossil Brown coal/Lignite": "Lignite", + "Fossil Peat": "Lignite", + "Fossil Hard coal": "Coal", + "Wind Onshore": "Onshore Wind", + "Wind Offshore": "Offshore Wind", + "Other renewable": "Other", + "Marine": "Other", +} + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("build_electricity_production") + configure_logging(snakemake) + + api_key = snakemake.config["private"]["keys"]["entsoe_api"] + client = EntsoePandasClient(api_key=api_key) + + start = pd.Timestamp(snakemake.params.snapshots["start"], tz="Europe/Brussels") + end = pd.Timestamp(snakemake.params.snapshots["end"], tz="Europe/Brussels") + + countries = snakemake.params.countries + + generation = [] + unavailable_countries = [] + + for country in countries: + country_code = country + + try: + gen = client.query_generation(country, start=start, end=end, nett=True) + gen = gen.tz_localize(None).resample("1h").mean() + gen = gen.loc[start.tz_localize(None) : end.tz_localize(None)] + gen = gen.rename(columns=carrier_grouper).groupby(level=0, axis=1).sum() + generation.append(gen) + except NoMatchingDataError: + unavailable_countries.append(country) + + if unavailable_countries: + logger.warning( + f"Historical electricity production for countries {', '.join(unavailable_countries)} not available." + ) + + keys = [c for c in countries if c not in unavailable_countries] + generation = pd.concat(generation, keys=keys, axis=1) + generation.to_csv(snakemake.output[0]) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index e93e43c2..335c6a36 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -92,6 +92,20 @@ def prepare_hotmaps_database(regions): gdf.rename(columns={"index_right": "bus"}, inplace=True) gdf["country"] = gdf.bus.str[:2] + + # the .sjoin can lead to duplicates if a geom is in two regions + if gdf.index.duplicated().any(): + import pycountry + # get all duplicated entries + duplicated_i = gdf.index[gdf.index.duplicated()] + # convert from raw data country name to iso-2-code + s = df.loc[duplicated_i, "Country"].apply(lambda x: pycountry.countries.lookup(x).alpha_2) + # Get a boolean mask where gdf's country column matches s's values for the same index + mask = gdf['country'] == gdf.index.map(s) + # Filter gdf using the mask + gdf_filtered = gdf[mask] + # concat not duplicated and filtered gdf + gdf = pd.concat([gdf.drop(duplicated_i), gdf_filtered]).sort_index() return gdf @@ -115,7 +129,9 @@ def build_nodal_distribution_key(hotmaps, regions, countries): facilities = hotmaps.query("country == @country and Subsector == @sector") if not facilities.empty: - emissions = facilities["Emissions_ETS_2014"] + emissions = facilities["Emissions_ETS_2014"].fillna( + hotmaps["Emissions_EPRTR_2014"] + ) if emissions.sum() == 0: key = pd.Series(1 / len(facilities), facilities.index) else: @@ -130,7 +146,7 @@ def build_nodal_distribution_key(hotmaps, regions, countries): return keys - +#%% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -138,7 +154,7 @@ if __name__ == "__main__": snakemake = mock_snakemake( "build_industrial_distribution_key", simpl="", - clusters=48, + clusters=128, ) logging.basicConfig(level=snakemake.config["logging"]["level"]) diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py new file mode 100755 index 00000000..7f842d43 --- /dev/null +++ b/scripts/build_line_rating.py @@ -0,0 +1,156 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2020 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +# coding: utf-8 +""" +Adds dynamic line rating timeseries to the base network. + +Relevant Settings +----------------- + +.. code:: yaml + + lines: + cutout: + line_rating: + + +.. seealso:: + Documentation of the configuration file ``config.yaml` +Inputs +------ + +- ``data/cutouts``: +- ``networks/base.nc``: confer :ref:`base` + +Outputs +------- + +- ``resources/line_rating.nc`` + + +Description +----------- + +The rule :mod:`build_line_rating` calculates the line rating for transmission lines. +The line rating provides the maximal capacity of a transmission line considering the heat exchange with the environment. + +The following heat gains and losses are considered: + +- heat gain through resistive losses +- heat gain through solar radiation +- heat loss through radiation of the trasnmission line +- heat loss through forced convection with wind +- heat loss through natural convection + + +With a heat balance considering the maximum temperature threshold of the transmission line, +the maximal possible capacity factor "s_max_pu" for each transmission line at each time step is calculated. +""" + +import logging +import re + +import atlite +import geopandas as gpd +import numpy as np +import pandas as pd +import pypsa +import xarray as xr +from _helpers import configure_logging +from shapely.geometry import LineString as Line +from shapely.geometry import Point + + +def calculate_resistance(T, R_ref, T_ref=293, alpha=0.00403): + """ + Calculates the resistance at other temperatures than the reference + temperature. + + Parameters + ---------- + T : Temperature at which resistance is calculated in [°C] or [K] + R_ref : Resistance at reference temperature in [Ohm] or [Ohm/Per Length Unit] + T_ref : Reference temperature in [°C] or [K] + alpha: Temperature coefficient in [1/K] + Defaults are: + * T_ref : 20 °C + * alpha : 0.00403 1/K + + Returns + ------- + Resistance of at given temperature. + """ + R = R_ref * (1 + alpha * (T - T_ref)) + return R + + +def calculate_line_rating(n, cutout): + """ + Calculates the maximal allowed power flow in each line for each time step + considering the maximal temperature. + + Parameters + ---------- + n : pypsa.Network object containing information on grid + + Returns + ------- + xarray DataArray object with maximal power. + """ + relevant_lines = n.lines[(n.lines["underground"] == False)] + buses = relevant_lines[["bus0", "bus1"]].values + x = n.buses.x + y = n.buses.y + shapes = [Line([Point(x[b0], y[b0]), Point(x[b1], y[b1])]) for (b0, b1) in buses] + shapes = gpd.GeoSeries(shapes, index=relevant_lines.index) + if relevant_lines.r_pu.eq(0).all(): + # Overwrite standard line resistance with line resistance obtained from line type + r_per_length = n.line_types["r_per_length"] + R = ( + relevant_lines.join(r_per_length, on=["type"])["r_per_length"] / 1000 + ) # in meters + # If line type with bundles is given retrieve number of conductors per bundle + relevant_lines["n_bundle"] = ( + relevant_lines["type"] + .where(relevant_lines["type"].str.contains("bundle")) + .dropna() + .apply(lambda x: int(re.findall(r"(\d+)-bundle", x)[0])) + ) + # Set default number of bundles per line + relevant_lines["n_bundle"].fillna(1, inplace=True) + R *= relevant_lines["n_bundle"] + R = calculate_resistance(T=353, R_ref=R) + Imax = cutout.line_rating(shapes, R, D=0.0218, Ts=353, epsilon=0.8, alpha=0.8) + line_factor = relevant_lines.eval("v_nom * n_bundle * num_parallel") / 1e3 # in mW + da = xr.DataArray( + data=np.sqrt(3) * Imax * line_factor.values.reshape(-1, 1), + attrs=dict( + description="Maximal possible power in MW for given line considering line rating" + ), + ) + return da + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake( + "build_line_rating", + network="elec", + simpl="", + clusters="5", + ll="v1.0", + opts="Co2L-4H", + ) + configure_logging(snakemake) + + n = pypsa.Network(snakemake.input.base_network) + time = pd.date_range(freq="h", **snakemake.config["snapshots"]) + cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) + + da = calculate_line_rating(n, cutout) + da.to_netcdf(snakemake.output[0]) diff --git a/scripts/build_monthly_prices.py b/scripts/build_monthly_prices.py new file mode 100644 index 00000000..c2e88972 --- /dev/null +++ b/scripts/build_monthly_prices.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Tue May 16 10:37:35 2023. + +This script extracts monthly fuel prices of oil, gas, coal and lignite, +as well as CO2 prices + + +Inputs +------ +- ``data/energy-price-trends-xlsx-5619002.xlsx``: energy price index of fossil fuels +- ``emission-spot-primary-market-auction-report-2019-data.xls``: CO2 Prices spot primary auction + + +Outputs +------- + +- ``data/validation/monthly_fuel_price.csv`` +- ``data/validation/CO2_price_2019.csv`` + + +Description +----------- + +The rule :mod:`build_monthly_prices` collects monthly fuel prices and CO2 prices +and translates them from different input sources to pypsa syntax + +Data sources: + [1] Fuel price index. Destatis + https://www.destatis.de/EN/Home/_node.html + [2] average annual fuel price lignite, ENTSO-E + https://2020.entsos-tyndp-scenarios.eu/fuel-commodities-and-carbon-prices/ + [3] CO2 Prices, Emission spot primary auction, EEX + https://www.eex.com/en/market-data/environmental-markets/eua-primary-auction-spot-download + + +Data was accessed at 16.5.2023 +""" + +import logging + +import pandas as pd +from _helpers import configure_logging + +logger = logging.getLogger(__name__) + + +# keywords in datasheet +keywords = { + "coal": " GP09-051 Hard coal", + "lignite": " GP09-052 Lignite and lignite briquettes", + "oil": " GP09-0610 10 Mineral oil, crude", + "gas": "GP09-062 Natural gas", +} + +# sheet names to pypsa syntax +sheet_name_map = { + "coal": "5.1 Hard coal and lignite", + "lignite": "5.1 Hard coal and lignite", + "oil": "5.2 Mineral oil", + "gas": "5.3.1 Natural gas - indices", +} + + +# import fuel price 2015 in Eur/MWh +# source lignite, price for 2020, scaled by price index, ENTSO-E [3] +price_2020 = ( + pd.Series({"coal": 3.0, "oil": 10.6, "gas": 5.6, "lignite": 1.1}) * 3.6 +) # Eur/MWh + +# manual adjustment of coal price +price_2020["coal"] = 2.4 * 3.6 +price_2020["lignite"] = 1.6 * 3.6 + + +def get_fuel_price(): + price = {} + for carrier, keyword in keywords.items(): + sheet_name = sheet_name_map[carrier] + df = pd.read_excel( + snakemake.input.fuel_price_raw, + sheet_name=sheet_name, + index_col=0, + skiprows=6, + nrows=18, + ) + df = df.dropna(axis=0).iloc[:, :12] + start, end = df.index[0], str(int(df.index[-1][:4]) + 1) + df = df.stack() + df.index = pd.date_range(start=start, end=end, freq="MS", inclusive="left") + scale = price_2020[carrier] / df["2020"].mean() # scale to 2020 price + df = df.mul(scale) + price[carrier] = df + + return pd.concat(price, axis=1) + + +def get_co2_price(): + # emission price + co2_price = pd.read_excel(snakemake.input.co2_price_raw, index_col=1, header=5) + return co2_price["Auction Price €/tCO2"] + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("build_monthly_prices") + + configure_logging(snakemake) + + fuel_price = get_fuel_price() + fuel_price.to_csv(snakemake.output.fuel_price) + + co2_price = get_co2_price() + co2_price.to_csv(snakemake.output.co2_price) diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py index 3cd62fd9..8fdb4ea3 100644 --- a/scripts/build_natura_raster.py +++ b/scripts/build_natura_raster.py @@ -54,6 +54,23 @@ logger = logging.getLogger(__name__) def determine_cutout_xXyY(cutout_name): + """ + Determine the full extent of a cutout. + + Since the coordinates of the cutout data are given as the + center of the grid cells, the extent of the cutout is + calculated by adding/subtracting half of the grid cell size. + + + Parameters + ---------- + cutout_name : str + Path to the cutout. + + Returns + ------- + A list of extent coordinates in the order [x, X, y, Y]. + """ cutout = atlite.Cutout(cutout_name) assert cutout.crs.to_epsg() == 4326 x, X, y, Y = cutout.extent diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index f25f111b..cbe94505 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -98,13 +98,15 @@ def add_custom_powerplants(ppl, custom_powerplants, custom_ppl_query=False): def replace_natural_gas_technology(df): - mapping = {"Steam Turbine": "OCGT", "Combustion Engine": "OCGT"} - tech = df.Technology.replace(mapping).fillna("OCGT") - return df.Technology.where(df.Fueltype != "Natural Gas", tech) + mapping = {"Steam Turbine": "CCGT", "Combustion Engine": "OCGT"} + tech = df.Technology.replace(mapping).fillna("CCGT") + return df.Technology.mask(df.Fueltype == "Natural Gas", tech) def replace_natural_gas_fueltype(df): - return df.Fueltype.where(df.Fueltype != "Natural Gas", df.Technology) + return df.Fueltype.mask( + (df.Technology == "OCGT") | (df.Technology == "CCGT"), "Natural Gas" + ) if __name__ == "__main__": diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 88a43f53..7b08325b 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -186,6 +186,7 @@ import time import atlite import geopandas as gpd import numpy as np +import pandas as pd import xarray as xr from _helpers import configure_logging from dask.distributed import Client @@ -222,7 +223,8 @@ if __name__ == "__main__": else: client = None - cutout = atlite.Cutout(snakemake.input.cutout) + sns = pd.date_range(freq="h", **snakemake.config["snapshots"]) + cutout = atlite.Cutout(snakemake.input.cutout).sel(time=sns) regions = gpd.read_file(snakemake.input.regions) assert not regions.empty, ( f"List of regions in {snakemake.input.regions} is empty, please " @@ -372,4 +374,6 @@ if __name__ == "__main__": ds["profile"] = ds["profile"].where(ds["profile"] >= min_p_max_pu, 0) ds.to_netcdf(snakemake.output.profile) - client.shutdown() + + if client is not None: + client.shutdown() diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index e19a96da..f6ad3526 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -28,9 +28,7 @@ def allocate_sequestration_potential( overlay["share"] = area(overlay) / overlay["area_sqkm"] adjust_cols = overlay.columns.difference({"name", "area_sqkm", "geometry", "share"}) overlay[adjust_cols] = overlay[adjust_cols].multiply(overlay["share"], axis=0) - gdf_regions = overlay.groupby("name").sum() - gdf_regions.drop(["area_sqkm", "share"], axis=1, inplace=True) - return gdf_regions.squeeze() + return overlay.dissolve("name", aggfunc="sum")[attr] if __name__ == "__main__": diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index f0fd80eb..884b6a2b 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -461,7 +461,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("cluster_network", simpl="", clusters="37c") + snakemake = mock_snakemake("cluster_network", simpl="", clusters="37") configure_logging(snakemake) params = snakemake.params @@ -483,6 +483,23 @@ if __name__ == "__main__": else: n_clusters = int(snakemake.wildcards.clusters) + if params.cluster_network.get("consider_efficiency_classes", False): + carriers = [] + for c in aggregate_carriers: + gens = n.generators.query("carrier == @c") + low = gens.efficiency.quantile(0.10) + high = gens.efficiency.quantile(0.90) + if low >= high: + carriers += [c] + else: + labels = ["low", "medium", "high"] + suffix = pd.cut( + gens.efficiency, bins=[0, low, high, 1], labels=labels + ).astype(str) + carriers += [f"{c} {label} efficiency" for label in labels] + n.generators.carrier.update(gens.carrier + " " + suffix + " efficiency") + aggregate_carriers = carriers + if n_clusters == len(n.buses): # Fast-path if no clustering is necessary busmap = n.buses.index.to_series() @@ -524,6 +541,11 @@ if __name__ == "__main__": update_p_nom_max(clustering.network) + if params.cluster_network.get("consider_efficiency_classes"): + labels = [f" {label} efficiency" for label in ["low", "medium", "high"]] + nc = clustering.network + nc.generators["carrier"] = nc.generators.carrier.replace(labels, "", regex=True) + clustering.network.meta = dict( snakemake.config, **dict(wildcards=dict(snakemake.wildcards)) ) diff --git a/scripts/copy_config.py b/scripts/copy_config.py index 79d2e32b..a549d893 100644 --- a/scripts/copy_config.py +++ b/scripts/copy_config.py @@ -11,25 +11,13 @@ from shutil import copy import yaml -files = { - "config/config.yaml": "config.yaml", - "Snakefile": "Snakefile", - "scripts/solve_network.py": "solve_network.py", - "scripts/prepare_sector_network.py": "prepare_sector_network.py", -} - if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake snakemake = mock_snakemake("copy_config") - basepath = Path(f"results/{snakemake.params.RDIR}config/") - - for f, name in files.items(): - copy(f, basepath / name) - - with open(basepath / "config.snakemake.yaml", "w") as yaml_file: + with open(snakemake.output[0], "w") as yaml_file: yaml.dump( snakemake.config, yaml_file, diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 00fca2fd..56ee98c9 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -16,7 +16,6 @@ import sys import numpy as np import pandas as pd import pypsa -from _helpers import override_component_attrs from prepare_sector_network import prepare_costs idx = pd.IndexSlice @@ -300,9 +299,9 @@ def calculate_energy(n, label, energy): ) # remove values where bus is missing (bug in nomopyomo) no_bus = c.df.index[c.df["bus" + port] == ""] - totals.loc[no_bus] = n.component_attrs[c.name].loc[ - "p" + port, "default" - ] + totals.loc[no_bus] = float( + n.component_attrs[c.name].loc["p" + port, "default"] + ) c_energies -= totals.groupby(c.df.carrier).sum() c_energies = pd.concat([c_energies], keys=[c.list_name]) @@ -659,8 +658,7 @@ def make_summaries(networks_dict): for label, filename in networks_dict.items(): logger.info(f"Make summary for scenario {label}, using {filename}") - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(filename, override_component_attrs=overrides) + n = pypsa.Network(filename) assign_carriers(n) assign_locations(n) diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 184d86f0..ae1d0e0a 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -20,7 +20,6 @@ import geopandas as gpd import matplotlib.pyplot as plt import pandas as pd import pypsa -from _helpers import override_component_attrs from make_summary import assign_carriers from plot_summary import preferred_order, rename_techs from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches @@ -930,8 +929,7 @@ if __name__ == "__main__": logging.basicConfig(level=snakemake.config["logging"]["level"]) - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) + n = pypsa.Network(snakemake.input.network) regions = gpd.read_file(snakemake.input.regions).set_index("name") diff --git a/scripts/plot_statistics.py b/scripts/plot_statistics.py new file mode 100644 index 00000000..1e75203f --- /dev/null +++ b/scripts/plot_statistics.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +import matplotlib.pyplot as plt +import pypsa +import seaborn as sns +from _helpers import configure_logging + +sns.set_theme("paper", style="whitegrid") + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake( + "plot_elec_statistics", + simpl="", + opts="Ept-12h", + clusters="37", + ll="v1.0", + ) + configure_logging(snakemake) + + n = pypsa.Network(snakemake.input.network) + + n.loads.carrier = "load" + n.carriers.loc["load", ["nice_name", "color"]] = "Load", "darkred" + colors = n.carriers.set_index("nice_name").color.where( + lambda s: s != "", "lightgrey" + ) + + # %% + + def rename_index(ds): + specific = ds.index.map(lambda x: f"{x[1]}\n({x[0]})") + generic = ds.index.get_level_values("carrier") + duplicated = generic.duplicated(keep=False) + index = specific.where(duplicated, generic) + return ds.set_axis(index) + + def plot_static_per_carrier(ds, ax, drop_zero=True): + if drop_zero: + ds = ds[ds != 0] + ds = ds.dropna() + c = colors[ds.index.get_level_values("carrier")] + ds = ds.pipe(rename_index) + label = f"{ds.attrs['name']} [{ds.attrs['unit']}]" + ds.plot.barh(color=c.values, xlabel=label, ax=ax) + ax.grid(axis="y") + + fig, ax = plt.subplots() + ds = n.statistics.capacity_factor().dropna() + plot_static_per_carrier(ds, ax) + fig.savefig(snakemake.output.capacity_factor_bar) + + fig, ax = plt.subplots() + ds = n.statistics.installed_capacity().dropna() + ds = ds.drop("Line") + ds = ds.drop(("Generator", "Load")) + ds = ds / 1e3 + ds.attrs["unit"] = "GW" + plot_static_per_carrier(ds, ax) + fig.savefig(snakemake.output.installed_capacity_bar) + + fig, ax = plt.subplots() + ds = n.statistics.optimal_capacity() + ds = ds.drop("Line") + ds = ds.drop(("Generator", "Load")) + ds = ds / 1e3 + ds.attrs["unit"] = "GW" + plot_static_per_carrier(ds, ax) + fig.savefig(snakemake.output.optimal_capacity_bar) + + fig, ax = plt.subplots() + ds = n.statistics.capex() + plot_static_per_carrier(ds, ax) + fig.savefig(snakemake.output.capital_expenditure_bar) + + fig, ax = plt.subplots() + ds = n.statistics.opex() + plot_static_per_carrier(ds, ax) + fig.savefig(snakemake.output.operational_expenditure_bar) + + fig, ax = plt.subplots() + ds = n.statistics.curtailment() + plot_static_per_carrier(ds, ax) + fig.savefig(snakemake.output.curtailment_bar) + + fig, ax = plt.subplots() + ds = n.statistics.supply() + ds = ds.drop("Line") + ds = ds / 1e6 + ds.attrs["unit"] = "TWh" + plot_static_per_carrier(ds, ax) + fig.savefig(snakemake.output.supply_bar) + + fig, ax = plt.subplots() + ds = n.statistics.withdrawal() + ds = ds.drop("Line") + ds = ds / -1e6 + ds.attrs["unit"] = "TWh" + plot_static_per_carrier(ds, ax) + fig.savefig(snakemake.output.withdrawal_bar) + + fig, ax = plt.subplots() + ds = n.statistics.market_value() + plot_static_per_carrier(ds, ax) + fig.savefig(snakemake.output.market_value_bar) + + # touch file + with open(snakemake.output.barplots_touch, "a"): + pass diff --git a/scripts/plot_validation_cross_border_flows.py b/scripts/plot_validation_cross_border_flows.py new file mode 100644 index 00000000..43ed45e9 --- /dev/null +++ b/scripts/plot_validation_cross_border_flows.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +import country_converter as coco +import matplotlib.pyplot as plt +import pandas as pd +import pypsa +import seaborn as sns +from _helpers import configure_logging + +sns.set_theme("paper", style="whitegrid") + +cc = coco.CountryConverter() + +color_country = { + "AL": "#440154", + "AT": "#482677", + "BA": "#43398e", + "BE": "#3953a4", + "BG": "#2c728e", + "CH": "#228b8d", + "CZ": "#1f9d8a", + "DE": "#29af7f", + "DK": "#3fbc73", + "EE": "#5ec962", + "ES": "#84d44b", + "FI": "#addc30", + "FR": "#d8e219", + "GB": "#fde725", + "GR": "#f0f921", + "HR": "#f1c25e", + "HU": "#f4a784", + "IE": "#f78f98", + "IT": "#f87ea0", + "LT": "#f87a9a", + "LU": "#f57694", + "LV": "#f3758d", + "ME": "#f37685", + "MK": "#f37b7c", + "NL": "#FF6666", + "NO": "#FF3333", + "PL": "#eb0000", + "PT": "#d70000", + "RO": "#c00000", + "RS": "#a50000", + "SE": "#8a0000", + "SI": "#6f0000", + "SK": "#550000", +} + + +def sort_one_country(country, df): + indices = [link for link in df.columns if country in link] + df_country = df[indices].copy() + for link in df_country.columns: + if country in link[5:]: + df_country[link] = -df_country[link] + link_reverse = str(link[5:] + " - " + link[:2]) + df_country = df_country.rename(columns={link: link_reverse}) + + return df_country.reindex(sorted(df_country.columns), axis=1) + + +def cross_border_time_series(countries, data): + fig, ax = plt.subplots(2 * len(countries), 1, figsize=(15, 10 * len(countries))) + axis = 0 + + for country in countries: + ymin = 0 + ymax = 0 + for df in data: + df_country = sort_one_country(country, df) + df_neg, df_pos = df_country.clip(upper=0), df_country.clip(lower=0) + + color = [color_country[link[5:]] for link in df_country.columns] + + df_pos.plot.area( + ax=ax[axis], stacked=True, linewidth=0.0, color=color, ylim=[-1, 1] + ) + + df_neg.plot.area( + ax=ax[axis], stacked=True, linewidth=0.0, color=color, ylim=[-1, 1] + ) + if (axis % 2) == 0: + title = "Historic" + else: + title = "Optimized" + + ax[axis].set_title( + title + " Import / Export for " + cc.convert(country, to="name_short") + ) + + # Custom legend elements + legend_elements = [] + + for link in df_country.columns: + legend_elements = legend_elements + [ + plt.fill_between( + [], + [], + color=color_country[link[5:]], + label=cc.convert(link[5:], to="name_short"), + ) + ] + + # Create the legend + ax[axis].legend(handles=legend_elements, loc="upper right") + + # rescale the y axis + neg_min = df_neg.sum(axis=1).min() * 1.2 + if neg_min < ymin: + ymin = neg_min + + pos_max = df_pos.sum(axis=1).max() * 1.2 + if pos_max < ymax: + ymax = pos_max + + axis = axis + 1 + + for x in range(axis - 2, axis): + ax[x].set_ylim([neg_min, pos_max]) + + fig.savefig(snakemake.output.trade_time_series, bbox_inches="tight") + + +def cross_border_bar(countries, data): + df_positive = pd.DataFrame() + df_negative = pd.DataFrame() + color = [] + + for country in countries: + order = 0 + for df in data: + df_country = sort_one_country(country, df) + df_neg, df_pos = df_country.clip(upper=0), df_country.clip(lower=0) + + if (order % 2) == 0: + title = "Historic" + else: + title = "Optimized" + + df_positive_new = pd.DataFrame(data=df_pos.sum()).T.rename( + {0: title + " " + cc.convert(country, to="name_short")} + ) + df_negative_new = pd.DataFrame(data=df_neg.sum()).T.rename( + {0: title + " " + cc.convert(country, to="name_short")} + ) + + df_positive = pd.concat([df_positive_new, df_positive]) + df_negative = pd.concat([df_negative_new, df_negative]) + + order = order + 1 + + color = [color_country[link[5:]] for link in df_positive.columns] + + fig, ax = plt.subplots(figsize=(15, 60)) + + df_positive.plot.barh(ax=ax, stacked=True, color=color, zorder=2) + df_negative.plot.barh(ax=ax, stacked=True, color=color, zorder=2) + + plt.grid(axis="x", zorder=0) + plt.grid(axis="y", zorder=0) + + # Custom legend elements + legend_elements = [] + + for country in list(color_country.keys()): + legend_elements = legend_elements + [ + plt.fill_between( + [], + [], + color=color_country[country], + label=cc.convert(country, to="name_short"), + ) + ] + + # Create the legend + plt.legend(handles=legend_elements, loc="upper right") + + fig.savefig(snakemake.output.cross_border_bar, bbox_inches="tight") + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake( + "plot_electricity_prices", + simpl="", + opts="Ept-12h", + clusters="37", + ll="v1.0", + ) + configure_logging(snakemake) + + countries = snakemake.params.countries + + n = pypsa.Network(snakemake.input.network) + n.loads.carrier = "load" + + historic = pd.read_csv( + snakemake.input.cross_border_flows, + index_col=0, + header=0, + parse_dates=True, + ) + + if len(historic.index) > len(n.snapshots): + historic = historic.resample(n.snapshots.inferred_freq).mean().loc[n.snapshots] + + # Preparing network data to be shaped similar to ENTSOE datastructure + optimized_links = n.links_t.p0.rename( + columns=dict(n.links.bus0.str[:2] + " - " + n.links.bus1.str[:2]) + ) + optimized_lines = n.lines_t.p0.rename( + columns=dict(n.lines.bus0.str[:2] + " - " + n.lines.bus1.str[:2]) + ) + optimized = pd.concat([optimized_links, optimized_lines], axis=1) + + # Drop internal country connection + optimized.drop( + [c for c in optimized.columns if c[:2] == c[5:]], axis=1, inplace=True + ) + + # align columns name + for c1 in optimized.columns: + for c2 in optimized.columns: + if c1[:2] == c2[5:] and c2[:2] == c1[5:]: + optimized = optimized.rename(columns={c1: c2}) + + optimized = optimized.groupby(lambda x: x, axis=1).sum() + + cross_border_bar(countries, [historic, optimized]) + + cross_border_time_series(countries, [historic, optimized]) + + # touch file + with open(snakemake.output.plots_touch, "a"): + pass diff --git a/scripts/plot_validation_electricity_prices.py b/scripts/plot_validation_electricity_prices.py new file mode 100644 index 00000000..2a187b9f --- /dev/null +++ b/scripts/plot_validation_electricity_prices.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +import matplotlib.pyplot as plt +import pandas as pd +import pypsa +import seaborn as sns +from _helpers import configure_logging +from pypsa.statistics import get_bus_and_carrier + +sns.set_theme("paper", style="whitegrid") + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake( + "plot_electricity_prices", + simpl="", + opts="Ept-12h", + clusters="37", + ll="v1.0", + ) + configure_logging(snakemake) + + n = pypsa.Network(snakemake.input.network) + n.loads.carrier = "load" + + historic = pd.read_csv( + snakemake.input.electricity_prices, + index_col=0, + header=0, + parse_dates=True, + ) + + if len(historic.index) > len(n.snapshots): + historic = historic.resample(n.snapshots.inferred_freq).mean().loc[n.snapshots] + + optimized = n.buses_t.marginal_price.groupby(n.buses.country, axis=1).mean() + + data = pd.concat([historic, optimized], keys=["Historic", "Optimized"], axis=1) + data.columns.names = ["Kind", "Country"] + + fig, ax = plt.subplots(figsize=(6, 6)) + + df = data.mean().unstack().T + df.plot.barh(ax=ax, xlabel="Electricity Price [€/MWh]", ylabel="") + ax.grid(axis="y") + fig.savefig(snakemake.output.price_bar, bbox_inches="tight") + + fig, ax = plt.subplots() + + df = data.groupby(level="Kind", axis=1).mean() + df.plot(ax=ax, xlabel="", ylabel="Electricity Price [€/MWh]", alpha=0.8) + ax.grid(axis="x") + fig.savefig(snakemake.output.price_line, bbox_inches="tight") + + # touch file + with open(snakemake.output.plots_touch, "a"): + pass diff --git a/scripts/plot_validation_electricity_production.py b/scripts/plot_validation_electricity_production.py new file mode 100644 index 00000000..5c5569d0 --- /dev/null +++ b/scripts/plot_validation_electricity_production.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +import matplotlib.pyplot as plt +import pandas as pd +import pypsa +import seaborn as sns +from _helpers import configure_logging +from pypsa.statistics import get_bus_and_carrier + +sns.set_theme("paper", style="whitegrid") + +carrier_groups = { + "Offshore Wind (AC)": "Offshore Wind", + "Offshore Wind (DC)": "Offshore Wind", + "Open-Cycle Gas": "Gas", + "Combined-Cycle Gas": "Gas", + "Reservoir & Dam": "Hydro", + "Pumped Hydro Storage": "Hydro", +} + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake( + "plot_validation_electricity_production", + simpl="", + opts="Ept", + clusters="37c", + ll="v1.0", + ) + configure_logging(snakemake) + + n = pypsa.Network(snakemake.input.network) + n.loads.carrier = "load" + + historic = pd.read_csv( + snakemake.input.electricity_production, + index_col=0, + header=[0, 1], + parse_dates=True, + ) + + colors = n.carriers.set_index("nice_name").color.where( + lambda s: s != "", "lightgrey" + ) + colors["Offshore Wind"] = colors["Offshore Wind (AC)"] + colors["Gas"] = colors["Combined-Cycle Gas"] + colors["Hydro"] = colors["Reservoir & Dam"] + colors["Other"] = "lightgray" + + if len(historic.index) > len(n.snapshots): + historic = historic.resample(n.snapshots.inferred_freq).mean().loc[n.snapshots] + + optimized = n.statistics.dispatch( + groupby=get_bus_and_carrier, aggregate_time=False + ).T + optimized = optimized[["Generator", "StorageUnit"]].droplevel(0, axis=1) + optimized = optimized.rename(columns=n.buses.country, level=0) + optimized = optimized.rename(columns=carrier_groups, level=1) + optimized = optimized.groupby(axis=1, level=[0, 1]).sum() + + data = pd.concat([historic, optimized], keys=["Historic", "Optimized"], axis=1) + data.columns.names = ["Kind", "Country", "Carrier"] + data = data.mul(n.snapshot_weightings.generators, axis=0) + + # total production per carrier + fig, ax = plt.subplots(figsize=(6, 6)) + + df = data.groupby(level=["Kind", "Carrier"], axis=1).sum().sum().unstack().T + df = df / 1e6 # TWh + df.plot.barh(ax=ax, xlabel="Electricity Production [TWh]", ylabel="") + ax.grid(axis="y") + fig.savefig(snakemake.output.production_bar, bbox_inches="tight") + + # highest diffs + + fig, ax = plt.subplots(figsize=(6, 10)) + + df = data.sum() / 1e6 # TWh + df = df["Optimized"] - df["Historic"] + df = df.dropna().sort_values() + df = pd.concat([df.iloc[:5], df.iloc[-5:]]) + c = colors[df.index.get_level_values(1)] + df.plot.barh( + xlabel="Optimized Production - Historic Production [TWh]", ax=ax, color=c.values + ) + ax.set_title("Strongest Deviations") + ax.grid(axis="y") + fig.savefig(snakemake.output.production_deviation_bar, bbox_inches="tight") + + # seasonal operation + + fig, axes = plt.subplots(3, 1, figsize=(9, 9)) + + df = ( + data.groupby(level=["Kind", "Carrier"], axis=1) + .sum() + .resample("1W") + .mean() + .clip(lower=0) + ) + df = df / 1e3 + + order = ( + (df["Historic"].diff().abs().sum() / df["Historic"].sum()).sort_values().index + ) + c = colors[order] + optimized = df["Optimized"].reindex(order, axis=1, level=1) + historical = df["Historic"].reindex(order, axis=1, level=1) + + kwargs = dict(color=c, legend=False, ylabel="Production [GW]", xlabel="") + + optimized.plot.area(ax=axes[0], **kwargs, title="Optimized") + historical.plot.area(ax=axes[1], **kwargs, title="Historic") + + diff = optimized - historical + diff.clip(lower=0).plot.area( + ax=axes[2], **kwargs, title="$\Delta$ (Optimized - Historic)" + ) + lim = axes[2].get_ylim()[1] + diff.clip(upper=0).plot.area(ax=axes[2], **kwargs) + axes[2].set_ylim(bottom=-lim, top=lim) + + h, l = axes[0].get_legend_handles_labels() + fig.legend( + h[::-1], + l[::-1], + loc="center left", + bbox_to_anchor=(1, 0.5), + ncol=1, + frameon=False, + labelspacing=1, + ) + fig.savefig(snakemake.output.seasonal_operation_area, bbox_inches="tight") + + # touch file + with open(snakemake.output.plots_touch, "a"): + pass diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 7b7f77f9..a5a00a3c 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -65,6 +65,7 @@ import pandas as pd import pypsa from _helpers import configure_logging from add_electricity import load_costs, update_transmission_costs +from pypsa.descriptors import expand_series idx = pd.IndexSlice @@ -103,10 +104,30 @@ def add_emission_prices(n, emission_prices={"co2": 0.0}, exclude_co2=False): ).sum(axis=1) gen_ep = n.generators.carrier.map(ep) / n.generators.efficiency n.generators["marginal_cost"] += gen_ep + n.generators_t["marginal_cost"] += gen_ep[n.generators_t["marginal_cost"].columns] su_ep = n.storage_units.carrier.map(ep) / n.storage_units.efficiency_dispatch n.storage_units["marginal_cost"] += su_ep +def add_dynamic_emission_prices(n): + co2_price = pd.read_csv(snakemake.input.co2_price, index_col=0, parse_dates=True) + co2_price = co2_price[~co2_price.index.duplicated()] + co2_price = ( + co2_price.reindex(n.snapshots).fillna(method="ffill").fillna(method="bfill") + ) + + emissions = ( + n.generators.carrier.map(n.carriers.co2_emissions) / n.generators.efficiency + ) + co2_cost = expand_series(emissions, n.snapshots).T.mul(co2_price.iloc[:, 0], axis=0) + + static = n.generators.marginal_cost + dynamic = n.get_switchable_as_dense("Generator", "marginal_cost") + + marginal_cost = dynamic + co2_cost.reindex(columns=dynamic.columns, fill_value=0) + n.generators_t.marginal_cost = marginal_cost.loc[:, marginal_cost.ne(static).any()] + + def set_line_s_max_pu(n, s_max_pu=0.7): n.lines["s_max_pu"] = s_max_pu logger.info(f"N-1 security margin of lines set to {s_max_pu}") @@ -253,12 +274,13 @@ def set_line_nom_max( n.links.p_nom_max.clip(upper=p_nom_max_set, inplace=True) +# %% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake snakemake = mock_snakemake( - "prepare_network", simpl="", clusters="40", ll="v0.3", opts="Co2L-24H" + "prepare_network", simpl="", clusters="37", ll="v1.0", opts="Ept" ) configure_logging(snakemake) @@ -332,7 +354,12 @@ if __name__ == "__main__": c.df.loc[sel, attr] *= factor for o in opts: - if "Ep" in o: + if "Ept" in o: + logger.info( + "Setting time dependent emission prices according spot market price" + ) + add_dynamic_emission_prices(n) + elif "Ep" in o: m = re.findall("[0-9]*\.?[0-9]+$", o) if len(m) > 0: logger.info("Setting emission prices according to wildcard value.") diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 12cfa55d..5ff10109 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -17,11 +17,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import ( - generate_periodic_profiles, - override_component_attrs, - update_config_with_sector_opts, -) +from _helpers import generate_periodic_profiles, update_config_with_sector_opts from add_electricity import calculate_annuity, sanitize_carriers from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2 from networkx.algorithms import complement @@ -2163,12 +2159,11 @@ def add_biomass(n, costs): ) if options["biomass_transport"]: - transport_costs = pd.read_csv( - snakemake.input.biomass_transport_costs, - index_col=0, - ).squeeze() - # add biomass transport + transport_costs = pd.read_csv( + snakemake.input.biomass_transport_costs, index_col=0 + ) + transport_costs = transport_costs.squeeze() biomass_transport = create_network_topology( n, "biomass transport ", bidirectional=False ) @@ -2192,6 +2187,27 @@ def add_biomass(n, costs): carrier="solid biomass transport", ) + elif options["biomass_spatial"]: + # add artificial biomass generators at nodes which include transport costs + transport_costs = pd.read_csv( + snakemake.input.biomass_transport_costs, index_col=0 + ) + transport_costs = transport_costs.squeeze() + bus_transport_costs = spatial.biomass.nodes.to_series().apply( + lambda x: transport_costs[x[:2]] + ) + average_distance = 200 # km #TODO: validate this assumption + + n.madd( + "Generator", + spatial.biomass.nodes, + bus=spatial.biomass.nodes, + carrier="solid biomass", + p_nom=10000, + marginal_cost=costs.at["solid biomass", "fuel"] + + bus_transport_costs * average_distance, + ) + # AC buses with district heating urban_central = n.buses.index[n.buses.carrier == "urban central heat"] if not urban_central.empty and options["chp"]: @@ -3283,8 +3299,7 @@ if __name__ == "__main__": investment_year = int(snakemake.wildcards.planning_horizons[-4:]) - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) + n = pypsa.Network(snakemake.input.network) pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) nhours = n.snapshot_weightings.generators.sum() diff --git a/scripts/retrieve_monthly_fuel_prices.py b/scripts/retrieve_monthly_fuel_prices.py new file mode 100644 index 00000000..11e351ce --- /dev/null +++ b/scripts/retrieve_monthly_fuel_prices.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +""" +Retrieve monthly fuel prices from Destatis. +""" + +import logging + +logger = logging.getLogger(__name__) + +from pathlib import Path + +from _helpers import configure_logging, progress_retrieve + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("retrieve_monthly_fuel_prices") + rootpath = ".." + else: + rootpath = "." + configure_logging(snakemake) + + url = "https://www.destatis.de/EN/Themes/Economy/Prices/Publications/Downloads-Energy-Price-Trends/energy-price-trends-xlsx-5619002.xlsx?__blob=publicationFile" + + to_fn = Path(rootpath) / Path(snakemake.output[0]) + + logger.info(f"Downloading monthly fuel prices from '{url}'.") + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + progress_retrieve(url, to_fn, disable=disable_progress) + + logger.info(f"Monthly fuel prices available at {to_fn}") diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 9fce8e06..cac25647 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -613,6 +613,7 @@ if __name__ == "__main__": "substation_lv", "substation_off", "geometry", + "underground", ] n.buses.drop(remove, axis=1, inplace=True, errors="ignore") n.lines.drop(remove, axis=1, errors="ignore", inplace=True) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 97c1923c..8c344233 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -33,11 +33,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import ( - configure_logging, - override_component_attrs, - update_config_with_sector_opts, -) +from _helpers import configure_logging, update_config_with_sector_opts logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) @@ -152,7 +148,7 @@ def prepare_network( if "clip_p_max_pu" in solve_opts: for df in ( n.generators_t.p_max_pu, - n.generators_t.p_min_pu, # TODO: check if this can be removed + n.generators_t.p_min_pu, n.storage_units_t.inflow, ): df.where(df > solve_opts["clip_p_max_pu"], other=0.0, inplace=True) @@ -280,13 +276,13 @@ def add_EQ_constraints(n, o, scaling=1e-1): float_regex = "[0-9]*\.?[0-9]+" level = float(re.findall(float_regex, o)[0]) if o[-1] == "c": - ggrouper = n.generators.bus.map(n.buses.country).to_xarray() - lgrouper = n.loads.bus.map(n.buses.country).to_xarray() - sgrouper = n.storage_units.bus.map(n.buses.country).to_xarray() + ggrouper = n.generators.bus.map(n.buses.country) + lgrouper = n.loads.bus.map(n.buses.country) + sgrouper = n.storage_units.bus.map(n.buses.country) else: - ggrouper = n.generators.bus.to_xarray() - lgrouper = n.loads.bus.to_xarray() - sgrouper = n.storage_units.bus.to_xarray() + ggrouper = n.generators.bus + lgrouper = n.loads.bus + sgrouper = n.storage_units.bus load = ( n.snapshot_weightings.generators @ n.loads_t.p_set.groupby(lgrouper, axis=1).sum() @@ -300,7 +296,7 @@ def add_EQ_constraints(n, o, scaling=1e-1): p = n.model["Generator-p"] lhs_gen = ( (p * (n.snapshot_weightings.generators * scaling)) - .groupby(ggrouper) + .groupby(ggrouper.to_xarray()) .sum() .sum("snapshot") ) @@ -309,7 +305,7 @@ def add_EQ_constraints(n, o, scaling=1e-1): spillage = n.model["StorageUnit-spill"] lhs_spill = ( (spillage * (-n.snapshot_weightings.stores * scaling)) - .groupby(sgrouper) + .groupby(sgrouper.to_xarray()) .sum() .sum("snapshot") ) @@ -599,47 +595,46 @@ def extra_functionality(n, snapshots): def solve_network(n, config, solving, opts="", **kwargs): set_of_options = solving["solver"]["options"] - solver_options = solving["solver_options"][set_of_options] if set_of_options else {} - solver_name = solving["solver"]["name"] cf_solving = solving["options"] - track_iterations = cf_solving.get("track_iterations", False) - min_iterations = cf_solving.get("min_iterations", 4) - max_iterations = cf_solving.get("max_iterations", 6) - transmission_losses = cf_solving.get("transmission_losses", 0) - multi_horizon = True if config["foresight"] == "perfect" else False + + kwargs["multi_investment_periods"] = True if config["foresight"] == "perfect" else False + kwargs["solver_options"] = ( + solving["solver_options"][set_of_options] if set_of_options else {} + ) + kwargs["solver_name"] = solving["solver"]["name"] + kwargs["extra_functionality"] = extra_functionality + kwargs["transmission_losses"] = cf_solving.get("transmission_losses", False) + kwargs["linearized_unit_commitment"] = cf_solving.get( + "linearized_unit_commitment", False + ) + kwargs["assign_all_duals"] = cf_solving.get("assign_all_duals", False) + + rolling_horizon = cf_solving.pop("rolling_horizon", False) + skip_iterations = cf_solving.pop("skip_iterations", False) + if not n.lines.s_nom_extendable.any(): + skip_iterations = True + logger.info("No expandable lines found. Skipping iterative solving.") # add to network for extra_functionality n.config = config n.opts = opts - skip_iterations = cf_solving.get("skip_iterations", False) - if not n.lines.s_nom_extendable.any(): - skip_iterations = True - logger.info("No expandable lines found. Skipping iterative solving.") - - if skip_iterations: - status, condition = n.optimize( - solver_name=solver_name, - transmission_losses=transmission_losses, - extra_functionality=extra_functionality, - multi_investment_periods=multi_horizon, - **solver_options, - **kwargs, - ) + if rolling_horizon: + kwargs["horizon"] = cf_solving.get("horizon", 365) + kwargs["overlap"] = cf_solving.get("overlap", 0) + n.optimize.optimize_with_rolling_horizon(**kwargs) + status, condition = "", "" + elif skip_iterations: + status, condition = n.optimize(**kwargs) else: + kwargs["track_iterations"] = (cf_solving.get("track_iterations", False),) + kwargs["min_iterations"] = (cf_solving.get("min_iterations", 4),) + kwargs["max_iterations"] = (cf_solving.get("max_iterations", 6),) status, condition = n.optimize.optimize_transmission_expansion_iteratively( - solver_name=solver_name, - track_iterations=track_iterations, - min_iterations=min_iterations, - max_iterations=max_iterations, - transmission_losses=transmission_losses, - extra_functionality=extra_functionality, - multi_investment_periods=multi_horizon, - **solver_options, - **kwargs, + **kwargs ) - if status != "ok": + if status != "ok" and not rolling_horizon: logger.warning( f"Solving status '{status}' with termination condition '{condition}'" ) @@ -677,11 +672,7 @@ if __name__ == "__main__": np.random.seed(solve_opts.get("seed", 123)) - if "overrides" in snakemake.input.keys(): - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) - else: - n = pypsa.Network(snakemake.input.network) + n = pypsa.Network(snakemake.input.network) n = prepare_network( n, diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 37e853e5..1a3855a9 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -11,11 +11,7 @@ import logging import numpy as np import pypsa -from _helpers import ( - configure_logging, - override_component_attrs, - update_config_with_sector_opts, -) +from _helpers import configure_logging, update_config_with_sector_opts from solve_network import prepare_network, solve_network logger = logging.getLogger(__name__) @@ -45,11 +41,7 @@ if __name__ == "__main__": np.random.seed(solve_opts.get("seed", 123)) - if "overrides" in snakemake.input: - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) - else: - n = pypsa.Network(snakemake.input.network) + n = pypsa.Network(snakemake.input.network) n.optimize.fix_optimal_capacities() n = prepare_network(n, solve_opts, config=snakemake.config)