From f2e53b59bf92fc477233c7919dd61a2f94a95ae0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 22 Apr 2023 09:44:13 +0200 Subject: [PATCH 01/78] add plain hydrogen turbine option for re-electrification --- config/config.default.yaml | 3 +++ doc/release_notes.rst | 4 +++ scripts/prepare_sector_network.py | 45 ++++++++++++++++++++++--------- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 1305456a..10e6a6ed 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -469,6 +469,8 @@ sector: dac: true co2_vent: false allam_cycle: false + hydrogen_fuel_cell: true + hydrogen_turbine: false SMR: true regional_co2_sequestration_potential: enable: false # enable regionally resolved geological co2 storage potential @@ -902,6 +904,7 @@ plotting: H2 pipeline: '#f081dc' H2 pipeline retrofitted: '#ba99b5' H2 Fuel Cell: '#c251ae' + H2 turbine: '#991f83' H2 Electrolysis: '#ff29d9' # ammonia NH3: '#46caf0' diff --git a/doc/release_notes.rst b/doc/release_notes.rst index f859646b..d81bee07 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -17,6 +17,10 @@ Upcoming Release * Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand``. +* Add plain hydrogen turbine as additional re-electrification option besides + hydrogen fuel cell. Add switches for both re-electrification options under + ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``. + PyPSA-Eur 0.8.0 (18th March 2023) ================================= diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 6129d3b5..e737867e 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1067,18 +1067,39 @@ def add_storage_and_grids(n, costs): lifetime=costs.at["electrolysis", "lifetime"], ) - n.madd( - "Link", - nodes + " H2 Fuel Cell", - bus0=nodes + " H2", - bus1=nodes, - p_nom_extendable=True, - carrier="H2 Fuel Cell", - efficiency=costs.at["fuel cell", "efficiency"], - capital_cost=costs.at["fuel cell", "fixed"] - * costs.at["fuel cell", "efficiency"], # NB: fixed cost is per MWel - lifetime=costs.at["fuel cell", "lifetime"], - ) + if options["hydrogen_fuel_cell"]: + + logger.info("Adding hydrogen fuel cell for re-electrification.") + + n.madd( + "Link", + nodes + " H2 Fuel Cell", + bus0=nodes + " H2", + bus1=nodes, + p_nom_extendable=True, + carrier="H2 Fuel Cell", + efficiency=costs.at["fuel cell", "efficiency"], + capital_cost=costs.at["fuel cell", "fixed"] + * costs.at["fuel cell", "efficiency"], # NB: fixed cost is per MWel + lifetime=costs.at["fuel cell", "lifetime"], + ) + + if options["hydrogen_turbine"]: + + logger.info("Adding hydrogen turbine for re-electrification.") + + n.madd( + "Link", + nodes + " H2 turbine", + bus0=nodes + " H2", + bus1=nodes, + p_nom_extendable=True, + carrier="H2 turbine", + efficiency=costs.at["OCGT", "efficiency"], + capital_cost=costs.at["OCGT", "fixed"] + * costs.at["OCGT", "efficiency"], # NB: fixed cost is per MWel + lifetime=costs.at["OCGT", "lifetime"], + ) cavern_types = snakemake.config["sector"]["hydrogen_underground_storage_locations"] h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0) From 0a3c177f4b6083a00f75810340cf7570f373bb15 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 22 Apr 2023 07:44:57 +0000 Subject: [PATCH 02/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/prepare_sector_network.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index e737867e..90891ad5 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1068,7 +1068,6 @@ def add_storage_and_grids(n, costs): ) if options["hydrogen_fuel_cell"]: - logger.info("Adding hydrogen fuel cell for re-electrification.") n.madd( @@ -1085,7 +1084,6 @@ def add_storage_and_grids(n, costs): ) if options["hydrogen_turbine"]: - logger.info("Adding hydrogen turbine for re-electrification.") n.madd( From a2e3e9163094c5ee2c712a82e98ceb972cbcf893 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Wed, 26 Apr 2023 18:05:56 +0200 Subject: [PATCH 03/78] fix bug in load shedding constraint --- scripts/solve_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index fcdc4ed9..ff1c0ccf 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -166,7 +166,7 @@ def prepare_network(n, solve_opts=None, config=None): "Generator", buses_i, " load", - bus=n.buses.index, + bus=buses_i, carrier="load", sign=1e-3, # Adjust sign to measure p and p_nom in kW instead of MW marginal_cost=load_shedding, # Eur/kWh From f3ca907fb0758831c49e3d01b5f8a39b61c8734b Mon Sep 17 00:00:00 2001 From: Laurent Drouet Date: Fri, 28 Apr 2023 15:23:44 +0200 Subject: [PATCH 04/78] Update installation.rst --- doc/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/installation.rst b/doc/installation.rst index b6fd574e..01fdafeb 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -39,7 +39,7 @@ The environment can be installed and activated using .. code:: bash - .../pypsa-eur % mamba create -f envs/environment.yaml + .../pypsa-eur % mamba env create -f envs/environment.yaml .../pypsa-eur % mamba activate pypsa-eur From 51e09f5e61828019cdc350bbd93081dcdbe57c1b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 30 Apr 2023 10:43:49 +0200 Subject: [PATCH 05/78] Address review comments --- scripts/prepare_sector_network.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 90891ad5..0c7a816c 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1084,7 +1084,8 @@ def add_storage_and_grids(n, costs): ) if options["hydrogen_turbine"]: - logger.info("Adding hydrogen turbine for re-electrification.") + logger.info("Adding hydrogen turbine for re-electrification. Assuming OCGT technology costs.") + # TODO: perhaps replace with hydrogen-specific technology assumptions. n.madd( "Link", From bdeab82b494d2b7edd1700b5339f7ce3cbe89fa6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 30 Apr 2023 08:44:05 +0000 Subject: [PATCH 06/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/prepare_sector_network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 0c7a816c..1cb7146e 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1084,7 +1084,9 @@ def add_storage_and_grids(n, costs): ) if options["hydrogen_turbine"]: - logger.info("Adding hydrogen turbine for re-electrification. Assuming OCGT technology costs.") + logger.info( + "Adding hydrogen turbine for re-electrification. Assuming OCGT technology costs." + ) # TODO: perhaps replace with hydrogen-specific technology assumptions. n.madd( From b997adcde7c21bef9e1788e38065310027359ace Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 May 2023 22:14:07 +0000 Subject: [PATCH 07/78] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/docformatter: v1.6.3 → v1.6.4](https://github.com/PyCQA/docformatter/compare/v1.6.3...v1.6.4) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03dffc2a..02471b57 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: # Make docstrings PEP 257 compliant - repo: https://github.com/PyCQA/docformatter - rev: v1.6.3 + rev: v1.6.4 hooks: - id: docformatter args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] From f6a40d36969b24c49b64c912f46f7d33c16cfd99 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 May 2023 22:14:42 +0000 Subject: [PATCH 08/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_retro_cost.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index cd1dae8d..9dbfc375 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -513,7 +513,7 @@ def prepare_cost_retro(country_iso_dic): def prepare_temperature_data(): """ - returns the temperature dependent data for each country: + Returns the temperature dependent data for each country: d_heat : length of heating season pd.Series(index=countries) [days/year] on those days, daily average temperature is below @@ -621,7 +621,7 @@ def calculate_costs(u_values, l, cost_retro, window_assumptions): def calculate_new_u(u_values, l, l_weight, window_assumptions, k=0.035): """ - calculate U-values after building retrofitting, depending on the old + Calculate U-values after building retrofitting, depending on the old U-values (u_values). This is for simple insulation measuers, adding an additional layer of insulation. @@ -682,7 +682,7 @@ def map_tabula_to_hotmaps(df_tabula, df_hotmaps, column_prefix): def get_solar_gains_per_year(window_area): """ - returns solar heat gains during heating season in [kWh/a] depending on the + Returns solar heat gains during heating season in [kWh/a] depending on the window area [m^2] of the building, assuming a equal distributed window orientation (east, south, north, west) """ @@ -718,7 +718,7 @@ def map_to_lstrength(l_strength, df): def calculate_heat_losses(u_values, data_tabula, l_strength, temperature_factor): """ - calculates total annual heat losses Q_ht for different insulation + Calculates total annual heat losses Q_ht for different insulation thicknesses (l_strength), depending on current insulation state (u_values), standard building topologies and air ventilation from TABULA (data_tabula) and the accumulated difference between internal and external temperature @@ -840,7 +840,7 @@ def calculate_heat_losses(u_values, data_tabula, l_strength, temperature_factor) def calculate_heat_gains(data_tabula, heat_transfer_perm2, d_heat): """ - calculates heat gains Q_gain [W/m^2], which consititure from gains by: + Calculates heat gains Q_gain [W/m^2], which consititure from gains by: (1) solar radiation (2) internal heat gains """ @@ -885,7 +885,7 @@ def calculate_space_heat_savings( u_values, data_tabula, l_strength, temperature_factor, d_heat ): """ - calculates space heat savings (dE_space [per unit of unrefurbished state]) + Calculates space heat savings (dE_space [per unit of unrefurbished state]) through retrofitting of the thermal envelope by additional insulation material (l_strength[m]) """ From f355f816c3d2452e6ffe8a6dd25154c5a2f9cc5d Mon Sep 17 00:00:00 2001 From: energyls Date: Tue, 2 May 2023 17:20:06 +0200 Subject: [PATCH 09/78] docs: add stackoverflow issue template --- .github/ISSUE_TEMPLATE/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index d8c04382..b357ab5f 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -3,3 +3,6 @@ contact_links: - name: PyPSA Mailing List url: https://groups.google.com/forum/#!forum/pypsa about: Please ask and answer general usage questions here. +- name: Stackoverflow + url: https://stackoverflow.com/questions/tagged/pypsa + about: Please ask and answer code-related questions here. \ No newline at end of file From 4f890799455b11ccd671ebe0785e32d2c7d34d76 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 May 2023 15:21:48 +0000 Subject: [PATCH 10/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/ISSUE_TEMPLATE/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index b357ab5f..03659213 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,4 +5,4 @@ contact_links: about: Please ask and answer general usage questions here. - name: Stackoverflow url: https://stackoverflow.com/questions/tagged/pypsa - about: Please ask and answer code-related questions here. \ No newline at end of file + about: Please ask and answer code-related questions here. From 2a1cdd1d7820ec242081cbab7a98789b2711772d Mon Sep 17 00:00:00 2001 From: energyls Date: Tue, 2 May 2023 17:39:01 +0200 Subject: [PATCH 11/78] docs: add shield and support section to readme --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 2973fa33..02cc3163 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ SPDX-License-Identifier: CC-BY-4.0 [![Zenodo PyPSA-Eur-Sec](https://zenodo.org/badge/DOI/10.5281/zenodo.3938042.svg)](https://doi.org/10.5281/zenodo.3938042) [![Snakemake](https://img.shields.io/badge/snakemake-≥5.0.0-brightgreen.svg?style=flat)](https://snakemake.readthedocs.io) [![REUSE status](https://api.reuse.software/badge/github.com/pypsa/pypsa-eur)](https://api.reuse.software/info/github.com/pypsa/pypsa-eur) +[![Stack Exchange questions](https://img.shields.io/stackexchange/stackoverflow/t/pypsa)](https://stackoverflow.com/questions/tagged/pypsa) # PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy System @@ -90,6 +91,14 @@ to 50-200 nodes. Already-built versions of the model can be found in the accompanying [Zenodo repository](https://doi.org/10.5281/zenodo.3601881). + +# Contributing and Support +We strongly welcome anyone interested in contributing to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on GitHub. +- In case of code-related **questions**, please post on [stack overflow](https://stackoverflow.com/questions/tagged/pypsa). +- For non-programming related and more general questions please refer to the [mailing list](https://groups.google.com/group/pypsa). +- To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb). +- For **bugs and feature requests**, please use the [PyPSA-Eur Github Issues page](https://github.com/PyPSA/pypsa-eur/issues). + # Licence The code in PyPSA-Eur is released as free software under the From 42a9f40680b958c918f5436887a8c73cd2d7fdc6 Mon Sep 17 00:00:00 2001 From: energyls Date: Tue, 2 May 2023 17:49:26 +0200 Subject: [PATCH 12/78] docs: add shield to rst --- doc/index.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/index.rst b/doc/index.rst index 521f080b..743444dd 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -31,7 +31,9 @@ PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy Syste :target: https://api.reuse.software/info/github.com/pypsa/pypsa-eur :alt: REUSE -| +|.. image:: https://img.shields.io/stackexchange/stackoverflow/t/pypsa + :target: https://stackoverflow.com/questions/tagged/pypsa + :alt: Stackoverflow PyPSA-Eur is an open model dataset of the European energy system at the transmission network level that covers the full ENTSO-E area. It covers demand From a6948ad39e44ecc4caaad8ccb8c3044ab1db7139 Mon Sep 17 00:00:00 2001 From: energyls Date: Wed, 3 May 2023 10:36:54 +0200 Subject: [PATCH 13/78] docs: add support.rst and add to TOC --- doc/index.rst | 1 + doc/support.rst | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 doc/support.rst diff --git a/doc/index.rst b/doc/index.rst index 743444dd..2d53cfb0 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -276,4 +276,5 @@ The included ``.nc`` files are PyPSA network files which can be imported with Py licenses limitations contributing + support publications diff --git a/doc/support.rst b/doc/support.rst new file mode 100644 index 00000000..1cd39e35 --- /dev/null +++ b/doc/support.rst @@ -0,0 +1,10 @@ + +------- +Support +------- + +* In case of code-related **questions**, please post on `stack overflow `_. +* For non-programming related and more general questions please refer to the `mailing list `_. +* To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb). +* For **bugs and feature requests**, please use the `issue tracker `_. +* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to :ref:`contributing`. \ No newline at end of file From 93d79da903c10ff51ebd1db494165ca99aa4c5ae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 08:37:21 +0000 Subject: [PATCH 14/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/support.rst b/doc/support.rst index 1cd39e35..f0a2c83c 100644 --- a/doc/support.rst +++ b/doc/support.rst @@ -7,4 +7,4 @@ Support * For non-programming related and more general questions please refer to the `mailing list `_. * To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb). * For **bugs and feature requests**, please use the `issue tracker `_. -* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to :ref:`contributing`. \ No newline at end of file +* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to :ref:`contributing`. From d5b1a05e379e1ce123383687983ed29a8db167d7 Mon Sep 17 00:00:00 2001 From: energyls Date: Wed, 3 May 2023 10:40:58 +0200 Subject: [PATCH 15/78] docs: add licence information to support.rst --- doc/support.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/support.rst b/doc/support.rst index f0a2c83c..36d1a2dd 100644 --- a/doc/support.rst +++ b/doc/support.rst @@ -1,7 +1,11 @@ +.. + SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors -------- + SPDX-License-Identifier: CC-BY-4.0 + +####################### Support -------- +####################### * In case of code-related **questions**, please post on `stack overflow `_. * For non-programming related and more general questions please refer to the `mailing list `_. From e9787e77e4ce9bad1206de2495e5e5d2023fe7b7 Mon Sep 17 00:00:00 2001 From: energyls Date: Wed, 3 May 2023 10:53:47 +0200 Subject: [PATCH 16/78] docs: remove hyperref to check CI --- doc/support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/support.rst b/doc/support.rst index 36d1a2dd..395c5f31 100644 --- a/doc/support.rst +++ b/doc/support.rst @@ -11,4 +11,4 @@ Support * For non-programming related and more general questions please refer to the `mailing list `_. * To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb). * For **bugs and feature requests**, please use the `issue tracker `_. -* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to :ref:`contributing`. +* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to ... \ No newline at end of file From 817e82a80529dd34254643b9786a806eb2b6769c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 08:54:47 +0000 Subject: [PATCH 17/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/support.rst b/doc/support.rst index 395c5f31..c0b4981b 100644 --- a/doc/support.rst +++ b/doc/support.rst @@ -11,4 +11,4 @@ Support * For non-programming related and more general questions please refer to the `mailing list `_. * To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb). * For **bugs and feature requests**, please use the `issue tracker `_. -* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to ... \ No newline at end of file +* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to ... From 673aab2925931ecc030a9d55b42c233774d45d86 Mon Sep 17 00:00:00 2001 From: energyls Date: Wed, 3 May 2023 10:58:46 +0200 Subject: [PATCH 18/78] fix: fix typo in index.rst --- doc/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.rst b/doc/index.rst index 2d53cfb0..314900f7 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -31,7 +31,7 @@ PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy Syste :target: https://api.reuse.software/info/github.com/pypsa/pypsa-eur :alt: REUSE -|.. image:: https://img.shields.io/stackexchange/stackoverflow/t/pypsa +.. image:: https://img.shields.io/stackexchange/stackoverflow/t/pypsa :target: https://stackoverflow.com/questions/tagged/pypsa :alt: Stackoverflow From da022db624b3d4fb1290ee4121ffee04c1a2a6dc Mon Sep 17 00:00:00 2001 From: energyls Date: Wed, 3 May 2023 11:03:54 +0200 Subject: [PATCH 19/78] docs: add reference to contributing --- doc/support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/support.rst b/doc/support.rst index c0b4981b..f57e2289 100644 --- a/doc/support.rst +++ b/doc/support.rst @@ -11,4 +11,4 @@ Support * For non-programming related and more general questions please refer to the `mailing list `_. * To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb). * For **bugs and feature requests**, please use the `issue tracker `_. -* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to ... +* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to :ref:`contributing`. \ No newline at end of file From f0453c42c0266468bc3cc0dfe1bf527f778dff2d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 09:05:26 +0000 Subject: [PATCH 20/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/support.rst b/doc/support.rst index f57e2289..36d1a2dd 100644 --- a/doc/support.rst +++ b/doc/support.rst @@ -11,4 +11,4 @@ Support * For non-programming related and more general questions please refer to the `mailing list `_. * To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb). * For **bugs and feature requests**, please use the `issue tracker `_. -* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to :ref:`contributing`. \ No newline at end of file +* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to :ref:`contributing`. From e9c5af27a31e2bded864035c7f8bfbaba48e0311 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 14:10:18 +0100 Subject: [PATCH 21/78] bump python version and add setuptools, myst-parser --- .readthedocs.yml | 6 +++++- doc/requirements.txt | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 3d7a86b2..c74e12a2 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,8 +4,12 @@ version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.11" + python: - version: 3.8 install: - requirements: doc/requirements.txt system_packages: true diff --git a/doc/requirements.txt b/doc/requirements.txt index d5c71da9..2f08b8d9 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -2,9 +2,11 @@ # # SPDX-License-Identifier: CC0-1.0 +setuptools sphinx sphinx_book_theme sphinxcontrib-bibtex +myst-parser # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820 pypsa vresutils>=0.3.1 From 510e544a4fd9350671eead4df41ae9d39abc9758 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 14:19:37 +0100 Subject: [PATCH 22/78] add release note --- doc/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 9be10a4a..8e6bfdd4 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -17,7 +17,7 @@ Upcoming Release * Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand`` and ``retrieve_load_data`` to ``retrieve_electricity_demand``. - +* Fix docs readthedocs built * Add plain hydrogen turbine as additional re-electrification option besides hydrogen fuel cell. Add switches for both re-electrification options under From 8a42ff77ad9e37f55eba2252e0e942902f5be08c Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 14:24:58 +0100 Subject: [PATCH 23/78] add myst_parser to conf.py --- doc/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/conf.py b/doc/conf.py index 4b3978af..8111c86c 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -36,6 +36,7 @@ sys.path.insert(0, os.path.abspath("../scripts")) extensions = [ #'sphinx.ext.autodoc', #'sphinx.ext.autosummary', + "myst_parser", "sphinx.ext.autosectionlabel", "sphinx.ext.intersphinx", "sphinx.ext.todo", From e62d947fb850686440bb2f55df74d994a854e35d Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 14:29:27 +0100 Subject: [PATCH 24/78] set system_packages false --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index c74e12a2..35eff5de 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,4 +12,4 @@ build: python: install: - requirements: doc/requirements.txt - system_packages: true + system_packages: false From 5bfa6237e8eaacad41ed9004dda86f4b1651afb4 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 15:44:33 +0100 Subject: [PATCH 25/78] add graphviz --- .readthedocs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 35eff5de..b5dc5eba 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,8 +8,11 @@ build: os: ubuntu-22.04 tools: python: "3.11" + apt_packages: + - graphviz python: install: - requirements: doc/requirements.txt system_packages: false + apt_packages: From e18b188e81f64656b45232da8ec15c9954ef2281 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 7 May 2023 14:44:54 +0000 Subject: [PATCH 26/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index b5dc5eba..0ee32009 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: python: "3.11" apt_packages: - - graphviz + - graphviz python: install: From dad04a64a1db152813c11ec02dd30c4b03747e1d Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 15:57:42 +0100 Subject: [PATCH 27/78] Update .readthedocs.yml --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 0ee32009..b5dc5eba 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: python: "3.11" apt_packages: - - graphviz + - graphviz python: install: From c431cf6f44a221dc6e7971d4977a1575c6d16867 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 7 May 2023 14:57:56 +0000 Subject: [PATCH 28/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index b5dc5eba..0ee32009 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: python: "3.11" apt_packages: - - graphviz + - graphviz python: install: From 702bb59a18ade47321440112ead6598bb0e4a0d2 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 16:06:33 +0100 Subject: [PATCH 29/78] fix artefact --- .readthedocs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 0ee32009..900dba1e 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -15,4 +15,3 @@ python: install: - requirements: doc/requirements.txt system_packages: false - apt_packages: From 8f7667f0f90048bafdff1f35d7ded7993eb68c83 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 22:30:27 +0000 Subject: [PATCH 30/78] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/docformatter: v1.6.4 → v1.6.5](https://github.com/PyCQA/docformatter/compare/v1.6.4...v1.6.5) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 02471b57..88685f3b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: # Make docstrings PEP 257 compliant - repo: https://github.com/PyCQA/docformatter - rev: v1.6.4 + rev: v1.6.5 hooks: - id: docformatter args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] From 5c2e3c9fc136b12ea8953fada095d335541d22ca Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 10 May 2023 09:58:25 +0200 Subject: [PATCH 31/78] add Shape2Shapes in add_electricit.py --- scripts/add_electricity.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 1d32bce1..64cf442e 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -94,7 +94,9 @@ import pypsa import xarray as xr from _helpers import configure_logging, update_p_nom_max from powerplantmatching.export import map_country_bus -from vresutils import transfer as vtransfer +from itertools import product +import scipy.sparse as sparse +from shapely.prepared import prep idx = pd.IndexSlice @@ -215,6 +217,20 @@ def load_powerplants(ppl_fn): .replace({"carrier": carrier_dict}) ) +def Shapes2Shapes(orig, dest): + """ + Adopted from vresutils.transfer.Shapes2Shapes() + """ + orig_prepped = list(map(prep, orig)) + transfer = sparse.lil_matrix((len(dest), len(orig)), dtype=float) + + for i,j in product(range(len(dest)), range(len(orig))): + if orig_prepped[j].intersects(dest[i]): + area = orig[j].intersection(dest[i]).area + transfer[i,j] = area/dest[i].area + + return transfer + def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0): substation_lv_i = n.buses.index[n.buses["substation_lv"]] @@ -232,7 +248,7 @@ def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0): return pd.DataFrame({group.index[0]: l}) else: nuts3_cntry = nuts3.loc[nuts3.country == cntry] - transfer = vtransfer.Shapes2Shapes( + transfer = Shapes2Shapes( group, nuts3_cntry.geometry, normed=False ).T.tocsr() gdp_n = pd.Series( From e91068196b9db670ca95d00184e273f7890be61b Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 10 May 2023 10:02:41 +0200 Subject: [PATCH 32/78] replace vresutils annuity function with add_electricity.py calculate_annuity --- scripts/prepare_sector_network.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 1cb7146e..944c3caa 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -28,7 +28,7 @@ from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentati from pypsa.geo import haversine_pts from pypsa.io import import_components_from_dataframe from scipy.stats import beta -from vresutils.costdata import annuity +from add_electricity import calculate_annuity logger = logging.getLogger(__name__) @@ -742,7 +742,7 @@ def prepare_costs(cost_file, config, nyears): costs = costs.fillna(config["fill_values"]) def annuity_factor(v): - return annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100 + return calculate_annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100 costs["fixed"] = [ annuity_factor(v) * v["investment"] * nyears for i, v in costs.iterrows() @@ -851,7 +851,7 @@ def add_wave(n, wave_cost_factor): capacity = pd.Series({"Attenuator": 750, "F2HB": 1000, "MultiPA": 600}) # in EUR/MW - annuity_factor = annuity(25, 0.07) + 0.03 + annuity_factor = calculate_annuity(25, 0.07) + 0.03 costs = ( 1e6 * wave_cost_factor From b21965a98600b4fcd3bbf286ac9b5f9bd9032fa6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 08:09:43 +0000 Subject: [PATCH 33/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_electricity.py | 9 +++++---- scripts/prepare_sector_network.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 64cf442e..83ec4bb9 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -85,17 +85,17 @@ It further adds extendable ``generators`` with **zero** capacity for """ import logging +from itertools import product import geopandas as gpd import numpy as np import pandas as pd import powerplantmatching as pm import pypsa +import scipy.sparse as sparse import xarray as xr from _helpers import configure_logging, update_p_nom_max from powerplantmatching.export import map_country_bus -from itertools import product -import scipy.sparse as sparse from shapely.prepared import prep idx = pd.IndexSlice @@ -217,6 +217,7 @@ def load_powerplants(ppl_fn): .replace({"carrier": carrier_dict}) ) + def Shapes2Shapes(orig, dest): """ Adopted from vresutils.transfer.Shapes2Shapes() @@ -224,10 +225,10 @@ def Shapes2Shapes(orig, dest): orig_prepped = list(map(prep, orig)) transfer = sparse.lil_matrix((len(dest), len(orig)), dtype=float) - for i,j in product(range(len(dest)), range(len(orig))): + for i, j in product(range(len(dest)), range(len(orig))): if orig_prepped[j].intersects(dest[i]): area = orig[j].intersection(dest[i]).area - transfer[i,j] = area/dest[i].area + transfer[i, j] = area / dest[i].area return transfer diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 944c3caa..012c9714 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -22,13 +22,13 @@ from _helpers import ( override_component_attrs, update_config_with_sector_opts, ) +from add_electricity import calculate_annuity from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2 from networkx.algorithms import complement from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation from pypsa.geo import haversine_pts from pypsa.io import import_components_from_dataframe from scipy.stats import beta -from add_electricity import calculate_annuity logger = logging.getLogger(__name__) From 654f46f8683cf0613db90d38ab1127a9db62ffd4 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Thu, 11 May 2023 16:58:35 +0200 Subject: [PATCH 34/78] alpha testing complete vresutils removal --- envs/environment.fixed.yaml | 3 +-- envs/environment.yaml | 3 +-- scripts/add_electricity.py | 6 ++---- scripts/solve_network.py | 31 ++++++++++++----------------- scripts/solve_operations_network.py | 30 ++++++++++++---------------- 5 files changed, 30 insertions(+), 43 deletions(-) diff --git a/envs/environment.fixed.yaml b/envs/environment.fixed.yaml index 7d8fcc45..1ff9313d 100644 --- a/envs/environment.fixed.yaml +++ b/envs/environment.fixed.yaml @@ -226,7 +226,7 @@ dependencies: - nspr=4.35 - nss=3.88 - numexpr=2.8.3 -- numpy=1.23.5 +- numpy=1.24 - openjdk=17.0.3 - openjpeg=2.5.0 - openpyxl=3.1.0 @@ -378,4 +378,3 @@ dependencies: - highspy==1.5.0.dev0 - pybind11==2.10.3 - tsam==2.2.2 - - vresutils==0.3.1 diff --git a/envs/environment.yaml b/envs/environment.yaml index 0a9891a5..f970c9ba 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -25,7 +25,7 @@ dependencies: - pytables - lxml - powerplantmatching>=0.5.5 -- numpy<1.24 +- numpy - pandas>=1.4 - geopandas>=0.11.0 - xarray @@ -55,5 +55,4 @@ dependencies: - rasterio!=1.2.10 - pip: - - vresutils>=0.3.1 - tsam>=1.1.0 diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 83ec4bb9..f910dee4 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -218,7 +218,7 @@ def load_powerplants(ppl_fn): ) -def Shapes2Shapes(orig, dest): +def shapes_to_shapes(orig, dest): """ Adopted from vresutils.transfer.Shapes2Shapes() """ @@ -249,9 +249,7 @@ def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0): return pd.DataFrame({group.index[0]: l}) else: nuts3_cntry = nuts3.loc[nuts3.country == cntry] - transfer = Shapes2Shapes( - group, nuts3_cntry.geometry, normed=False - ).T.tocsr() + transfer = shapes_to_shapes(group, nuts3_cntry.geometry).T.tocsr() gdp_n = pd.Series( transfer.dot(nuts3_cntry["gdp"].fillna(1.0).values), index=group.index ) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ff1c0ccf..53ac8816 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -38,7 +38,6 @@ from _helpers import ( override_component_attrs, update_config_with_sector_opts, ) -from vresutils.benchmark import memory_logger logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) @@ -667,23 +666,19 @@ if __name__ == "__main__": np.random.seed(solve_opts.get("seed", 123)) - fn = getattr(snakemake.log, "memory", None) - with memory_logger(filename=fn, interval=30.0) as mem: - if "overrides" in snakemake.input.keys(): - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network( - snakemake.input.network, override_component_attrs=overrides - ) - else: - n = pypsa.Network(snakemake.input.network) - - n = prepare_network(n, solve_opts, config=snakemake.config) - - n = solve_network( - n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + if "overrides" in snakemake.input.keys(): + overrides = override_component_attrs(snakemake.input.overrides) + n = pypsa.Network( + snakemake.input.network, override_component_attrs=overrides ) + else: + n = pypsa.Network(snakemake.input.network) - n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) - n.export_to_netcdf(snakemake.output[0]) + n = prepare_network(n, solve_opts, config=snakemake.config) - logger.info("Maximum memory usage: {}".format(mem.mem_usage)) + n = solve_network( + n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + ) + + n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) + n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 25fe0753..9c2fe3c1 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -46,23 +46,19 @@ if __name__ == "__main__": np.random.seed(solve_opts.get("seed", 123)) - fn = getattr(snakemake.log, "memory", None) - with memory_logger(filename=fn, interval=30.0) as mem: - if "overrides" in snakemake.input: - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network( - snakemake.input.network, override_component_attrs=overrides - ) - else: - n = pypsa.Network(snakemake.input.network) - - n.optimize.fix_optimal_capacities() - n = prepare_network(n, solve_opts, config=snakemake.config) - n = solve_network( - n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + if "overrides" in snakemake.input: + overrides = override_component_attrs(snakemake.input.overrides) + n = pypsa.Network( + snakemake.input.network, override_component_attrs=overrides ) + else: + n = pypsa.Network(snakemake.input.network) - n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) - n.export_to_netcdf(snakemake.output[0]) + n.optimize.fix_optimal_capacities() + n = prepare_network(n, solve_opts, config=snakemake.config) + n = solve_network( + n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + ) - logger.info("Maximum memory usage: {}".format(mem.mem_usage)) + n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) + n.export_to_netcdf(snakemake.output[0]) From 7d6d6d2805f34a6de6d6e1400d376e8970886c4d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 May 2023 14:59:10 +0000 Subject: [PATCH 35/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/solve_network.py | 4 +--- scripts/solve_operations_network.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 53ac8816..8f4fdf5b 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -668,9 +668,7 @@ if __name__ == "__main__": if "overrides" in snakemake.input.keys(): overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network( - snakemake.input.network, override_component_attrs=overrides - ) + n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) else: n = pypsa.Network(snakemake.input.network) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 9c2fe3c1..987302bb 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -48,9 +48,7 @@ if __name__ == "__main__": if "overrides" in snakemake.input: overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network( - snakemake.input.network, override_component_attrs=overrides - ) + n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) else: n = pypsa.Network(snakemake.input.network) From 1758076815049e5e38b52859ba81132c21c446d2 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Thu, 11 May 2023 20:34:11 +0200 Subject: [PATCH 36/78] test param in add_electricity --- rules/build_electricity.smk | 8 ++++++ scripts/add_electricity.py | 50 ++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 133ceb6e..194e186d 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -273,6 +273,14 @@ rule add_electricity: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", output: RESOURCES + "networks/elec.nc", + params: + costs=snakemake.config["costs"] + electricity=snakemake.config["electricity"] + renewable=snakemake.config["renewable"] + conventional=snakemake.config.get("conventional", {}) + countries=snakemake.config["countries"] + scaling_factor=snakemake.config["load"]["scaling_factor"] + length_factor=snakemake.config["lines"]["length_factor"] log: LOGS + "add_electricity.log", benchmark: diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 1d32bce1..bed5ee6b 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -686,15 +686,15 @@ def estimate_renewable_capacities(n, config): ) -def add_nice_carrier_names(n, config): +def add_nice_carrier_names(n, plotting): carrier_i = n.carriers.index nice_names = ( - pd.Series(config["plotting"]["nice_names"]) + pd.Series(plotting["nice_names"]) .reindex(carrier_i) .fillna(carrier_i.to_series().str.title()) ) n.carriers["nice_name"] = nice_names - colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i) + colors = pd.Series(plotting["tech_colors"]).reindex(carrier_i) if colors.isna().any(): missing_i = list(colors.index[colors.isna()]) logger.warning(f"tech_colors for carriers {missing_i} not defined in config.") @@ -713,23 +713,23 @@ if __name__ == "__main__": costs = load_costs( snakemake.input.tech_costs, - snakemake.config["costs"], - snakemake.config["electricity"], + snakemake.param["costs"], + snakemake.param["electricity"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in snakemake.config["electricity"]: - renewable_carriers = set(snakemake.config["electricity"]["renewable_carriers"]) + if "renewable_carriers" in snakemake.param["electricity"]: + renewable_carriers = set(snakemake.param["electricity"]["renewable_carriers"]) else: logger.warning( "Missing key `renewable_carriers` under config entry `electricity`. " "In future versions, this will raise an error. " "Falling back to carriers listed under `renewable`." ) - renewable_carriers = snakemake.config["renewable"] + renewable_carriers = snakemake.param["renewable"] - extendable_carriers = snakemake.config["electricity"]["extendable_carriers"] + extendable_carriers = snakemake.param["electricity"]["extendable_carriers"] if not (set(renewable_carriers) & set(extendable_carriers["Generator"])): logger.warning( "No renewables found in config entry `extendable_carriers`. " @@ -737,18 +737,18 @@ if __name__ == "__main__": "Falling back to all renewables." ) - conventional_carriers = snakemake.config["electricity"]["conventional_carriers"] + conventional_carriers = snakemake.param["electricity"]["conventional_carriers"] attach_load( n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.config["countries"], - snakemake.config["load"]["scaling_factor"], + snakemake.param["countries"], + snakemake.param["load"]["scaling_factor"], ) - update_transmission_costs(n, costs, snakemake.config["lines"]["length_factor"]) + update_transmission_costs(n, costs, snakemake.param["lines"]["length_factor"]) conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") @@ -759,7 +759,7 @@ if __name__ == "__main__": ppl, conventional_carriers, extendable_carriers, - snakemake.config.get("conventional", {}), + snakemake.param.get("conventional", {}), conventional_inputs, ) @@ -769,11 +769,11 @@ if __name__ == "__main__": snakemake.input, renewable_carriers, extendable_carriers, - snakemake.config["lines"]["length_factor"], + snakemake.param["lines"]["length_factor"], ) if "hydro" in renewable_carriers: - conf = snakemake.config["renewable"]["hydro"] + conf = snakemake.param["renewable"]["hydro"] attach_hydro( n, costs, @@ -784,7 +784,7 @@ if __name__ == "__main__": **conf, ) - if "estimate_renewable_capacities" not in snakemake.config["electricity"]: + if "estimate_renewable_capacities" not in snakemake.param["electricity"]: logger.warning( "Missing key `estimate_renewable_capacities` under config entry `electricity`. " "In future versions, this will raise an error. " @@ -792,18 +792,18 @@ if __name__ == "__main__": ) if ( "estimate_renewable_capacities_from_capacity_stats" - in snakemake.config["electricity"] + in snakemake.param["electricity"] ): estimate_renewable_caps = { "enable": True, - **snakemake.config["electricity"][ + **snakemake.param["electricity"][ "estimate_renewable_capacities_from_capacity_stats" ], } else: estimate_renewable_caps = {"enable": False} else: - estimate_renewable_caps = snakemake.config["electricity"][ + estimate_renewable_caps = snakemake.param["electricity"][ "estimate_renewable_capacities" ] if "enable" not in estimate_renewable_caps: @@ -819,21 +819,21 @@ if __name__ == "__main__": "Falling back to whether `renewable_capacities_from_opsd` is non-empty." ) from_opsd = bool( - snakemake.config["electricity"].get("renewable_capacities_from_opsd", False) + snakemake.param["electricity"].get("renewable_capacities_from_opsd", False) ) estimate_renewable_caps["from_opsd"] = from_opsd if estimate_renewable_caps["enable"]: if estimate_renewable_caps["from_opsd"]: - tech_map = snakemake.config["electricity"]["estimate_renewable_capacities"][ + tech_map = snakemake.param["electricity"]["estimate_renewable_capacities"][ "technology_mapping" ] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.config) + estimate_renewable_capacities(n, snakemake.param) update_p_nom_max(n) - add_nice_carrier_names(n, snakemake.config) + add_nice_carrier_names(n, snakemake.param["plotting"]) - n.meta = snakemake.config + n.meta = snakemake.param n.export_to_netcdf(snakemake.output[0]) From aa50ea44cc550433b8db38bb241c359b8ccc92ce Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Thu, 11 May 2023 20:59:37 +0200 Subject: [PATCH 37/78] fix add_electricity mistake --- rules/build_electricity.smk | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 194e186d..8b7780db 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -274,13 +274,13 @@ rule add_electricity: output: RESOURCES + "networks/elec.nc", params: - costs=snakemake.config["costs"] - electricity=snakemake.config["electricity"] - renewable=snakemake.config["renewable"] - conventional=snakemake.config.get("conventional", {}) - countries=snakemake.config["countries"] - scaling_factor=snakemake.config["load"]["scaling_factor"] - length_factor=snakemake.config["lines"]["length_factor"] + costs=snakemake.config["costs"], + electricity=snakemake.config["electricity"], + renewable=snakemake.config["renewable"], + conventional=snakemake.config.get("conventional", {}), + countries=snakemake.config["countries"], + scaling_factor=snakemake.config["load"]["scaling_factor"], + length_factor=snakemake.config["lines"]["length_factor"], log: LOGS + "add_electricity.log", benchmark: From e29571535b504c2db3733e877ca0d0c70248b07f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 07:11:41 +0200 Subject: [PATCH 38/78] remove references to memory logfile in *.smk, add release note --- doc/release_notes.rst | 2 ++ doc/requirements.txt | 1 - rules/solve_electricity.smk | 4 ---- rules/solve_myopic.smk | 2 -- rules/solve_overnight.smk | 2 -- scripts/solve_operations_network.py | 1 - 6 files changed, 2 insertions(+), 10 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 8e6bfdd4..14f2939d 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -23,6 +23,8 @@ Upcoming Release hydrogen fuel cell. Add switches for both re-electrification options under ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``. +* Remove ``vresutils`` dependency. + PyPSA-Eur 0.8.0 (18th March 2023) ================================= diff --git a/doc/requirements.txt b/doc/requirements.txt index 2f08b8d9..3e760c81 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -9,7 +9,6 @@ sphinxcontrib-bibtex myst-parser # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820 pypsa -vresutils>=0.3.1 powerplantmatching>=0.5.5 atlite>=0.2.9 dask[distributed] diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 8ddeca92..fc70ce42 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -14,8 +14,6 @@ rule solve_network: ), python=LOGS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", - memory=LOGS - + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log", benchmark: BENCHMARKS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 @@ -41,8 +39,6 @@ rule solve_operations_network: ), python=LOGS + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", - memory=LOGS - + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log", benchmark: ( BENCHMARKS diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index f10d8157..ec4281ff 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -90,8 +90,6 @@ rule solve_sector_network_myopic: + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log", python=LOGS + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", - memory=LOGS - + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log", threads: 4 resources: mem_mb=config["solving"]["mem"], diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index c2e103e5..f05925b0 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -21,8 +21,6 @@ rule solve_sector_network: + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log", python=LOGS + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", - memory=LOGS - + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log", threads: config["solving"]["solver"].get("threads", 4) resources: mem_mb=config["solving"]["mem"], diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 987302bb..27520485 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -17,7 +17,6 @@ from _helpers import ( update_config_with_sector_opts, ) from solve_network import prepare_network, solve_network -from vresutils.benchmark import memory_logger logger = logging.getLogger(__name__) From 5be21dfc5a4cefd5e01bbf7798663d18eeacc638 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 09:07:16 +0200 Subject: [PATCH 39/78] add option for piecewise linear transmission loss approximation --- config/config.default.yaml | 2 +- doc/configtables/solving.csv | 2 +- doc/release_notes.rst | 4 ++++ envs/environment.yaml | 2 +- scripts/solve_network.py | 3 +++ 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 10e6a6ed..78c1385c 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -623,9 +623,9 @@ clustering: solving: #tmpdir: "path/to/tmp" options: - formulation: kirchhoff clip_p_max_pu: 1.e-2 load_shedding: false + transmission_losses: 0 noisy_costs: true skip_iterations: true track_iterations: false diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv index cba28cbe..c252ff32 100644 --- a/doc/configtables/solving.csv +++ b/doc/configtables/solving.csv @@ -1,7 +1,7 @@ ,Unit,Values,Description options,,, --- formulation,--,"Any of {'angles', 'kirchhoff', 'cycles', 'ptdf'}","Specifies which variant of linearized power flow formulations to use in the optimisation problem. Recommended is 'kirchhoff'. Explained in `this article `_." -- load_shedding,bool/float,"{'true','false', float}","Add generators with very high marginal cost to simulate load shedding and avoid problem infeasibilities. If load shedding is a float, it denotes the marginal cost in EUR/kWh." +-- transmission_losses,int,"[0-9]","Add piecewise linear approximation of transmission losses based on n tangents. Defaults to 0, which means losses are ignored." -- noisy_costs,bool,"{'true','false'}","Add random noise to marginal cost of generators by :math:`\mathcal{U}(0.009,0,011)` and capital cost of lines and links by :math:`\mathcal{U}(0.09,0,11)`." -- min_iterations,--,int,"Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run." -- max_iterations,--,int,"Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run." diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 14f2939d..3af16477 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -25,6 +25,10 @@ Upcoming Release * Remove ``vresutils`` dependency. +* Add option to include a piecewise linear approximation of transmission losses, + e.g. by setting ``solving: options: transmission_losses: 2`` for an + approximation with two tangents. + PyPSA-Eur 0.8.0 (18th March 2023) ================================= diff --git a/envs/environment.yaml b/envs/environment.yaml index f970c9ba..9d800fdc 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -10,7 +10,7 @@ dependencies: - python>=3.8 - pip -- pypsa>=0.21.3 +- pypsa>=0.23 - atlite>=0.2.9 - dask diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 8f4fdf5b..e671ffd3 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -600,6 +600,7 @@ def solve_network(n, config, opts="", **kwargs): track_iterations = cf_solving.get("track_iterations", False) min_iterations = cf_solving.get("min_iterations", 4) max_iterations = cf_solving.get("max_iterations", 6) + transmission_losses = cf_solving.get("transmission_losses", 0) # add to network for extra_functionality n.config = config @@ -613,6 +614,7 @@ def solve_network(n, config, opts="", **kwargs): if skip_iterations: status, condition = n.optimize( solver_name=solver_name, + transmission_losses=transmission_losses, extra_functionality=extra_functionality, **solver_options, **kwargs, @@ -623,6 +625,7 @@ def solve_network(n, config, opts="", **kwargs): track_iterations=track_iterations, min_iterations=min_iterations, max_iterations=max_iterations, + transmission_losses=transmission_losses, extra_functionality=extra_functionality, **solver_options, **kwargs, From 67ef38b35eb2e46d8dcc5252216b73ee062465f0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 13:59:41 +0200 Subject: [PATCH 40/78] retrieve*: handle extraction path via snakemake.output --- scripts/retrieve_databundle.py | 3 +-- scripts/retrieve_gas_infrastructure_data.py | 2 +- scripts/retrieve_sector_databundle.py | 16 +++++++++------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 0c6a7feb..75d8519e 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -58,9 +58,8 @@ if __name__ == "__main__": else: url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz" - # Save locations tarball_fn = Path(f"{rootpath}/bundle.tar.xz") - to_fn = Path(f"{rootpath}/data") + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) diff --git a/scripts/retrieve_gas_infrastructure_data.py b/scripts/retrieve_gas_infrastructure_data.py index dda7bd8c..42b726db 100644 --- a/scripts/retrieve_gas_infrastructure_data.py +++ b/scripts/retrieve_gas_infrastructure_data.py @@ -29,7 +29,7 @@ if __name__ == "__main__": # Save locations zip_fn = Path(f"{rootpath}/IGGIELGN.zip") - to_fn = Path(f"{rootpath}/data/gas_network/scigrid-gas") + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py index 97426ab2..0991bbe3 100644 --- a/scripts/retrieve_sector_databundle.py +++ b/scripts/retrieve_sector_databundle.py @@ -10,23 +10,25 @@ import logging logger = logging.getLogger(__name__) -import os -import sys import tarfile from pathlib import Path -# Add pypsa-eur scripts to path for import of _helpers -sys.path.insert(0, os.getcwd() + "/../pypsa-eur/scripts") - from _helpers import configure_logging, progress_retrieve if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("retrieve_databundle") + rootpath = ".." + else: + rootpath = "." configure_logging(snakemake) url = "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz" - tarball_fn = Path("sector-bundle.tar.gz") - to_fn = Path("data") + tarball_fn = Path(f"{rootpath}/sector-bundle.tar.gz") + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) From b2216355f10bf11147cc7f03e74b17f6dc4b60e6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 14:10:04 +0200 Subject: [PATCH 41/78] retrieve_sector_data: add another .parent --- scripts/retrieve_sector_databundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py index 0991bbe3..0d172c8d 100644 --- a/scripts/retrieve_sector_databundle.py +++ b/scripts/retrieve_sector_databundle.py @@ -28,7 +28,7 @@ if __name__ == "__main__": url = "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz" tarball_fn = Path(f"{rootpath}/sector-bundle.tar.gz") - to_fn = Path(rootpath) / Path(snakemake.output[0]).parent + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) From 8f91963e7623d6cf574c0fd267b80b581f9560d8 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 14:50:58 +0200 Subject: [PATCH 42/78] properly reference p_max_pu files --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index f910dee4..85ab35e2 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -418,7 +418,7 @@ def attach_conventional_generators( if f"conventional_{carrier}_{attr}" in conventional_inputs: # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - values = pd.read_csv(values, index_col=0).iloc[:, 0] + values = pd.read_csv(snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0).iloc[:, 0] bus_values = n.buses.country.map(values) n.generators[attr].update( n.generators.loc[idx].bus.map(bus_values).dropna() From 38bae672da42db241eb885eceeb3a6097644eeb5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 May 2023 12:51:39 +0000 Subject: [PATCH 43/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_electricity.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 85ab35e2..4afa3e22 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -418,7 +418,9 @@ def attach_conventional_generators( if f"conventional_{carrier}_{attr}" in conventional_inputs: # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - values = pd.read_csv(snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0).iloc[:, 0] + values = pd.read_csv( + snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0 + ).iloc[:, 0] bus_values = n.buses.country.map(values) n.generators[attr].update( n.generators.loc[idx].bus.map(bus_values).dropna() From 1fc48d8753f93b2b2d2f24c2516cf16777d72cb4 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Mon, 15 May 2023 10:33:17 +0200 Subject: [PATCH 44/78] using python algorithm to edit rules and script --- rules/build_electricity.smk | 55 +++++++++++++--- rules/build_sector.smk | 65 +++++++++++++++++++ rules/postprocess.smk | 13 ++++ rules/retrieve.smk | 2 + rules/solve_electricity.smk | 4 ++ rules/solve_myopic.smk | 12 ++++ rules/solve_overnight.smk | 2 + scripts/_helpers.py | 2 +- scripts/add_brownfield.py | 8 +-- scripts/add_electricity.py | 50 +++++++------- scripts/add_existing_baseyear.py | 20 +++--- scripts/add_extra_components.py | 4 +- scripts/build_ammonia_production.py | 2 +- scripts/build_biomass_potentials.py | 2 +- scripts/build_bus_regions.py | 2 +- scripts/build_cop_profiles.py | 2 +- scripts/build_cutout.py | 4 +- scripts/build_electricity_demand.py | 12 ++-- scripts/build_energy_totals.py | 12 ++-- scripts/build_gas_input_locations.py | 2 +- scripts/build_gas_network.py | 2 +- scripts/build_heat_demand.py | 2 +- scripts/build_hydro_profile.py | 4 +- scripts/build_industrial_distribution_key.py | 6 +- ...ustrial_energy_demand_per_country_today.py | 6 +- ...build_industrial_production_per_country.py | 10 +-- ...ustrial_production_per_country_tomorrow.py | 2 +- scripts/build_industry_sector_ratios.py | 4 +- scripts/build_population_layouts.py | 2 +- scripts/build_powerplants.py | 6 +- scripts/build_renewable_profiles.py | 4 +- scripts/build_retro_cost.py | 4 +- scripts/build_sequestration_potentials.py | 2 +- scripts/build_shapes.py | 4 +- scripts/build_solar_thermal_profiles.py | 4 +- scripts/build_temperature_profiles.py | 2 +- scripts/build_transport_demand.py | 4 +- scripts/cluster_gas_network.py | 2 +- scripts/cluster_network.py | 18 ++--- scripts/make_summary.py | 22 +++---- scripts/plot_network.py | 16 ++--- scripts/plot_summary.py | 32 ++++----- scripts/prepare_network.py | 20 +++--- scripts/prepare_sector_network.py | 42 ++++++------ scripts/retrieve_databundle.py | 4 +- scripts/simplify_network.py | 8 +-- scripts/solve_network.py | 2 +- scripts/solve_operations_network.py | 2 +- 48 files changed, 324 insertions(+), 187 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 8b7780db..bb000f2b 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -19,6 +19,10 @@ if config["enable"].get("prepare_links_p_nom", False): rule build_electricity_demand: + params: + snapshots=config["snapshots"], + countries=config["countries"], + load=config["load"], input: ancient("data/load_raw.csv"), output: @@ -34,6 +38,9 @@ rule build_electricity_demand: rule build_powerplants: + params: + electricity=config["electricity"], + countries=config["countries"], input: base_network=RESOURCES + "networks/base.nc", custom_powerplants="data/custom_powerplants.csv", @@ -79,6 +86,8 @@ rule base_network: rule build_shapes: + params: + countries=config["countries"], input: naturalearth=ancient("data/bundle/naturalearth/ne_10m_admin_0_countries.shp"), eez=ancient("data/bundle/eez/World_EEZ_v8_2014.shp"), @@ -104,6 +113,8 @@ rule build_shapes: rule build_bus_regions: + params: + countries=config["countries"], input: country_shapes=RESOURCES + "country_shapes.geojson", offshore_shapes=RESOURCES + "offshore_shapes.geojson", @@ -125,6 +136,9 @@ rule build_bus_regions: if config["enable"].get("build_cutout", False): rule build_cutout: + params: + snapshots=config["snapshots"], + atlite=config["atlite"], input: regions_onshore=RESOURCES + "regions_onshore.geojson", regions_offshore=RESOURCES + "regions_offshore.geojson", @@ -186,6 +200,8 @@ rule build_ship_raster: rule build_renewable_profiles: + params: + renewable=config["renewable"], input: base_network=RESOURCES + "networks/base.nc", corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"), @@ -235,6 +251,9 @@ rule build_renewable_profiles: rule build_hydro_profile: + params: + countries=config["countries"], + renewable=config["renewable"], input: country_shapes=RESOURCES + "country_shapes.geojson", eia_hydro_generation="data/eia_hydro_annual_generation.csv", @@ -252,6 +271,13 @@ rule build_hydro_profile: rule add_electricity: + params: + lines=config["lines"], + load=config["load"], + countries=config["countries"], + renewable=config["renewable"], + electricity=config["electricity"], + costs=config["costs"], input: **{ f"profile_{tech}": RESOURCES + f"profile_{tech}.nc" @@ -273,14 +299,6 @@ rule add_electricity: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", output: RESOURCES + "networks/elec.nc", - params: - costs=snakemake.config["costs"], - electricity=snakemake.config["electricity"], - renewable=snakemake.config["renewable"], - conventional=snakemake.config.get("conventional", {}), - countries=snakemake.config["countries"], - scaling_factor=snakemake.config["load"]["scaling_factor"], - length_factor=snakemake.config["lines"]["length_factor"], log: LOGS + "add_electricity.log", benchmark: @@ -295,6 +313,10 @@ rule add_electricity: rule simplify_network: + params: + clustering=config["clustering"], + electricity=config["electricity"], + costs=config["costs"], input: network=RESOURCES + "networks/elec.nc", tech_costs=COSTS, @@ -320,6 +342,14 @@ rule simplify_network: rule cluster_network: + params: + solving=config["solving"], + electricity=config["electricity"], + costs=config["costs"], + lines=config["lines"], + renewable=config["renewable"], + clustering=config["clustering"], + enable=config["enable"], input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", @@ -351,6 +381,9 @@ rule cluster_network: rule add_extra_components: + params: + costs=config["costs"], + electricity=config["electricity"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", tech_costs=COSTS, @@ -370,6 +403,12 @@ rule add_extra_components: rule prepare_network: + params: + links=config["links"], + solving=config["solving"], + lines=config["lines"], + electricity=config["electricity"], + costs=config["costs"], input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", tech_costs=COSTS, diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 1b724d1a..d375b7b9 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -4,6 +4,8 @@ rule build_population_layouts: + params: + logging=config["logging"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", urban_percent="data/urban_percent.csv", @@ -70,6 +72,8 @@ rule build_simplified_population_layouts: if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule build_gas_network: + params: + logging=config["logging"], input: gas_network="data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson", output: @@ -84,6 +88,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/build_gas_network.py" rule build_gas_input_locations: + params: + logging=config["logging"], input: lng=HTTP.remote( "https://globalenergymonitor.org/wp-content/uploads/2022/09/Europe-Gas-Tracker-August-2022.xlsx", @@ -110,6 +116,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/build_gas_input_locations.py" rule cluster_gas_network: + params: + logging=config["logging"], input: cleaned_gas_network=RESOURCES + "gas_network.csv", regions_onshore=RESOURCES @@ -140,6 +148,8 @@ if not (config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]): rule build_heat_demands: + params: + snapshots=config["snapshots"], input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -160,6 +170,8 @@ rule build_heat_demands: rule build_temperature_profiles: + params: + snapshots=config["snapshots"], input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -181,6 +193,8 @@ rule build_temperature_profiles: rule build_cop_profiles: + params: + sector=config["sector"], input: temp_soil_total=RESOURCES + "temp_soil_total_elec_s{simpl}_{clusters}.nc", temp_soil_rural=RESOURCES + "temp_soil_rural_elec_s{simpl}_{clusters}.nc", @@ -208,6 +222,9 @@ rule build_cop_profiles: rule build_solar_thermal_profiles: + params: + snapshots=config["snapshots"], + solar_thermal=config["solar_thermal"], input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -228,6 +245,11 @@ rule build_solar_thermal_profiles: rule build_energy_totals: + params: + run=config["run"], + countries=config["countries"], + energy=config["energy"], + logging=config["logging"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", co2="data/eea/UNFCCC_v23.csv", @@ -253,6 +275,8 @@ rule build_energy_totals: rule build_biomass_potentials: + params: + biomass=config["biomass"], input: enspreso_biomass=HTTP.remote( "https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", @@ -315,6 +339,8 @@ if not config["sector"]["biomass_transport"]: if config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_sequestration_potentials: + params: + sector=config["sector"], input: sequestration_potential=HTTP.remote( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", @@ -368,6 +394,8 @@ rule build_salt_cavern_potentials: rule build_ammonia_production: + params: + countries=config["countries"], input: usgs="data/myb1-2017-nitro.xls", output: @@ -386,6 +414,9 @@ rule build_ammonia_production: rule build_industry_sector_ratios: + params: + industry=config["industry"], + sector=config["sector"], input: ammonia_production=RESOURCES + "ammonia_production.csv", idees="data/jrc-idees-2015", @@ -405,6 +436,11 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: + params: + run=config["run"], + industry=config["industry"], + countries=config["countries"], + logging=config["logging"], input: ammonia_production=RESOURCES + "ammonia_production.csv", jrc="data/jrc-idees-2015", @@ -426,6 +462,8 @@ rule build_industrial_production_per_country: rule build_industrial_production_per_country_tomorrow: + params: + industry=config["industry"], input: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -450,6 +488,10 @@ rule build_industrial_production_per_country_tomorrow: rule build_industrial_distribution_key: + params: + industry=config["industry"], + countries=config["countries"], + logging=config["logging"], input: regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", @@ -524,6 +566,10 @@ rule build_industrial_energy_demand_per_node: rule build_industrial_energy_demand_per_country_today: + params: + run=config["run"], + countries=config["countries"], + industry=config["industry"], input: jrc="data/jrc-idees-2015", ammonia_production=RESOURCES + "ammonia_production.csv", @@ -570,6 +616,9 @@ rule build_industrial_energy_demand_per_node_today: if config["sector"]["retrofitting"]["retro_endogen"]: rule build_retro_cost: + params: + sector=config["sector"], + countries=config["countries"], input: building_stock="data/retro/data_building_stock.csv", data_tabula="data/retro/tabula-calculator-calcsetbuilding.csv", @@ -640,6 +689,9 @@ rule build_shipping_demand: rule build_transport_demand: + params: + snapshots=config["snapshots"], + sector=config["sector"], input: clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", pop_weighted_energy_totals=RESOURCES @@ -666,6 +718,19 @@ rule build_transport_demand: rule prepare_sector_network: params: + co2_budget=config["co2_budget"], + solving=config["solving"], + existing_capacities=config["existing_capacities"], + foresight=config["foresight"], + costs=config["costs"], + logging=config["logging"], + sector=config["sector"], + industry=config["industry"], + pypsa_eur=config["pypsa_eur"], + lines=config["lines"], + scenario=config["scenario"], + countries=config["countries"], + energy=config["energy"], RDIR=RDIR, input: **build_retro_cost_output, diff --git a/rules/postprocess.smk b/rules/postprocess.smk index fae0f856..1dfdd098 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -9,6 +9,10 @@ localrules: rule plot_network: + params: + logging=config["logging"], + foresight=config["foresight"], + plotting=config["plotting"], input: overrides="data/override_component_attrs", network=RESULTS @@ -67,6 +71,11 @@ rule copy_conda_env: rule make_summary: params: + foresight=config["foresight"], + costs=config["costs"], + snapshots=config["snapshots"], + logging=config["logging"], + scenario=config["scenario"], RDIR=RDIR, input: overrides="data/override_component_attrs", @@ -114,6 +123,10 @@ rule make_summary: rule plot_summary: params: + logging=config["logging"], + countries=config["countries"], + scenario=config["scenario"], + plotting=config["plotting"], RDIR=RDIR, input: costs=RESULTS + "csvs/costs.csv", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 0a96406a..4bfbd6c6 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -19,6 +19,8 @@ if config["enable"].get("retrieve_databundle", True): datafiles.extend(["natura/Natura2000_end2015.shp", "GEBCO_2014_2D.nc"]) rule retrieve_databundle: + params: + tutorial=config["tutorial"], output: expand("data/bundle/{file}", file=datafiles), log: diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 8ddeca92..400220c1 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -4,6 +4,8 @@ rule solve_network: + params: + solving=config["solving"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: @@ -30,6 +32,8 @@ rule solve_network: rule solve_operations_network: + params: + solving=config["solving"], input: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index f10d8157..2ecba999 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -4,6 +4,12 @@ rule add_existing_baseyear: + params: + scenario=config["scenario"], + sector=config["sector"], + logging=config["logging"], + existing_capacities=config["existing_capacities"], + costs=config["costs"], input: overrides="data/override_component_attrs", network=RESULTS @@ -42,6 +48,10 @@ rule add_existing_baseyear: rule add_brownfield: + params: + logging=config["logging"], + sector=config["sector"], + existing_capacities=config["existing_capacities"], input: overrides="data/override_component_attrs", network=RESULTS @@ -74,6 +84,8 @@ ruleorder: add_existing_baseyear > add_brownfield rule solve_sector_network_myopic: + params: + solving=config["solving"], input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index c2e103e5..c3608471 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -4,6 +4,8 @@ rule solve_sector_network: + params: + solving=config["solving"], input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/_helpers.py b/scripts/_helpers.py index a67fb105..2cc9a05a 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -82,7 +82,7 @@ def load_network(import_name=None, custom_components=None): As in pypsa.Network(import_name) custom_components : dict Dictionary listing custom components. - For using ``snakemake.config['override_components']`` + For using ``snakemake.params['override_components']`` in ``config/config.yaml`` define: .. code:: yaml diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index e08b86d7..1d4e3a80 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -49,7 +49,7 @@ def add_brownfield(n, n_p, year): ) ] - threshold = snakemake.config["existing_capacities"]["threshold_capacity"] + threshold = snakemake.params["existing_capacities"]["threshold_capacity"] if not chp_heat.empty: threshold_chp_heat = ( @@ -87,7 +87,7 @@ def add_brownfield(n, n_p, year): # deal with gas network pipe_carrier = ["gas pipeline"] - if snakemake.config["sector"]["H2_retrofit"]: + if snakemake.params["sector"]["H2_retrofit"]: # drop capacities of previous year to avoid duplicating to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year) n.mremove("Link", n.links.loc[to_drop].index) @@ -98,7 +98,7 @@ def add_brownfield(n, n_p, year): & (n.links.build_year != year) ].index gas_pipes_i = n.links[n.links.carrier.isin(pipe_carrier)].index - CH4_per_H2 = 1 / snakemake.config["sector"]["H2_retrofit_capacity_per_CH4"] + CH4_per_H2 = 1 / snakemake.params["sector"]["H2_retrofit_capacity_per_CH4"] fr = "H2 pipeline retrofitted" to = "gas pipeline" # today's pipe capacity @@ -139,7 +139,7 @@ if __name__ == "__main__": planning_horizons=2030, ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index bed5ee6b..c3e847ce 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -686,15 +686,15 @@ def estimate_renewable_capacities(n, config): ) -def add_nice_carrier_names(n, plotting): +def add_nice_carrier_names(n, config): carrier_i = n.carriers.index nice_names = ( - pd.Series(plotting["nice_names"]) + pd.Series(config["plotting"]["nice_names"]) .reindex(carrier_i) .fillna(carrier_i.to_series().str.title()) ) n.carriers["nice_name"] = nice_names - colors = pd.Series(plotting["tech_colors"]).reindex(carrier_i) + colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i) if colors.isna().any(): missing_i = list(colors.index[colors.isna()]) logger.warning(f"tech_colors for carriers {missing_i} not defined in config.") @@ -713,23 +713,23 @@ if __name__ == "__main__": costs = load_costs( snakemake.input.tech_costs, - snakemake.param["costs"], - snakemake.param["electricity"], + snakemake.params["costs"], + snakemake.params["electricity"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in snakemake.param["electricity"]: - renewable_carriers = set(snakemake.param["electricity"]["renewable_carriers"]) + if "renewable_carriers" in snakemake.params["electricity"]: + renewable_carriers = set(snakemake.params["electricity"]["renewable_carriers"]) else: logger.warning( "Missing key `renewable_carriers` under config entry `electricity`. " "In future versions, this will raise an error. " "Falling back to carriers listed under `renewable`." ) - renewable_carriers = snakemake.param["renewable"] + renewable_carriers = snakemake.params["renewable"] - extendable_carriers = snakemake.param["electricity"]["extendable_carriers"] + extendable_carriers = snakemake.params["electricity"]["extendable_carriers"] if not (set(renewable_carriers) & set(extendable_carriers["Generator"])): logger.warning( "No renewables found in config entry `extendable_carriers`. " @@ -737,18 +737,18 @@ if __name__ == "__main__": "Falling back to all renewables." ) - conventional_carriers = snakemake.param["electricity"]["conventional_carriers"] + conventional_carriers = snakemake.params["electricity"]["conventional_carriers"] attach_load( n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.param["countries"], - snakemake.param["load"]["scaling_factor"], + snakemake.params["countries"], + snakemake.params["load"]["scaling_factor"], ) - update_transmission_costs(n, costs, snakemake.param["lines"]["length_factor"]) + update_transmission_costs(n, costs, snakemake.params["lines"]["length_factor"]) conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") @@ -759,7 +759,7 @@ if __name__ == "__main__": ppl, conventional_carriers, extendable_carriers, - snakemake.param.get("conventional", {}), + snakemake.config.get("conventional", {}), conventional_inputs, ) @@ -769,11 +769,11 @@ if __name__ == "__main__": snakemake.input, renewable_carriers, extendable_carriers, - snakemake.param["lines"]["length_factor"], + snakemake.params["lines"]["length_factor"], ) if "hydro" in renewable_carriers: - conf = snakemake.param["renewable"]["hydro"] + conf = snakemake.params["renewable"]["hydro"] attach_hydro( n, costs, @@ -784,7 +784,7 @@ if __name__ == "__main__": **conf, ) - if "estimate_renewable_capacities" not in snakemake.param["electricity"]: + if "estimate_renewable_capacities" not in snakemake.params["electricity"]: logger.warning( "Missing key `estimate_renewable_capacities` under config entry `electricity`. " "In future versions, this will raise an error. " @@ -792,18 +792,18 @@ if __name__ == "__main__": ) if ( "estimate_renewable_capacities_from_capacity_stats" - in snakemake.param["electricity"] + in snakemake.params["electricity"] ): estimate_renewable_caps = { "enable": True, - **snakemake.param["electricity"][ + **snakemake.params["electricity"][ "estimate_renewable_capacities_from_capacity_stats" ], } else: estimate_renewable_caps = {"enable": False} else: - estimate_renewable_caps = snakemake.param["electricity"][ + estimate_renewable_caps = snakemake.params["electricity"][ "estimate_renewable_capacities" ] if "enable" not in estimate_renewable_caps: @@ -819,21 +819,21 @@ if __name__ == "__main__": "Falling back to whether `renewable_capacities_from_opsd` is non-empty." ) from_opsd = bool( - snakemake.param["electricity"].get("renewable_capacities_from_opsd", False) + snakemake.params["electricity"].get("renewable_capacities_from_opsd", False) ) estimate_renewable_caps["from_opsd"] = from_opsd if estimate_renewable_caps["enable"]: if estimate_renewable_caps["from_opsd"]: - tech_map = snakemake.param["electricity"]["estimate_renewable_capacities"][ + tech_map = snakemake.params["electricity"]["estimate_renewable_capacities"][ "technology_mapping" ] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.param) + estimate_renewable_capacities(n, snakemake.config) update_p_nom_max(n) - add_nice_carrier_names(n, snakemake.param["plotting"]) + add_nice_carrier_names(n, snakemake.config) - n.meta = snakemake.param + n.meta = snakemake.config n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 5bc0960d..a24b078d 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -157,7 +157,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas # Fill missing DateOut dateout = ( df_agg.loc[biomass_i, "DateIn"] - + snakemake.config["costs"]["fill_values"]["lifetime"] + + snakemake.params["costs"]["fill_values"]["lifetime"] ) df_agg.loc[biomass_i, "DateOut"] = df_agg.loc[biomass_i, "DateOut"].fillna(dateout) @@ -218,7 +218,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas capacity = df.loc[grouping_year, generator] capacity = capacity[~capacity.isna()] capacity = capacity[ - capacity > snakemake.config["existing_capacities"]["threshold_capacity"] + capacity > snakemake.params["existing_capacities"]["threshold_capacity"] ] suffix = "-ac" if generator == "offwind" else "" name_suffix = f" {generator}{suffix}-{grouping_year}" @@ -582,7 +582,7 @@ def add_heating_capacities_installed_before_baseyear( ) # delete links with capacities below threshold - threshold = snakemake.config["existing_capacities"]["threshold_capacity"] + threshold = snakemake.params["existing_capacities"]["threshold_capacity"] n.mremove( "Link", [ @@ -608,14 +608,14 @@ if __name__ == "__main__": planning_horizons=2020, ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) - options = snakemake.config["sector"] + options = snakemake.params["sector"] opts = snakemake.wildcards.sector_opts.split("-") - baseyear = snakemake.config["scenario"]["planning_horizons"][0] + baseyear = snakemake.params["scenario"]["planning_horizons"][0] overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) @@ -626,14 +626,14 @@ if __name__ == "__main__": Nyears = n.snapshot_weightings.generators.sum() / 8760.0 costs = prepare_costs( snakemake.input.costs, - snakemake.config["costs"], + snakemake.params["costs"], Nyears, ) - grouping_years_power = snakemake.config["existing_capacities"][ + grouping_years_power = snakemake.params["existing_capacities"][ "grouping_years_power" ] - grouping_years_heat = snakemake.config["existing_capacities"]["grouping_years_heat"] + grouping_years_heat = snakemake.params["existing_capacities"]["grouping_years_heat"] add_power_capacities_installed_before_baseyear( n, grouping_years_power, costs, baseyear ) @@ -650,7 +650,7 @@ if __name__ == "__main__": .to_pandas() .reindex(index=n.snapshots) ) - default_lifetime = snakemake.config["costs"]["fill_values"]["lifetime"] + default_lifetime = snakemake.params["costs"]["fill_values"]["lifetime"] add_heating_capacities_installed_before_baseyear( n, baseyear, diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 020370e5..08178c0a 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -235,11 +235,11 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) - elec_config = snakemake.config["electricity"] + elec_config = snakemake.params["electricity"] Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( - snakemake.input.tech_costs, snakemake.config["costs"], elec_config, Nyears + snakemake.input.tech_costs, snakemake.params["costs"], elec_config, Nyears ) attach_storageunits(n, costs, elec_config) diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index d78d627e..6f03324f 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -30,7 +30,7 @@ if __name__ == "__main__": ammonia.index = cc.convert(ammonia.index, to="iso2") years = [str(i) for i in range(2013, 2018)] - countries = ammonia.index.intersection(snakemake.config["countries"]) + countries = ammonia.index.intersection(snakemake.params["countries"]) ammonia = ammonia.loc[countries, years].astype(float) # convert from ktonN to ktonNH3 diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 21d0e623..35218e2c 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -210,7 +210,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_biomass_potentials", simpl="", clusters="5") - config = snakemake.config["biomass"] + config = snakemake.params["biomass"] year = config["year"] scenario = config["scenario"] diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 6dc3b5a4..e9378792 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -116,7 +116,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_bus_regions") configure_logging(snakemake) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] n = pypsa.Network(snakemake.input.base_network) diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index 5d36cd5b..7128ec0d 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -39,7 +39,7 @@ if __name__ == "__main__": for source in ["air", "soil"]: source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"]) - delta_T = snakemake.config["sector"]["heat_pump_sink_T"] - source_T + delta_T = snakemake.params["sector"]["heat_pump_sink_T"] - source_T cop = coefficient_of_performance(delta_T, source) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 365797d2..2f61f017 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -106,9 +106,9 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5") configure_logging(snakemake) - cutout_params = snakemake.config["atlite"]["cutouts"][snakemake.wildcards.cutout] + cutout_params = snakemake.params["atlite"]["cutouts"][snakemake.wildcards.cutout] - snapshots = pd.date_range(freq="h", **snakemake.config["snapshots"]) + snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) time = [snapshots[0], snapshots[-1]] cutout_params["time"] = slice(*cutout_params.get("time", time)) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index b86b4a5f..4ef56d1d 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -279,16 +279,16 @@ if __name__ == "__main__": configure_logging(snakemake) - powerstatistics = snakemake.config["load"]["power_statistics"] - interpolate_limit = snakemake.config["load"]["interpolate_limit"] - countries = snakemake.config["countries"] - snapshots = pd.date_range(freq="h", **snakemake.config["snapshots"]) + powerstatistics = snakemake.params["load"]["power_statistics"] + interpolate_limit = snakemake.params["load"]["interpolate_limit"] + countries = snakemake.params["countries"] + snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) years = slice(snapshots[0], snapshots[-1]) - time_shift = snakemake.config["load"]["time_shift_for_large_gaps"] + time_shift = snakemake.params["load"]["time_shift_for_large_gaps"] load = load_timeseries(snakemake.input[0], years, countries, powerstatistics) - if snakemake.config["load"]["manual_adjustments"]: + if snakemake.params["load"]["manual_adjustments"]: load = manual_adjustment(load, snakemake.input[0], powerstatistics) logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.") diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 45fc960f..6cedff97 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -373,7 +373,7 @@ def idees_per_country(ct, year, base_dir): def build_idees(countries, year): nprocesses = snakemake.threads - disable_progress = snakemake.config["run"].get("disable_progressbar", False) + disable_progress = snakemake.params["run"].get("disable_progressbar", False) func = partial(idees_per_country, year=year, base_dir=snakemake.input.idees) tqdm_kwargs = dict( @@ -735,18 +735,18 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_energy_totals") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) - config = snakemake.config["energy"] + config = snakemake.params["energy"] nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index") population = nuts3["pop"].groupby(nuts3.country).sum() - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] idees_countries = pd.Index(countries).intersection(eu28) data_year = config["energy_totals_year"] - report_year = snakemake.config["energy"]["eurostat_report_year"] + report_year = snakemake.params["energy"]["eurostat_report_year"] input_eurostat = snakemake.input.eurostat eurostat = build_eurostat(input_eurostat, countries, report_year, data_year) swiss = build_swiss(data_year) @@ -756,7 +756,7 @@ if __name__ == "__main__": energy.to_csv(snakemake.output.energy_name) base_year_emissions = config["base_emissions_year"] - emissions_scope = snakemake.config["energy"]["emissions"] + emissions_scope = snakemake.params["energy"]["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope) eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, base_year_emissions diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index a3b945ab..59f31f5c 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -86,7 +86,7 @@ if __name__ == "__main__": clusters="37", ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) regions = load_bus_regions( snakemake.input.regions_onshore, snakemake.input.regions_offshore diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py index 23f58caa..a70f15ff 100644 --- a/scripts/build_gas_network.py +++ b/scripts/build_gas_network.py @@ -147,7 +147,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_gas_network") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) gas_network = load_dataset(snakemake.input.gas_network) diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 56ceb4b7..655df28f 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -27,7 +27,7 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - time = pd.date_range(freq="h", **snakemake.config["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params["snapshots"]) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 0e8cfa27..5453ac5c 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -130,10 +130,10 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) - config_hydro = snakemake.config["renewable"]["hydro"] + config_hydro = snakemake.params["renewable"]["hydro"] cutout = atlite.Cutout(snakemake.input.cutout) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] country_shapes = ( gpd.read_file(snakemake.input.country_shapes) .set_index("name")["geometry"] diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 69daf64d..30c652d8 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -73,7 +73,7 @@ def prepare_hotmaps_database(regions): df[["srid", "coordinates"]] = df.geom.str.split(";", expand=True) - if snakemake.config["industry"].get("hotmaps_locate_missing", False): + if snakemake.params["industry"].get("hotmaps_locate_missing", False): df = locate_missing_industrial_sites(df) # remove those sites without valid locations @@ -141,9 +141,9 @@ if __name__ == "__main__": clusters=48, ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] regions = gpd.read_file(snakemake.input.regions_onshore).set_index("name") diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 703997b1..4fec95cc 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -153,7 +153,7 @@ def add_non_eu28_industrial_energy_demand(countries, demand): def industrial_energy_demand(countries, year): nprocesses = snakemake.threads - disable_progress = snakemake.config["run"].get("disable_progressbar", False) + disable_progress = snakemake.params["run"].get("disable_progressbar", False) func = partial( industrial_energy_demand_per_country, year=year, jrc_dir=snakemake.input.jrc ) @@ -178,9 +178,9 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today") - config = snakemake.config["industry"] + config = snakemake.params["industry"] year = config.get("reference_year", 2015) - countries = pd.Index(snakemake.config["countries"]) + countries = pd.Index(snakemake.params["countries"]) demand = industrial_energy_demand(countries.intersection(eu28), year) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 437806b3..eb1b16cb 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -217,7 +217,7 @@ def industry_production_per_country(country, year, eurostat_dir, jrc_dir): def industry_production(countries, year, eurostat_dir, jrc_dir): nprocesses = snakemake.threads - disable_progress = snakemake.config["run"].get("disable_progressbar", False) + disable_progress = snakemake.params["run"].get("disable_progressbar", False) func = partial( industry_production_per_country, @@ -277,13 +277,13 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_production_per_country") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] - year = snakemake.config["industry"]["reference_year"] + year = snakemake.params["industry"]["reference_year"] - config = snakemake.config["industry"] + config = snakemake.params["industry"] jrc_dir = snakemake.input.jrc eurostat_dir = snakemake.input.eurostat diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py index 6c445608..b9ac9b16 100644 --- a/scripts/build_industrial_production_per_country_tomorrow.py +++ b/scripts/build_industrial_production_per_country_tomorrow.py @@ -15,7 +15,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow") - config = snakemake.config["industry"] + config = snakemake.params["industry"] investment_year = int(snakemake.wildcards.planning_horizons) diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index 54f2cfdc..c3e2bd49 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -439,7 +439,7 @@ def chemicals_industry(): sector = "Ammonia" df[sector] = 0.0 - if snakemake.config["sector"].get("ammonia", False): + if snakemake.params["sector"].get("ammonia", False): df.loc["ammonia", sector] = config["MWh_NH3_per_tNH3"] else: df.loc["hydrogen", sector] = config["MWh_H2_per_tNH3_electrolysis"] @@ -1468,7 +1468,7 @@ if __name__ == "__main__": # TODO make config option year = 2015 - config = snakemake.config["industry"] + config = snakemake.params["industry"] df = pd.concat( [ diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index e864d925..bc35dbcf 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -23,7 +23,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_population_layouts") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) cutout = atlite.Cutout(snakemake.input.cutout) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 9ca67a53..bd0ee74e 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -115,7 +115,7 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] ppl = ( pm.powerplants(from_url=True) @@ -134,12 +134,12 @@ if __name__ == "__main__": ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")') ppl = pd.concat([ppl, opsd]) - ppl_query = snakemake.config["electricity"]["powerplants_filter"] + ppl_query = snakemake.params["electricity"]["powerplants_filter"] if isinstance(ppl_query, str): ppl.query(ppl_query, inplace=True) # add carriers from own powerplant files: - custom_ppl_query = snakemake.config["electricity"]["custom_powerplants"] + custom_ppl_query = snakemake.params["electricity"]["custom_powerplants"] ppl = add_custom_powerplants( ppl, snakemake.input.custom_powerplants, custom_ppl_query ) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index c0288aee..55541522 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -203,8 +203,8 @@ if __name__ == "__main__": configure_logging(snakemake) nprocesses = int(snakemake.threads) - noprogress = snakemake.config["run"].get("disable_progressbar", True) - config = snakemake.config["renewable"][snakemake.wildcards.technology] + noprogress = snakemake.params["run"].get("disable_progressbar", True) + config = snakemake.params["renewable"][snakemake.wildcards.technology] resource = config["resource"] # pv panel config / wind turbine config correction_factor = config.get("correction_factor", 1.0) capacity_per_sqkm = config["capacity_per_sqkm"] diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index 9dbfc375..c52d4eb6 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -305,7 +305,7 @@ def prepare_building_stock_data(): u_values.set_index(["country_code", "subsector", "bage", "type"], inplace=True) # only take in config.yaml specified countries into account - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] area_tot = area_tot.loc[countries] return u_values, country_iso_dic, countries, area_tot, area @@ -1040,7 +1040,7 @@ if __name__ == "__main__": # ******** config ********************************************************* - retro_opts = snakemake.config["sector"]["retrofitting"] + retro_opts = snakemake.params["sector"]["retrofitting"] interest_rate = retro_opts["interest_rate"] annualise_cost = retro_opts["annualise_cost"] # annualise the investment costs tax_weighting = retro_opts[ diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 012effe8..5c388b2e 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -41,7 +41,7 @@ if __name__ == "__main__": "build_sequestration_potentials", simpl="", clusters="181" ) - cf = snakemake.config["sector"]["regional_co2_sequestration_potential"] + cf = snakemake.params["sector"]["regional_co2_sequestration_potential"] gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 50d21e12..f529517c 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -255,12 +255,12 @@ if __name__ == "__main__": configure_logging(snakemake) country_shapes = countries( - snakemake.input.naturalearth, snakemake.config["countries"] + snakemake.input.naturalearth, snakemake.params["countries"] ) country_shapes.reset_index().to_file(snakemake.output.country_shapes) offshore_shapes = eez( - country_shapes, snakemake.input.eez, snakemake.config["countries"] + country_shapes, snakemake.input.eez, snakemake.params["countries"] ) offshore_shapes.reset_index().to_file(snakemake.output.offshore_shapes) diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index f4eb1557..180007b7 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -27,9 +27,9 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - config = snakemake.config["solar_thermal"] + config = snakemake.params["solar_thermal"] - time = pd.date_range(freq="h", **snakemake.config["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params["snapshots"]) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index 8f6d6c6c..ee06aebb 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -27,7 +27,7 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - time = pd.date_range(freq="h", **snakemake.config["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params["snapshots"]) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index 6b8bd04f..dc2b94b9 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -175,9 +175,9 @@ if __name__ == "__main__": snakemake.input.pop_weighted_energy_totals, index_col=0 ) - options = snakemake.config["sector"] + options = snakemake.params["sector"] - snapshots = pd.date_range(freq="h", **snakemake.config["snapshots"], tz="UTC") + snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"], tz="UTC") nyears = len(snapshots) / 8760 diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index e7554dff..ba11ce1b 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -110,7 +110,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("cluster_gas_network", simpl="", clusters="37") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) fn = snakemake.input.cleaned_gas_network df = pd.read_csv(fn, index_col=0) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 7572d3b3..78798804 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -468,18 +468,18 @@ if __name__ == "__main__": [ tech for tech in n.generators.carrier.unique() - if tech in snakemake.config["renewable"] + if tech in snakemake.params["renewable"] ] ) - exclude_carriers = snakemake.config["clustering"]["cluster_network"].get( + exclude_carriers = snakemake.params["clustering"]["cluster_network"].get( "exclude_carriers", [] ) aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers) if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) conventional = set( - snakemake.config["electricity"].get("conventional_carriers", []) + snakemake.params["electricity"].get("conventional_carriers", []) ) aggregate_carriers = conventional.intersection(aggregate_carriers) elif snakemake.wildcards.clusters == "all": @@ -495,13 +495,13 @@ if __name__ == "__main__": n, busmap, linemap, linemap, pd.Series(dtype="O") ) else: - line_length_factor = snakemake.config["lines"]["length_factor"] + line_length_factor = snakemake.params["lines"]["length_factor"] Nyears = n.snapshot_weightings.objective.sum() / 8760 hvac_overhead_cost = load_costs( snakemake.input.tech_costs, - snakemake.config["costs"], - snakemake.config["electricity"], + snakemake.params["costs"], + snakemake.params["electricity"], Nyears, ).at["HVAC overhead", "capital_cost"] @@ -512,7 +512,7 @@ if __name__ == "__main__": ).all() or x.isnull().all(), "The `potential` configuration option must agree for all renewable carriers, for now!" return v - aggregation_strategies = snakemake.config["clustering"].get( + aggregation_strategies = snakemake.params["clustering"].get( "aggregation_strategies", {} ) # translate str entries of aggregation_strategies to pd.Series functions: @@ -521,7 +521,7 @@ if __name__ == "__main__": for p in aggregation_strategies.keys() } - custom_busmap = snakemake.config["enable"].get("custom_busmap", False) + custom_busmap = snakemake.params["enable"].get("custom_busmap", False) if custom_busmap: custom_busmap = pd.read_csv( snakemake.input.custom_busmap, index_col=0, squeeze=True @@ -539,7 +539,7 @@ if __name__ == "__main__": aggregate_carriers, line_length_factor, aggregation_strategies, - snakemake.config["solving"]["solver"]["name"], + snakemake.params["solving"]["solver"]["name"], cluster_config.get("algorithm", "hac"), cluster_config.get("feature", "solar+onwind-time"), hvac_overhead_cost, diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 3d743942..56074525 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -198,7 +198,7 @@ def calculate_costs(n, label, costs): def calculate_cumulative_cost(): - planning_horizons = snakemake.config["scenario"]["planning_horizons"] + planning_horizons = snakemake.params["scenario"]["planning_horizons"] cumulative_cost = pd.DataFrame( index=df["costs"].sum().index, @@ -682,25 +682,25 @@ if __name__ == "__main__": snakemake = mock_snakemake("make_summary") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) networks_dict = { (cluster, ll, opt + sector_opt, planning_horizon): "results/" + snakemake.params.RDIR + f"/postnetworks/elec_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc" - for simpl in snakemake.config["scenario"]["simpl"] - for cluster in snakemake.config["scenario"]["clusters"] - for opt in snakemake.config["scenario"]["opts"] - for sector_opt in snakemake.config["scenario"]["sector_opts"] - for ll in snakemake.config["scenario"]["ll"] - for planning_horizon in snakemake.config["scenario"]["planning_horizons"] + for simpl in snakemake.params["scenario"]["simpl"] + for cluster in snakemake.params["scenario"]["clusters"] + for opt in snakemake.params["scenario"]["opts"] + for sector_opt in snakemake.params["scenario"]["sector_opts"] + for ll in snakemake.params["scenario"]["ll"] + for planning_horizon in snakemake.params["scenario"]["planning_horizons"] } - Nyears = len(pd.date_range(freq="h", **snakemake.config["snapshots"])) / 8760 + Nyears = len(pd.date_range(freq="h", **snakemake.params["snapshots"])) / 8760 costs_db = prepare_costs( snakemake.input.costs, - snakemake.config["costs"], + snakemake.params["costs"], Nyears, ) @@ -710,7 +710,7 @@ if __name__ == "__main__": to_csv(df) - if snakemake.config["foresight"] == "myopic": + if snakemake.params["foresight"] == "myopic": cumulative_cost = calculate_cumulative_cost() cumulative_cost.to_csv( "results/" + snakemake.params.RDIR + "/csvs/cumulative_cost.csv" diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 0a22b2e5..8aac9db1 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -70,7 +70,7 @@ def plot_map( transmission=False, with_legend=True, ): - tech_colors = snakemake.config["plotting"]["tech_colors"] + tech_colors = snakemake.params["plotting"]["tech_colors"] n = network.copy() assign_location(n) @@ -116,7 +116,7 @@ def plot_map( costs = costs.stack() # .sort_index() # hack because impossible to drop buses... - eu_location = snakemake.config["plotting"].get( + eu_location = snakemake.params["plotting"].get( "eu_node_location", dict(x=-5.5, y=46) ) n.buses.loc["EU gas", "x"] = eu_location["x"] @@ -315,7 +315,7 @@ def plot_h2_map(network, regions): h2_new = n.links[n.links.carrier == "H2 pipeline"] h2_retro = n.links[n.links.carrier == "H2 pipeline retrofitted"] - if snakemake.config["foresight"] == "myopic": + if snakemake.params["foresight"] == "myopic": # sum capacitiy for pipelines from different investment periods h2_new = group_pipes(h2_new) @@ -558,7 +558,7 @@ def plot_ch4_map(network): link_widths_used = max_usage / linewidth_factor link_widths_used[max_usage < line_lower_threshold] = 0.0 - tech_colors = snakemake.config["plotting"]["tech_colors"] + tech_colors = snakemake.params["plotting"]["tech_colors"] pipe_colors = { "gas pipeline": "#f08080", @@ -700,7 +700,7 @@ def plot_map_without(network): # hack because impossible to drop buses... if "EU gas" in n.buses.index: - eu_location = snakemake.config["plotting"].get( + eu_location = snakemake.params["plotting"].get( "eu_node_location", dict(x=-5.5, y=46) ) n.buses.loc["EU gas", "x"] = eu_location["x"] @@ -876,7 +876,7 @@ def plot_series(network, carrier="AC", name="test"): stacked=True, linewidth=0.0, color=[ - snakemake.config["plotting"]["tech_colors"][i.replace(suffix, "")] + snakemake.params["plotting"]["tech_colors"][i.replace(suffix, "")] for i in new_columns ], ) @@ -930,14 +930,14 @@ if __name__ == "__main__": planning_horizons="2030", ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) regions = gpd.read_file(snakemake.input.regions).set_index("name") - map_opts = snakemake.config["plotting"]["map"] + map_opts = snakemake.params["plotting"]["map"] if map_opts["boundaries"] is None: map_opts["boundaries"] = regions.total_bounds[[0, 2, 1, 3]] + [-1, 1, -1, 1] diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index cfa8e361..e18ff61b 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -142,10 +142,10 @@ def plot_costs(): df = df.groupby(df.index.map(rename_techs)).sum() - to_drop = df.index[df.max(axis=1) < snakemake.config["plotting"]["costs_threshold"]] + to_drop = df.index[df.max(axis=1) < snakemake.params["plotting"]["costs_threshold"]] logger.info( - f"Dropping technology with costs below {snakemake.config['plotting']['costs_threshold']} EUR billion per year" + f"Dropping technology with costs below {snakemake.params['plotting']['costs_threshold']} EUR billion per year" ) logger.debug(df.loc[to_drop]) @@ -165,7 +165,7 @@ def plot_costs(): kind="bar", ax=ax, stacked=True, - color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -173,7 +173,7 @@ def plot_costs(): handles.reverse() labels.reverse() - ax.set_ylim([0, snakemake.config["plotting"]["costs_max"]]) + ax.set_ylim([0, snakemake.params["plotting"]["costs_max"]]) ax.set_ylabel("System Cost [EUR billion per year]") @@ -201,11 +201,11 @@ def plot_energy(): df = df.groupby(df.index.map(rename_techs)).sum() to_drop = df.index[ - df.abs().max(axis=1) < snakemake.config["plotting"]["energy_threshold"] + df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] ] logger.info( - f"Dropping all technology with energy consumption or production below {snakemake.config['plotting']['energy_threshold']} TWh/a" + f"Dropping all technology with energy consumption or production below {snakemake.params['plotting']['energy_threshold']} TWh/a" ) logger.debug(df.loc[to_drop]) @@ -227,7 +227,7 @@ def plot_energy(): kind="bar", ax=ax, stacked=True, - color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -237,8 +237,8 @@ def plot_energy(): ax.set_ylim( [ - snakemake.config["plotting"]["energy_min"], - snakemake.config["plotting"]["energy_max"], + snakemake.params["plotting"]["energy_min"], + snakemake.params["plotting"]["energy_max"], ] ) @@ -287,7 +287,7 @@ def plot_balances(): df = df.groupby(df.index.map(rename_techs)).sum() to_drop = df.index[ - df.abs().max(axis=1) < snakemake.config["plotting"]["energy_threshold"] / 10 + df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] / 10 ] if v[0] in co2_carriers: @@ -296,7 +296,7 @@ def plot_balances(): units = "TWh/a" logger.debug( - f"Dropping technology energy balance smaller than {snakemake.config['plotting']['energy_threshold']/10} {units}" + f"Dropping technology energy balance smaller than {snakemake.params['plotting']['energy_threshold']/10} {units}" ) logger.debug(df.loc[to_drop]) @@ -317,7 +317,7 @@ def plot_balances(): kind="bar", ax=ax, stacked=True, - color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -455,10 +455,10 @@ def plot_carbon_budget_distribution(input_eurostat): ax1 = plt.subplot(gs1[0, 0]) ax1.set_ylabel("CO$_2$ emissions (Gt per year)", fontsize=22) ax1.set_ylim([0, 5]) - ax1.set_xlim([1990, snakemake.config["scenario"]["planning_horizons"][-1] + 1]) + ax1.set_xlim([1990, snakemake.params["scenario"]["planning_horizons"][-1] + 1]) path_cb = "results/" + snakemake.params.RDIR + "/csvs/" - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990) CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0) @@ -545,7 +545,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("plot_summary") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) n_header = 4 @@ -555,7 +555,7 @@ if __name__ == "__main__": plot_balances() - for sector_opts in snakemake.config["scenario"]["sector_opts"]: + for sector_opts in snakemake.params["scenario"]["sector_opts"]: opts = sector_opts.split("-") for o in opts: if "cb" in o: diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 14a003ae..46e4e74a 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -253,12 +253,12 @@ if __name__ == "__main__": Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( snakemake.input.tech_costs, - snakemake.config["costs"], - snakemake.config["electricity"], + snakemake.params["costs"], + snakemake.params["electricity"], Nyears, ) - set_line_s_max_pu(n, snakemake.config["lines"]["s_max_pu"]) + set_line_s_max_pu(n, snakemake.params["lines"]["s_max_pu"]) for o in opts: m = re.match(r"^\d+h$", o, re.IGNORECASE) @@ -269,7 +269,7 @@ if __name__ == "__main__": for o in opts: m = re.match(r"^\d+seg$", o, re.IGNORECASE) if m is not None: - solver_name = snakemake.config["solving"]["solver"]["name"] + solver_name = snakemake.params["solving"]["solver"]["name"] n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) break @@ -277,11 +277,11 @@ if __name__ == "__main__": if "Co2L" in o: m = re.findall("[0-9]*\.?[0-9]+$", o) if len(m) > 0: - co2limit = float(m[0]) * snakemake.config["electricity"]["co2base"] + co2limit = float(m[0]) * snakemake.params["electricity"]["co2base"] add_co2limit(n, co2limit, Nyears) logger.info("Setting CO2 limit according to wildcard value.") else: - add_co2limit(n, snakemake.config["electricity"]["co2limit"], Nyears) + add_co2limit(n, snakemake.params["electricity"]["co2limit"], Nyears) logger.info("Setting CO2 limit according to config value.") break @@ -293,7 +293,7 @@ if __name__ == "__main__": add_gaslimit(n, limit, Nyears) logger.info("Setting gas usage limit according to wildcard value.") else: - add_gaslimit(n, snakemake.config["electricity"].get("gaslimit"), Nyears) + add_gaslimit(n, snakemake.params["electricity"].get("gaslimit"), Nyears) logger.info("Setting gas usage limit according to config value.") break @@ -322,7 +322,7 @@ if __name__ == "__main__": add_emission_prices(n, dict(co2=float(m[0]))) else: logger.info("Setting emission prices according to config value.") - add_emission_prices(n, snakemake.config["costs"]["emission_prices"]) + add_emission_prices(n, snakemake.params["costs"]["emission_prices"]) break ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:] @@ -330,8 +330,8 @@ if __name__ == "__main__": set_line_nom_max( n, - s_nom_max_set=snakemake.config["lines"].get("s_nom_max,", np.inf), - p_nom_max_set=snakemake.config["links"].get("p_nom_max,", np.inf), + s_nom_max_set=snakemake.params["lines"].get("s_nom_max,", np.inf), + p_nom_max_set=snakemake.params["links"].get("p_nom_max,", np.inf), ) if "ATK" in opts: diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 1cb7146e..c69f7290 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -200,12 +200,12 @@ def co2_emissions_year( """ Calculate CO2 emissions in one specific year (e.g. 1990 or 2018). """ - emissions_scope = snakemake.config["energy"]["emissions"] + emissions_scope = snakemake.params["energy"]["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope) # TODO: read Eurostat data from year > 2014 # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK - report_year = snakemake.config["energy"]["eurostat_report_year"] + report_year = snakemake.params["energy"]["eurostat_report_year"] if year > 2014: eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, year=2014 @@ -241,7 +241,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): carbon_budget = float(o[o.find("cb") + 2 : o.find("ex")]) r = float(o[o.find("ex") + 2 :]) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] e_1990 = co2_emissions_year( countries, input_eurostat, opts, emissions_scope, report_year, year=1990 @@ -252,7 +252,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries, input_eurostat, opts, emissions_scope, report_year, year=2018 ) - planning_horizons = snakemake.config["scenario"]["planning_horizons"] + planning_horizons = snakemake.params["scenario"]["planning_horizons"] t_0 = planning_horizons[0] if "be" in o: @@ -391,7 +391,7 @@ def update_wind_solar_costs(n, costs): with xr.open_dataset(profile) as ds: underwater_fraction = ds["underwater_fraction"].to_pandas() connection_cost = ( - snakemake.config["lines"]["length_factor"] + snakemake.params["lines"]["length_factor"] * ds["average_distance"].to_pandas() * ( underwater_fraction @@ -483,8 +483,8 @@ def remove_elec_base_techs(n): batteries and H2) from base electricity-only network, since they're added here differently using links. """ - for c in n.iterate_components(snakemake.config["pypsa_eur"]): - to_keep = snakemake.config["pypsa_eur"][c.name] + for c in n.iterate_components(snakemake.params["pypsa_eur"]): + to_keep = snakemake.params["pypsa_eur"][c.name] to_remove = pd.Index(c.df.carrier.unique()).symmetric_difference(to_keep) if to_remove.empty: continue @@ -674,7 +674,7 @@ def add_dac(n, costs): def add_co2limit(n, nyears=1.0, limit=0.0): logger.info(f"Adding CO2 budget limit as per unit of 1990 levels of {limit}") - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] sectors = emission_sectors_from_opts(opts) @@ -787,7 +787,7 @@ def add_ammonia(n, costs): nodes = pop_layout.index - cf_industry = snakemake.config["industry"] + cf_industry = snakemake.params["industry"] n.add("Carrier", "NH3") @@ -1102,7 +1102,7 @@ def add_storage_and_grids(n, costs): lifetime=costs.at["OCGT", "lifetime"], ) - cavern_types = snakemake.config["sector"]["hydrogen_underground_storage_locations"] + cavern_types = snakemake.params["sector"]["hydrogen_underground_storage_locations"] h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0) if not h2_caverns.empty and options["hydrogen_underground_storage"]: @@ -3266,11 +3266,11 @@ if __name__ == "__main__": planning_horizons="2030", ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) - options = snakemake.config["sector"] + options = snakemake.params["sector"] opts = snakemake.wildcards.sector_opts.split("-") @@ -3285,7 +3285,7 @@ if __name__ == "__main__": costs = prepare_costs( snakemake.input.costs, - snakemake.config["costs"], + snakemake.params["costs"], nyears, ) @@ -3297,10 +3297,10 @@ if __name__ == "__main__": spatial = define_spatial(pop_layout.index, options) - if snakemake.config["foresight"] == "myopic": + if snakemake.params["foresight"] == "myopic": add_lifetime_wind_solar(n, costs) - conventional = snakemake.config["existing_capacities"]["conventional_carriers"] + conventional = snakemake.params["existing_capacities"]["conventional_carriers"] for carrier in conventional: add_carrier_buses(n, carrier) @@ -3365,19 +3365,19 @@ if __name__ == "__main__": if options["allam_cycle"]: add_allam(n, costs) - solver_name = snakemake.config["solving"]["solver"]["name"] + solver_name = snakemake.params["solving"]["solver"]["name"] n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" - limit = get(snakemake.config["co2_budget"], investment_year) + limit = get(snakemake.params["co2_budget"], investment_year) for o in opts: if "cb" not in o: continue limit_type = "carbon budget" fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): - emissions_scope = snakemake.config["energy"]["emissions"] - report_year = snakemake.config["energy"]["eurostat_report_year"] + emissions_scope = snakemake.params["energy"]["emissions"] + report_year = snakemake.params["energy"]["eurostat_report_year"] build_carbon_budget( o, snakemake.input.eurostat, fn, emissions_scope, report_year ) @@ -3412,8 +3412,8 @@ if __name__ == "__main__": if options["electricity_grid_connection"]: add_electricity_grid_connection(n, costs) - first_year_myopic = (snakemake.config["foresight"] == "myopic") and ( - snakemake.config["scenario"]["planning_horizons"][0] == investment_year + first_year_myopic = (snakemake.params["foresight"] == "myopic") and ( + snakemake.params["scenario"]["planning_horizons"][0] == investment_year ) if options.get("cluster_heat_buses", False) and not first_year_myopic: diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 0c6a7feb..0271166c 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -53,7 +53,7 @@ if __name__ == "__main__": snakemake ) # TODO Make logging compatible with progressbar (see PR #102) - if snakemake.config["tutorial"]: + if snakemake.params["tutorial"]: url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz" else: url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz" @@ -63,7 +63,7 @@ if __name__ == "__main__": to_fn = Path(f"{rootpath}/data") logger.info(f"Downloading databundle from '{url}'.") - disable_progress = snakemake.config["run"].get("disable_progressbar", False) + disable_progress = snakemake.params["run"].get("disable_progressbar", False) progress_retrieve(url, tarball_fn, disable=disable_progress) logger.info("Extracting databundle.") diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 2be8c36a..bb7f615c 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -510,7 +510,7 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.network) - aggregation_strategies = snakemake.config["clustering"].get( + aggregation_strategies = snakemake.params["clustering"].get( "aggregation_strategies", {} ) # translate str entries of aggregation_strategies to pd.Series functions: @@ -525,8 +525,8 @@ if __name__ == "__main__": technology_costs = load_costs( snakemake.input.tech_costs, - snakemake.config["costs"], - snakemake.config["electricity"], + snakemake.params["costs"], + snakemake.params["electricity"], Nyears, ) @@ -536,7 +536,7 @@ if __name__ == "__main__": busmaps = [trafo_map, simplify_links_map] - cluster_config = snakemake.config["clustering"]["simplify_network"] + cluster_config = snakemake.params["clustering"]["simplify_network"] if cluster_config.get("remove_stubs", True): n, stub_map = remove_stubs( n, diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ff1c0ccf..c7041e85 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -663,7 +663,7 @@ if __name__ == "__main__": if "sector_opts" in snakemake.wildcards.keys(): opts += "-" + snakemake.wildcards.sector_opts opts = [o for o in opts.split("-") if o != ""] - solve_opts = snakemake.config["solving"]["options"] + solve_opts = snakemake.params["solving"]["options"] np.random.seed(solve_opts.get("seed", 123)) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 25fe0753..c1b2be6f 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -42,7 +42,7 @@ if __name__ == "__main__": opts = (snakemake.wildcards.opts + "-" + snakemake.wildcards.sector_opts).split("-") opts = [o for o in opts if o != ""] - solve_opts = snakemake.config["solving"]["options"] + solve_opts = snakemake.params["solving"]["options"] np.random.seed(solve_opts.get("seed", 123)) From 61893c3c9b9e9f738d795f31ab03be07f6558ebd Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Mon, 15 May 2023 11:26:36 +0200 Subject: [PATCH 45/78] fixing build_electricity.smk and retrieve.smk --- rules/build_electricity.smk | 1 + rules/retrieve.smk | 1 + 2 files changed, 2 insertions(+) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index bb000f2b..56098ddd 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -201,6 +201,7 @@ rule build_ship_raster: rule build_renewable_profiles: params: + run=config["run"], renewable=config["renewable"], input: base_network=RESOURCES + "networks/base.nc", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 4bfbd6c6..bc8756d5 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -20,6 +20,7 @@ if config["enable"].get("retrieve_databundle", True): rule retrieve_databundle: params: + run=config["run"], tutorial=config["tutorial"], output: expand("data/bundle/{file}", file=datafiles), From 748cfaec65aeaa1b4a671ac854aaa46cfd0941e7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 22:01:44 +0000 Subject: [PATCH 46/78] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/macisamuele/language-formatters-pre-commit-hooks: v2.8.0 → v2.9.0](https://github.com/macisamuele/language-formatters-pre-commit-hooks/compare/v2.8.0...v2.9.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 88685f3b..48915f4c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,7 +67,7 @@ repos: # Do YAML formatting (before the linter checks it for misses) - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.8.0 + rev: v2.9.0 hooks: - id: pretty-format-yaml args: [--autofix, --indent, "2", --preserve-quotes] From 875fcdd0e2ee1ef85e9d769446f1ebf7e836b4e0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 16 May 2023 13:34:23 +0200 Subject: [PATCH 47/78] address geopandas future warning --- scripts/build_shapes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 50d21e12..54a7774e 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -234,6 +234,7 @@ def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp): manual = gpd.GeoDataFrame( [["BA1", "BA", 3871.0], ["RS1", "RS", 7210.0], ["AL1", "AL", 2893.0]], columns=["NUTS_ID", "country", "pop"], + geometry=gpd.GeoSeries() ) manual["geometry"] = manual["country"].map(country_shapes) manual = manual.dropna() From 7d4dacf8bdeb24513888719065069102152b9be4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 May 2023 11:34:58 +0000 Subject: [PATCH 48/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_shapes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 54a7774e..5561cfb2 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -234,7 +234,7 @@ def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp): manual = gpd.GeoDataFrame( [["BA1", "BA", 3871.0], ["RS1", "RS", 7210.0], ["AL1", "AL", 2893.0]], columns=["NUTS_ID", "country", "pop"], - geometry=gpd.GeoSeries() + geometry=gpd.GeoSeries(), ) manual["geometry"] = manual["country"].map(country_shapes) manual = manual.dropna() From 7fad9e263569db7eaf392c56e47e01c552343342 Mon Sep 17 00:00:00 2001 From: Lissy Langer <54096244+lilanger@users.noreply.github.com> Date: Tue, 16 May 2023 14:57:45 +0200 Subject: [PATCH 49/78] Update supply_demand.rst Heat pump COP equations signs are wrong: https://pubs.rsc.org/en/content/articlelanding/2012/EE/c2ee22653g --- doc/supply_demand.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/supply_demand.rst b/doc/supply_demand.rst index 16242405..b043268b 100644 --- a/doc/supply_demand.rst +++ b/doc/supply_demand.rst @@ -133,12 +133,12 @@ The coefficient of performance (COP) of air- and ground-sourced heat pumps depen For the sink water temperature Tsink we assume 55 °C [`Config `_ file]. For the time- and location-dependent source temperatures Tsource, we rely on the `ERA5 `_ reanalysis weather data. The temperature differences are converted into COP time series using results from a regression analysis performed in the study by `Stafell et al. `_. For air-sourced heat pumps (ASHP), we use the function: .. math:: - COP (\Delta T) = 6.81 + 0.121\Delta T + 0.000630\Delta T^2 + COP (\Delta T) = 6.81 - 0.121\Delta T + 0.000630\Delta T^2 for ground-sourced heat pumps (GSHP), we use the function: .. math:: - COP(\Delta T) = 8.77 + 0.150\Delta T + 0.000734\Delta T^2 + COP(\Delta T) = 8.77 - 0.150\Delta T + 0.000734\Delta T^2 **Resistive heaters** From 8d6d6610712e1f5a2ce3bfab13a1a23afec21029 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Tue, 16 May 2023 20:44:22 +0100 Subject: [PATCH 50/78] Update support.rst --- doc/support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/support.rst b/doc/support.rst index 36d1a2dd..1d512d59 100644 --- a/doc/support.rst +++ b/doc/support.rst @@ -9,6 +9,6 @@ Support * In case of code-related **questions**, please post on `stack overflow `_. * For non-programming related and more general questions please refer to the `mailing list `_. -* To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb). +* To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the `discord server `_. * For **bugs and feature requests**, please use the `issue tracker `_. * We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to :ref:`contributing`. From 8208ac033baff982fbeedb11ef019d4bd581576c Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 17 May 2023 18:43:30 +0200 Subject: [PATCH 51/78] convert param back to config for "logging" and "run", made "enable" more specific --- rules/build_electricity.smk | 3 +-- rules/build_sector.smk | 15 --------------- rules/postprocess.smk | 3 --- rules/retrieve.smk | 1 - rules/solve_myopic.smk | 2 -- scripts/add_brownfield.py | 2 +- scripts/add_existing_baseyear.py | 2 +- scripts/build_energy_totals.py | 4 ++-- scripts/build_gas_input_locations.py | 2 +- scripts/build_gas_network.py | 2 +- scripts/build_industrial_distribution_key.py | 2 +- ..._industrial_energy_demand_per_country_today.py | 2 +- .../build_industrial_production_per_country.py | 4 ++-- scripts/build_population_layouts.py | 2 +- scripts/build_renewable_profiles.py | 2 +- scripts/cluster_gas_network.py | 2 +- scripts/make_summary.py | 2 +- scripts/plot_network.py | 2 +- scripts/plot_summary.py | 2 +- scripts/prepare_sector_network.py | 2 +- scripts/retrieve_databundle.py | 2 +- 21 files changed, 19 insertions(+), 41 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 56098ddd..9414c25b 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -201,7 +201,6 @@ rule build_ship_raster: rule build_renewable_profiles: params: - run=config["run"], renewable=config["renewable"], input: base_network=RESOURCES + "networks/base.nc", @@ -350,7 +349,7 @@ rule cluster_network: lines=config["lines"], renewable=config["renewable"], clustering=config["clustering"], - enable=config["enable"], + enable=config["enable"].get("custom_busmap", False), input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index d375b7b9..9faae4b9 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -4,8 +4,6 @@ rule build_population_layouts: - params: - logging=config["logging"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", urban_percent="data/urban_percent.csv", @@ -72,8 +70,6 @@ rule build_simplified_population_layouts: if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule build_gas_network: - params: - logging=config["logging"], input: gas_network="data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson", output: @@ -88,8 +84,6 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/build_gas_network.py" rule build_gas_input_locations: - params: - logging=config["logging"], input: lng=HTTP.remote( "https://globalenergymonitor.org/wp-content/uploads/2022/09/Europe-Gas-Tracker-August-2022.xlsx", @@ -116,8 +110,6 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/build_gas_input_locations.py" rule cluster_gas_network: - params: - logging=config["logging"], input: cleaned_gas_network=RESOURCES + "gas_network.csv", regions_onshore=RESOURCES @@ -246,10 +238,8 @@ rule build_solar_thermal_profiles: rule build_energy_totals: params: - run=config["run"], countries=config["countries"], energy=config["energy"], - logging=config["logging"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", co2="data/eea/UNFCCC_v23.csv", @@ -437,10 +427,8 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: params: - run=config["run"], industry=config["industry"], countries=config["countries"], - logging=config["logging"], input: ammonia_production=RESOURCES + "ammonia_production.csv", jrc="data/jrc-idees-2015", @@ -491,7 +479,6 @@ rule build_industrial_distribution_key: params: industry=config["industry"], countries=config["countries"], - logging=config["logging"], input: regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", @@ -567,7 +554,6 @@ rule build_industrial_energy_demand_per_node: rule build_industrial_energy_demand_per_country_today: params: - run=config["run"], countries=config["countries"], industry=config["industry"], input: @@ -723,7 +709,6 @@ rule prepare_sector_network: existing_capacities=config["existing_capacities"], foresight=config["foresight"], costs=config["costs"], - logging=config["logging"], sector=config["sector"], industry=config["industry"], pypsa_eur=config["pypsa_eur"], diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 1dfdd098..9eb04ef5 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -10,7 +10,6 @@ localrules: rule plot_network: params: - logging=config["logging"], foresight=config["foresight"], plotting=config["plotting"], input: @@ -74,7 +73,6 @@ rule make_summary: foresight=config["foresight"], costs=config["costs"], snapshots=config["snapshots"], - logging=config["logging"], scenario=config["scenario"], RDIR=RDIR, input: @@ -123,7 +121,6 @@ rule make_summary: rule plot_summary: params: - logging=config["logging"], countries=config["countries"], scenario=config["scenario"], plotting=config["plotting"], diff --git a/rules/retrieve.smk b/rules/retrieve.smk index bc8756d5..4bfbd6c6 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -20,7 +20,6 @@ if config["enable"].get("retrieve_databundle", True): rule retrieve_databundle: params: - run=config["run"], tutorial=config["tutorial"], output: expand("data/bundle/{file}", file=datafiles), diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 2ecba999..49b93a80 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -7,7 +7,6 @@ rule add_existing_baseyear: params: scenario=config["scenario"], sector=config["sector"], - logging=config["logging"], existing_capacities=config["existing_capacities"], costs=config["costs"], input: @@ -49,7 +48,6 @@ rule add_existing_baseyear: rule add_brownfield: params: - logging=config["logging"], sector=config["sector"], existing_capacities=config["existing_capacities"], input: diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 1d4e3a80..9330953b 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -139,7 +139,7 @@ if __name__ == "__main__": planning_horizons=2030, ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index a24b078d..a4d3748b 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -608,7 +608,7 @@ if __name__ == "__main__": planning_horizons=2020, ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 6cedff97..5099d140 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -373,7 +373,7 @@ def idees_per_country(ct, year, base_dir): def build_idees(countries, year): nprocesses = snakemake.threads - disable_progress = snakemake.params["run"].get("disable_progressbar", False) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) func = partial(idees_per_country, year=year, base_dir=snakemake.input.idees) tqdm_kwargs = dict( @@ -735,7 +735,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_energy_totals") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) config = snakemake.params["energy"] diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index 59f31f5c..a3b945ab 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -86,7 +86,7 @@ if __name__ == "__main__": clusters="37", ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) regions = load_bus_regions( snakemake.input.regions_onshore, snakemake.input.regions_offshore diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py index a70f15ff..23f58caa 100644 --- a/scripts/build_gas_network.py +++ b/scripts/build_gas_network.py @@ -147,7 +147,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_gas_network") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) gas_network = load_dataset(snakemake.input.gas_network) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 30c652d8..9a513673 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -141,7 +141,7 @@ if __name__ == "__main__": clusters=48, ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) countries = snakemake.params["countries"] diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 4fec95cc..c28351c1 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -153,7 +153,7 @@ def add_non_eu28_industrial_energy_demand(countries, demand): def industrial_energy_demand(countries, year): nprocesses = snakemake.threads - disable_progress = snakemake.params["run"].get("disable_progressbar", False) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) func = partial( industrial_energy_demand_per_country, year=year, jrc_dir=snakemake.input.jrc ) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index eb1b16cb..62073ea1 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -217,7 +217,7 @@ def industry_production_per_country(country, year, eurostat_dir, jrc_dir): def industry_production(countries, year, eurostat_dir, jrc_dir): nprocesses = snakemake.threads - disable_progress = snakemake.params["run"].get("disable_progressbar", False) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) func = partial( industry_production_per_country, @@ -277,7 +277,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_production_per_country") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) countries = snakemake.params["countries"] diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index bc35dbcf..e864d925 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -23,7 +23,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_population_layouts") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) cutout = atlite.Cutout(snakemake.input.cutout) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 55541522..8fba74e6 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -203,7 +203,7 @@ if __name__ == "__main__": configure_logging(snakemake) nprocesses = int(snakemake.threads) - noprogress = snakemake.params["run"].get("disable_progressbar", True) + noprogress = snakemake.config["run"].get("disable_progressbar", True) config = snakemake.params["renewable"][snakemake.wildcards.technology] resource = config["resource"] # pv panel config / wind turbine config correction_factor = config.get("correction_factor", 1.0) diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index ba11ce1b..e7554dff 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -110,7 +110,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("cluster_gas_network", simpl="", clusters="37") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) fn = snakemake.input.cleaned_gas_network df = pd.read_csv(fn, index_col=0) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 56074525..da0712d7 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -682,7 +682,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("make_summary") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) networks_dict = { (cluster, ll, opt + sector_opt, planning_horizon): "results/" diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 8aac9db1..399f46e8 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -930,7 +930,7 @@ if __name__ == "__main__": planning_horizons="2030", ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index e18ff61b..4aa37de5 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -545,7 +545,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("plot_summary") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) n_header = 4 diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index c69f7290..86a9bc82 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3266,7 +3266,7 @@ if __name__ == "__main__": planning_horizons="2030", ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 0271166c..de42587d 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -63,7 +63,7 @@ if __name__ == "__main__": to_fn = Path(f"{rootpath}/data") logger.info(f"Downloading databundle from '{url}'.") - disable_progress = snakemake.params["run"].get("disable_progressbar", False) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) progress_retrieve(url, tarball_fn, disable=disable_progress) logger.info("Extracting databundle.") From 8af1fe5649feb6df991db0d5b991c66fc9e8c25e Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 17 May 2023 19:25:45 +0200 Subject: [PATCH 52/78] replace the word "config" to "params" in functions --- rules/build_electricity.smk | 1 + scripts/add_electricity.py | 47 +++++++++-------- scripts/build_biomass_potentials.py | 8 +-- scripts/build_energy_totals.py | 6 +-- scripts/build_hydro_profile.py | 6 +-- ...ustrial_energy_demand_per_country_today.py | 10 ++-- ...build_industrial_production_per_country.py | 8 +-- ...ustrial_production_per_country_tomorrow.py | 14 +++--- scripts/build_industry_sector_ratios.py | 50 +++++++++---------- scripts/build_renewable_profiles.py | 48 +++++++++--------- scripts/prepare_sector_network.py | 4 +- 11 files changed, 101 insertions(+), 101 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 9414c25b..88e7c548 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -277,6 +277,7 @@ rule add_electricity: countries=config["countries"], renewable=config["renewable"], electricity=config["electricity"], + conventional=config.get("conventional", {}) costs=config["costs"], input: **{ diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index c3e847ce..a3e033e7 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -135,7 +135,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers): n.import_components_from_dataframe(emissions, "Carrier") -def load_costs(tech_costs, config, elec_config, Nyears=1.0): +def load_costs(tech_costs, params, elec_params, Nyears=1.0): # set all asset costs and other parameters costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index() @@ -143,7 +143,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0): costs.loc[costs.unit.str.contains("/kW"), "value"] *= 1e3 costs.unit = costs.unit.str.replace("/kW", "/MW") - fill_values = config["fill_values"] + fill_values = params["fill_values"] costs = costs.value.unstack().fillna(fill_values) costs["capital_cost"] = ( @@ -166,8 +166,8 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0): costs.at["CCGT", "co2_emissions"] = costs.at["gas", "co2_emissions"] costs.at["solar", "capital_cost"] = ( - config["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] - + (1 - config["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] + params["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] + + (1 - params["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] ) def costs_for_storage(store, link1, link2=None, max_hours=1.0): @@ -178,7 +178,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0): dict(capital_cost=capital_cost, marginal_cost=0.0, co2_emissions=0.0) ) - max_hours = elec_config["max_hours"] + max_hours = elec_params["max_hours"] costs.loc["battery"] = costs_for_storage( costs.loc["battery storage"], costs.loc["battery inverter"], @@ -192,7 +192,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0): ) for attr in ("marginal_cost", "capital_cost"): - overwrites = config.get(attr) + overwrites = params.get(attr) if overwrites is not None: overwrites = pd.Series(overwrites) costs.loc[overwrites.index, attr] = overwrites @@ -356,7 +356,7 @@ def attach_conventional_generators( ppl, conventional_carriers, extendable_carriers, - conventional_config, + conventional_params, conventional_inputs, ): carriers = set(conventional_carriers) | set(extendable_carriers["Generator"]) @@ -393,12 +393,12 @@ def attach_conventional_generators( lifetime=(ppl.dateout - ppl.datein).fillna(np.inf), ) - for carrier in conventional_config: + for carrier in conventional_params: # Generators with technology affected idx = n.generators.query("carrier == @carrier").index - for attr in list(set(conventional_config[carrier]) & set(n.generators)): - values = conventional_config[carrier][attr] + for attr in list(set(conventional_params[carrier]) & set(n.generators)): + values = conventional_params[carrier][attr] if f"conventional_{carrier}_{attr}" in conventional_inputs: # Values affecting generators of technology k country-specific @@ -413,7 +413,7 @@ def attach_conventional_generators( n.generators.loc[idx, attr] = values -def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **config): +def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **params): _add_missing_carriers_from_costs(n, costs, carriers) ppl = ( @@ -468,9 +468,9 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **con ) if "PHS" in carriers and not phs.empty: - # fill missing max hours to config value and + # fill missing max hours to params value and # assume no natural inflow due to lack of data - max_hours = config.get("PHS_max_hours", 6) + max_hours = params.get("PHS_max_hours", 6) phs = phs.replace({"max_hours": {0: max_hours}}) n.madd( "StorageUnit", @@ -486,7 +486,7 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **con ) if "hydro" in carriers and not hydro.empty: - hydro_max_hours = config.get("hydro_max_hours") + hydro_max_hours = params.get("hydro_max_hours") assert hydro_max_hours is not None, "No path for hydro capacities given." @@ -636,13 +636,12 @@ def attach_OPSD_renewables(n, tech_map): n.generators.p_nom_min.update(gens.bus.map(caps).dropna()) -def estimate_renewable_capacities(n, config): - year = config["electricity"]["estimate_renewable_capacities"]["year"] - tech_map = config["electricity"]["estimate_renewable_capacities"][ +def estimate_renewable_capacities(n, electricity_params, countries): + year = electricity_params["estimate_renewable_capacities"]["year"] + tech_map = electricity_params["estimate_renewable_capacities"][ "technology_mapping" ] - countries = config["countries"] - expansion_limit = config["electricity"]["estimate_renewable_capacities"][ + expansion_limit = electricity_params["estimate_renewable_capacities"][ "expansion_limit" ] @@ -759,7 +758,7 @@ if __name__ == "__main__": ppl, conventional_carriers, extendable_carriers, - snakemake.config.get("conventional", {}), + snakemake.params.get("conventional", {}), conventional_inputs, ) @@ -773,15 +772,15 @@ if __name__ == "__main__": ) if "hydro" in renewable_carriers: - conf = snakemake.params["renewable"]["hydro"] + para = snakemake.params["renewable"]["hydro"] attach_hydro( n, costs, ppl, snakemake.input.profile_hydro, snakemake.input.hydro_capacities, - conf.pop("carriers", []), - **conf, + para.pop("carriers", []), + **para, ) if "estimate_renewable_capacities" not in snakemake.params["electricity"]: @@ -829,7 +828,7 @@ if __name__ == "__main__": "technology_mapping" ] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.config) + estimate_renewable_capacities(n, snakemake.params["electricity"],snakemake.params["countries"]) update_p_nom_max(n) diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 35218e2c..0b423ad5 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -210,9 +210,9 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_biomass_potentials", simpl="", clusters="5") - config = snakemake.params["biomass"] - year = config["year"] - scenario = config["scenario"] + params = snakemake.params["biomass"] + year = params["year"] + scenario = params["scenario"] enspreso = enspreso_biomass_potentials(year, scenario) @@ -228,7 +228,7 @@ if __name__ == "__main__": df.to_csv(snakemake.output.biomass_potentials_all) - grouper = {v: k for k, vv in config["classes"].items() for v in vv} + grouper = {v: k for k, vv in params["classes"].items() for v in vv} df = df.groupby(grouper, axis=1).sum() df *= 1e6 # TWh/a to MWh/a diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 5099d140..3e3cb485 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -737,7 +737,7 @@ if __name__ == "__main__": logging.basicConfig(level=snakemake.config["logging"]["level"]) - config = snakemake.params["energy"] + params = snakemake.params["energy"] nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index") population = nuts3["pop"].groupby(nuts3.country).sum() @@ -745,7 +745,7 @@ if __name__ == "__main__": countries = snakemake.params["countries"] idees_countries = pd.Index(countries).intersection(eu28) - data_year = config["energy_totals_year"] + data_year = params["energy_totals_year"] report_year = snakemake.params["energy"]["eurostat_report_year"] input_eurostat = snakemake.input.eurostat eurostat = build_eurostat(input_eurostat, countries, report_year, data_year) @@ -755,7 +755,7 @@ if __name__ == "__main__": energy = build_energy_totals(countries, eurostat, swiss, idees) energy.to_csv(snakemake.output.energy_name) - base_year_emissions = config["base_emissions_year"] + base_year_emissions = params["base_emissions_year"] emissions_scope = snakemake.params["energy"]["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope) eurostat_co2 = build_eurostat_co2( diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 5453ac5c..dd686be3 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -130,7 +130,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) - config_hydro = snakemake.params["renewable"]["hydro"] + params_hydro = snakemake.params["renewable"]["hydro"] cutout = atlite.Cutout(snakemake.input.cutout) countries = snakemake.params["countries"] @@ -151,7 +151,7 @@ if __name__ == "__main__": normalize_using_yearly=eia_stats, ) - if "clip_min_inflow" in config_hydro: - inflow = inflow.where(inflow > config_hydro["clip_min_inflow"], 0) + if "clip_min_inflow" in params_hydro: + inflow = inflow.where(inflow > params_hydro["clip_min_inflow"], 0) inflow.to_netcdf(snakemake.output[0]) diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index c28351c1..9f8c47d0 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -101,8 +101,8 @@ def add_ammonia_energy_demand(demand): def get_ammonia_by_fuel(x): fuels = { - "gas": config["MWh_CH4_per_tNH3_SMR"], - "electricity": config["MWh_elec_per_tNH3_SMR"], + "gas": params["MWh_CH4_per_tNH3_SMR"], + "electricity": params["MWh_elec_per_tNH3_SMR"], } return pd.Series({k: x * v for k, v in fuels.items()}) @@ -112,7 +112,7 @@ def add_ammonia_energy_demand(demand): index=demand.index, fill_value=0.0 ) - ammonia = pd.DataFrame({"ammonia": ammonia * config["MWh_NH3_per_tNH3"]}).T + ammonia = pd.DataFrame({"ammonia": ammonia * params["MWh_NH3_per_tNH3"]}).T demand["Ammonia"] = ammonia.unstack().reindex(index=demand.index, fill_value=0.0) @@ -178,8 +178,8 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today") - config = snakemake.params["industry"] - year = config.get("reference_year", 2015) + params = snakemake.params["industry"] + year = params.get("reference_year", 2015) countries = pd.Index(snakemake.params["countries"]) demand = industrial_energy_demand(countries.intersection(eu28), year) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 62073ea1..889c9ecd 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -264,9 +264,9 @@ def separate_basic_chemicals(demand, year): # assume HVC, methanol, chlorine production proportional to non-ammonia basic chemicals distribution_key = demand["Basic chemicals"] / demand["Basic chemicals"].sum() - demand["HVC"] = config["HVC_production_today"] * 1e3 * distribution_key - demand["Chlorine"] = config["chlorine_production_today"] * 1e3 * distribution_key - demand["Methanol"] = config["methanol_production_today"] * 1e3 * distribution_key + demand["HVC"] = params["HVC_production_today"] * 1e3 * distribution_key + demand["Chlorine"] = params["chlorine_production_today"] * 1e3 * distribution_key + demand["Methanol"] = params["methanol_production_today"] * 1e3 * distribution_key demand.drop(columns=["Basic chemicals"], inplace=True) @@ -283,7 +283,7 @@ if __name__ == "__main__": year = snakemake.params["industry"]["reference_year"] - config = snakemake.params["industry"] + params = snakemake.params["industry"] jrc_dir = snakemake.input.jrc eurostat_dir = snakemake.input.eurostat diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py index b9ac9b16..609170aa 100644 --- a/scripts/build_industrial_production_per_country_tomorrow.py +++ b/scripts/build_industrial_production_per_country_tomorrow.py @@ -15,7 +15,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow") - config = snakemake.params["industry"] + params = snakemake.params["industry"] investment_year = int(snakemake.wildcards.planning_horizons) @@ -25,8 +25,8 @@ if __name__ == "__main__": keys = ["Integrated steelworks", "Electric arc"] total_steel = production[keys].sum(axis=1) - st_primary_fraction = get(config["St_primary_fraction"], investment_year) - dri_fraction = get(config["DRI_fraction"], investment_year) + st_primary_fraction = get(params["St_primary_fraction"], investment_year) + dri_fraction = get(params["DRI_fraction"], investment_year) int_steel = production["Integrated steelworks"].sum() fraction_persistent_primary = st_primary_fraction * total_steel.sum() / int_steel @@ -51,7 +51,7 @@ if __name__ == "__main__": key_pri = "Aluminium - primary production" key_sec = "Aluminium - secondary production" - al_primary_fraction = get(config["Al_primary_fraction"], investment_year) + al_primary_fraction = get(params["Al_primary_fraction"], investment_year) fraction_persistent_primary = ( al_primary_fraction * total_aluminium.sum() / production[key_pri].sum() ) @@ -60,15 +60,15 @@ if __name__ == "__main__": production[key_sec] = total_aluminium - production[key_pri] production["HVC (mechanical recycling)"] = ( - get(config["HVC_mechanical_recycling_fraction"], investment_year) + get(params["HVC_mechanical_recycling_fraction"], investment_year) * production["HVC"] ) production["HVC (chemical recycling)"] = ( - get(config["HVC_chemical_recycling_fraction"], investment_year) + get(params["HVC_chemical_recycling_fraction"], investment_year) * production["HVC"] ) - production["HVC"] *= get(config["HVC_primary_fraction"], investment_year) + production["HVC"] *= get(params["HVC_primary_fraction"], investment_year) fn = snakemake.output.industrial_production_per_country_tomorrow production.to_csv(fn, float_format="%.2f") diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index c3e2bd49..2ec007a9 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -185,10 +185,10 @@ def iron_and_steel(): df[sector] = df["Electric arc"] # add H2 consumption for DRI at 1.7 MWh H2 /ton steel - df.at["hydrogen", sector] = config["H2_DRI"] + df.at["hydrogen", sector] = params["H2_DRI"] # add electricity consumption in DRI shaft (0.322 MWh/tSl) - df.at["elec", sector] += config["elec_DRI"] + df.at["elec", sector] += params["elec_DRI"] ## Integrated steelworks # could be used in combination with CCS) @@ -383,19 +383,19 @@ def chemicals_industry(): assert s_emi.index[0] == sector # convert from MtHVC/a to ktHVC/a - s_out = config["HVC_production_today"] * 1e3 + s_out = params["HVC_production_today"] * 1e3 # tCO2/t material df.loc["process emission", sector] += ( s_emi["Process emissions"] - - config["petrochemical_process_emissions"] * 1e3 - - config["NH3_process_emissions"] * 1e3 + - params["petrochemical_process_emissions"] * 1e3 + - params["NH3_process_emissions"] * 1e3 ) / s_out # emissions originating from feedstock, could be non-fossil origin # tCO2/t material df.loc["process emission from feedstock", sector] += ( - config["petrochemical_process_emissions"] * 1e3 + params["petrochemical_process_emissions"] * 1e3 ) / s_out # convert from ktoe/a to GWh/a @@ -405,18 +405,18 @@ def chemicals_industry(): # subtract ammonia energy demand (in ktNH3/a) ammonia = pd.read_csv(snakemake.input.ammonia_production, index_col=0) ammonia_total = ammonia.loc[ammonia.index.intersection(eu28), str(year)].sum() - df.loc["methane", sector] -= ammonia_total * config["MWh_CH4_per_tNH3_SMR"] - df.loc["elec", sector] -= ammonia_total * config["MWh_elec_per_tNH3_SMR"] + df.loc["methane", sector] -= ammonia_total * params["MWh_CH4_per_tNH3_SMR"] + df.loc["elec", sector] -= ammonia_total * params["MWh_elec_per_tNH3_SMR"] # subtract chlorine demand - chlorine_total = config["chlorine_production_today"] - df.loc["hydrogen", sector] -= chlorine_total * config["MWh_H2_per_tCl"] - df.loc["elec", sector] -= chlorine_total * config["MWh_elec_per_tCl"] + chlorine_total = params["chlorine_production_today"] + df.loc["hydrogen", sector] -= chlorine_total * params["MWh_H2_per_tCl"] + df.loc["elec", sector] -= chlorine_total * params["MWh_elec_per_tCl"] # subtract methanol demand - methanol_total = config["methanol_production_today"] - df.loc["methane", sector] -= methanol_total * config["MWh_CH4_per_tMeOH"] - df.loc["elec", sector] -= methanol_total * config["MWh_elec_per_tMeOH"] + methanol_total = params["methanol_production_today"] + df.loc["methane", sector] -= methanol_total * params["MWh_CH4_per_tMeOH"] + df.loc["elec", sector] -= methanol_total * params["MWh_elec_per_tMeOH"] # MWh/t material df.loc[sources, sector] = df.loc[sources, sector] / s_out @@ -427,37 +427,37 @@ def chemicals_industry(): sector = "HVC (mechanical recycling)" df[sector] = 0.0 - df.loc["elec", sector] = config["MWh_elec_per_tHVC_mechanical_recycling"] + df.loc["elec", sector] = params["MWh_elec_per_tHVC_mechanical_recycling"] # HVC chemical recycling sector = "HVC (chemical recycling)" df[sector] = 0.0 - df.loc["elec", sector] = config["MWh_elec_per_tHVC_chemical_recycling"] + df.loc["elec", sector] = params["MWh_elec_per_tHVC_chemical_recycling"] # Ammonia sector = "Ammonia" df[sector] = 0.0 if snakemake.params["sector"].get("ammonia", False): - df.loc["ammonia", sector] = config["MWh_NH3_per_tNH3"] + df.loc["ammonia", sector] = params["MWh_NH3_per_tNH3"] else: - df.loc["hydrogen", sector] = config["MWh_H2_per_tNH3_electrolysis"] - df.loc["elec", sector] = config["MWh_elec_per_tNH3_electrolysis"] + df.loc["hydrogen", sector] = params["MWh_H2_per_tNH3_electrolysis"] + df.loc["elec", sector] = params["MWh_elec_per_tNH3_electrolysis"] # Chlorine sector = "Chlorine" df[sector] = 0.0 - df.loc["hydrogen", sector] = config["MWh_H2_per_tCl"] - df.loc["elec", sector] = config["MWh_elec_per_tCl"] + df.loc["hydrogen", sector] = params["MWh_H2_per_tCl"] + df.loc["elec", sector] = params["MWh_elec_per_tCl"] # Methanol sector = "Methanol" df[sector] = 0.0 - df.loc["methane", sector] = config["MWh_CH4_per_tMeOH"] - df.loc["elec", sector] = config["MWh_elec_per_tMeOH"] + df.loc["methane", sector] = params["MWh_CH4_per_tMeOH"] + df.loc["elec", sector] = params["MWh_elec_per_tMeOH"] # Other chemicals @@ -1465,10 +1465,10 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industry_sector_ratios") - # TODO make config option + # TODO make params option year = 2015 - config = snakemake.params["industry"] + params = snakemake.params["industry"] df = pd.concat( [ diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 8fba74e6..91463c23 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -64,7 +64,7 @@ Inputs - ``resources/offshore_shapes.geojson``: confer :ref:`shapes` - ``resources/regions_onshore.geojson``: (if not offshore wind), confer :ref:`busregions` - ``resources/regions_offshore.geojson``: (if offshore wind), :ref:`busregions` -- ``"cutouts/" + config["renewable"][{technology}]['cutout']``: :ref:`cutout` +- ``"cutouts/" + params["renewable"][{technology}]['cutout']``: :ref:`cutout` - ``networks/base.nc``: :ref:`base` Outputs @@ -204,14 +204,14 @@ if __name__ == "__main__": nprocesses = int(snakemake.threads) noprogress = snakemake.config["run"].get("disable_progressbar", True) - config = snakemake.params["renewable"][snakemake.wildcards.technology] - resource = config["resource"] # pv panel config / wind turbine config - correction_factor = config.get("correction_factor", 1.0) - capacity_per_sqkm = config["capacity_per_sqkm"] - p_nom_max_meth = config.get("potential", "conservative") + params = snakemake.params["renewable"][snakemake.wildcards.technology] + resource = params["resource"] # pv panel params / wind turbine params + correction_factor = params.get("correction_factor", 1.0) + capacity_per_sqkm = params["capacity_per_sqkm"] + p_nom_max_meth = params.get("potential", "conservative") - if isinstance(config.get("corine", {}), list): - config["corine"] = {"grid_codes": config["corine"]} + if isinstance(params.get("corine", {}), list): + params["corine"] = {"grid_codes": params["corine"]} if correction_factor != 1.0: logger.info(f"correction_factor is set as {correction_factor}") @@ -229,13 +229,13 @@ if __name__ == "__main__": regions = regions.set_index("name").rename_axis("bus") buses = regions.index - res = config.get("excluder_resolution", 100) + res = params.get("excluder_resolution", 100) excluder = atlite.ExclusionContainer(crs=3035, res=res) - if config["natura"]: + if params["natura"]: excluder.add_raster(snakemake.input.natura, nodata=0, allow_no_overlap=True) - corine = config.get("corine", {}) + corine = params.get("corine", {}) if "grid_codes" in corine: codes = corine["grid_codes"] excluder.add_raster(snakemake.input.corine, codes=codes, invert=True, crs=3035) @@ -246,28 +246,28 @@ if __name__ == "__main__": snakemake.input.corine, codes=codes, buffer=buffer, crs=3035 ) - if "ship_threshold" in config: + if "ship_threshold" in params: shipping_threshold = ( - config["ship_threshold"] * 8760 * 6 + params["ship_threshold"] * 8760 * 6 ) # approximation because 6 years of data which is hourly collected func = functools.partial(np.less, shipping_threshold) excluder.add_raster( snakemake.input.ship_density, codes=func, crs=4326, allow_no_overlap=True ) - if config.get("max_depth"): + if params.get("max_depth"): # lambda not supported for atlite + multiprocessing # use named function np.greater with partially frozen argument instead # and exclude areas where: -max_depth > grid cell depth - func = functools.partial(np.greater, -config["max_depth"]) + func = functools.partial(np.greater, -params["max_depth"]) excluder.add_raster(snakemake.input.gebco, codes=func, crs=4326, nodata=-1000) - if "min_shore_distance" in config: - buffer = config["min_shore_distance"] + if "min_shore_distance" in params: + buffer = params["min_shore_distance"] excluder.add_geometry(snakemake.input.country_shapes, buffer=buffer) - if "max_shore_distance" in config: - buffer = config["max_shore_distance"] + if "max_shore_distance" in params: + buffer = params["max_shore_distance"] excluder.add_geometry( snakemake.input.country_shapes, buffer=buffer, invert=True ) @@ -309,7 +309,7 @@ if __name__ == "__main__": p_nom_max = capacities / max_cap_factor else: raise AssertionError( - 'Config key `potential` should be one of "simple" ' + 'params key `potential` should be one of "simple" ' f'(default) or "conservative", not "{p_nom_max_meth}"' ) @@ -358,13 +358,13 @@ if __name__ == "__main__": # select only buses with some capacity and minimal capacity factor ds = ds.sel( bus=( - (ds["profile"].mean("time") > config.get("min_p_max_pu", 0.0)) - & (ds["p_nom_max"] > config.get("min_p_nom_max", 0.0)) + (ds["profile"].mean("time") > params.get("min_p_max_pu", 0.0)) + & (ds["p_nom_max"] > params.get("min_p_nom_max", 0.0)) ) ) - if "clip_p_max_pu" in config: - min_p_max_pu = config["clip_p_max_pu"] + if "clip_p_max_pu" in params: + min_p_max_pu = params["clip_p_max_pu"] ds["profile"] = ds["profile"].where(ds["profile"] >= min_p_max_pu, 0) ds.to_netcdf(snakemake.output.profile) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 86a9bc82..1db9b916 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -727,7 +727,7 @@ def cycling_shift(df, steps=1): return df -def prepare_costs(cost_file, config, nyears): +def prepare_costs(cost_file, params, nyears): # set all asset costs and other parameters costs = pd.read_csv(cost_file, index_col=[0, 1]).sort_index() @@ -739,7 +739,7 @@ def prepare_costs(cost_file, config, nyears): costs.loc[:, "value"].unstack(level=1).groupby("technology").sum(min_count=1) ) - costs = costs.fillna(config["fill_values"]) + costs = costs.fillna(params["fill_values"]) def annuity_factor(v): return annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100 From 6366dc4c55076c084caddcbd67f997c17e9360bd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 17 May 2023 17:26:00 +0000 Subject: [PATCH 53/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_electricity.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index a3e033e7..adb8bf31 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -638,9 +638,7 @@ def attach_OPSD_renewables(n, tech_map): def estimate_renewable_capacities(n, electricity_params, countries): year = electricity_params["estimate_renewable_capacities"]["year"] - tech_map = electricity_params["estimate_renewable_capacities"][ - "technology_mapping" - ] + tech_map = electricity_params["estimate_renewable_capacities"]["technology_mapping"] expansion_limit = electricity_params["estimate_renewable_capacities"][ "expansion_limit" ] @@ -828,7 +826,9 @@ if __name__ == "__main__": "technology_mapping" ] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.params["electricity"],snakemake.params["countries"]) + estimate_renewable_capacities( + n, snakemake.params["electricity"], snakemake.params["countries"] + ) update_p_nom_max(n) From d97a5434630d93c2f41ab64c8304ccc94961bee9 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 17 May 2023 19:37:47 +0200 Subject: [PATCH 54/78] missing a coma in build_electricity.smk --- rules/build_electricity.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 88e7c548..1ba1a1c5 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -277,7 +277,7 @@ rule add_electricity: countries=config["countries"], renewable=config["renewable"], electricity=config["electricity"], - conventional=config.get("conventional", {}) + conventional=config.get("conventional", {}), costs=config["costs"], input: **{ From c043100ada790d088ea190e9f8ba634e28a204e1 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Fri, 19 May 2023 15:42:07 +0200 Subject: [PATCH 55/78] add more params in simplify_network.py --- rules/build_electricity.smk | 8 +++- scripts/add_electricity.py | 2 +- scripts/cluster_network.py | 2 +- scripts/simplify_network.py | 76 +++++++++++++++++++------------------ 4 files changed, 49 insertions(+), 39 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 1ba1a1c5..f33d1686 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -318,6 +318,12 @@ rule simplify_network: clustering=config["clustering"], electricity=config["electricity"], costs=config["costs"], + renewable=config["renewable"], + length_factor=config["lines"]["length_factor"], + p_max_pu=config["links"].get("p_max_pu", 1.0), + exclude_carriers=config["clustering"]["simplify_network"].get("exclude_carriers", []), + focus_weights=config.get("focus_weights", None), + solver_name=config["solving"]["solver"]["name"], input: network=RESOURCES + "networks/elec.nc", tech_costs=COSTS, @@ -350,7 +356,7 @@ rule cluster_network: lines=config["lines"], renewable=config["renewable"], clustering=config["clustering"], - enable=config["enable"].get("custom_busmap", False), + custom_busmap=config["enable"].get("custom_busmap", False), input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index adb8bf31..3717f0e8 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -756,7 +756,7 @@ if __name__ == "__main__": ppl, conventional_carriers, extendable_carriers, - snakemake.params.get("conventional", {}), + snakemake.params["conventional"], conventional_inputs, ) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 78798804..d217e5ed 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -521,7 +521,7 @@ if __name__ == "__main__": for p in aggregation_strategies.keys() } - custom_busmap = snakemake.params["enable"].get("custom_busmap", False) + custom_busmap = snakemake.params["custom_busmap"] if custom_busmap: custom_busmap = pd.read_csv( snakemake.input.custom_busmap, index_col=0, squeeze=True diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index bb7f615c..91dde6b1 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -149,17 +149,17 @@ def simplify_network_to_380(n): return n, trafo_map -def _prepare_connection_costs_per_link(n, costs, config): +def _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param): if n.links.empty: return {} connection_costs_per_link = {} - for tech in config["renewable"]: + for tech in renewable_param: if tech.startswith("offwind"): connection_costs_per_link[tech] = ( n.links.length - * config["lines"]["length_factor"] + * length_factor_param * ( n.links.underwater_fraction * costs.at[tech + "-connection-submarine", "capital_cost"] @@ -172,10 +172,10 @@ def _prepare_connection_costs_per_link(n, costs, config): def _compute_connection_costs_to_bus( - n, busmap, costs, config, connection_costs_per_link=None, buses=None + n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link=None, buses=None ): if connection_costs_per_link is None: - connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, config) + connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param) if buses is None: buses = busmap.index[busmap.index != busmap.values] @@ -265,7 +265,7 @@ def _aggregate_and_move_components( n.mremove(c, df.index[df.bus0.isin(buses_to_del) | df.bus1.isin(buses_to_del)]) -def simplify_links(n, costs, config, output, aggregation_strategies=dict()): +def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_param, exclude_carriers_param, output, aggregation_strategies=dict()): ## Complex multi-node links are folded into end-points logger.info("Simplifying connected link components") @@ -315,7 +315,7 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()): busmap = n.buses.index.to_series() - connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, config) + connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param) connection_costs_to_bus = pd.DataFrame( 0.0, index=n.buses.index, columns=list(connection_costs_per_link) ) @@ -333,12 +333,11 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()): ) busmap.loc[buses] = b[np.r_[0, m.argmin(axis=0), 1]] connection_costs_to_bus.loc[buses] += _compute_connection_costs_to_bus( - n, busmap, costs, config, connection_costs_per_link, buses + n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link, buses ) all_links = [i for _, i in sum(links, [])] - p_max_pu = config["links"].get("p_max_pu", 1.0) lengths = n.links.loc[all_links, "length"] name = lengths.idxmax() + "+{}".format(len(links) - 1) params = dict( @@ -354,8 +353,8 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()): / lengths.sum() * n.links.loc[all_links, "underwater_fraction"] ), - p_max_pu=p_max_pu, - p_min_pu=-p_max_pu, + p_max_pu=p_max_pu_param, + p_min_pu=-p_max_pu_param, underground=False, under_construction=False, ) @@ -377,33 +376,29 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()): logger.debug("Collecting all components using the busmap") - exclude_carriers = config["clustering"]["simplify_network"].get( - "exclude_carriers", [] - ) - _aggregate_and_move_components( n, busmap, connection_costs_to_bus, output, aggregation_strategies=aggregation_strategies, - exclude_carriers=exclude_carriers, + exclude_carriers=exclude_carriers_param, ) return n, busmap -def remove_stubs(n, costs, config, output, aggregation_strategies=dict()): +def remove_stubs(n, costs, renewable_param, length_factor_param, clustering_param, exclude_carriers_param, output, aggregation_strategies=dict()): logger.info("Removing stubs") - across_borders = config["clustering"]["simplify_network"].get( + across_borders = clustering_param["simplify_network"].get( "remove_stubs_across_borders", True ) matching_attrs = [] if across_borders else ["country"] busmap = busmap_by_stubs(n, matching_attrs) - connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, config) + connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, renewable_param, length_factor_param) - exclude_carriers = config["clustering"]["simplify_network"].get( + exclude_carriers = clustering_param["simplify_network"].get( "exclude_carriers", [] ) @@ -473,17 +468,15 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): def cluster( - n, n_clusters, config, algorithm="hac", feature=None, aggregation_strategies=dict() + n, n_clusters, focus_weights_param, renewable_param, solver_name_param, algorithm="hac", feature=None, aggregation_strategies=dict() ): logger.info(f"Clustering to {n_clusters} buses") - focus_weights = config.get("focus_weights", None) - renewable_carriers = pd.Index( [ tech for tech in n.generators.carrier.unique() - if tech.split("-", 2)[0] in config["renewable"] + if tech.split("-", 2)[0] in renewable_param ] ) @@ -492,10 +485,10 @@ def cluster( n_clusters, custom_busmap=False, aggregation_strategies=aggregation_strategies, - solver_name=config["solving"]["solver"]["name"], + solver_name=solver_name_param, algorithm=algorithm, feature=feature, - focus_weights=focus_weights, + focus_weights=focus_weights_param, ) return clustering.network, clustering.busmap @@ -531,23 +524,32 @@ if __name__ == "__main__": ) n, simplify_links_map = simplify_links( - n, technology_costs, snakemake.config, snakemake.output, aggregation_strategies + n, technology_costs, + snakemake.params['renewable'], + snakemake.params['length_factor'], + snakemake.params['p_max_pu'], + snakemake.params['exclude_carriers'], + snakemake.output, + aggregation_strategies ) busmaps = [trafo_map, simplify_links_map] - cluster_config = snakemake.params["clustering"]["simplify_network"] - if cluster_config.get("remove_stubs", True): + cluster_param = snakemake.params["clustering"]["simplify_network"] + if cluster_param.get("remove_stubs", True): n, stub_map = remove_stubs( n, technology_costs, - snakemake.config, + snakemake.params['renewable'], + snakemake.params['length_factor'], + snakemake.params["clustering"], + snakemake.params['exclude_carriers'], snakemake.output, aggregation_strategies=aggregation_strategies, ) busmaps.append(stub_map) - if cluster_config.get("to_substations", False): + if cluster_param.get("to_substations", False): n, substation_map = aggregate_to_substations(n, aggregation_strategies) busmaps.append(substation_map) @@ -558,10 +560,10 @@ if __name__ == "__main__": .get("cluster_network", {}) .get("algorithm", "hac") == "hac" - or cluster_config.get("algorithm", "hac") == "hac" + or cluster_param.get("algorithm", "hac") == "hac" ): carriers = ( - cluster_config.get("feature", "solar+onwind-time").split("-")[0].split("+") + cluster_param.get("feature", "solar+onwind-time").split("-")[0].split("+") ) for carrier in carriers: buses_i = list( @@ -577,9 +579,11 @@ if __name__ == "__main__": n, cluster_map = cluster( n, int(snakemake.wildcards.simpl), - snakemake.config, - cluster_config.get("algorithm", "hac"), - cluster_config.get("feature", None), + snakemake.params['focus_weights'], + snakemake.params['renewable'], + snakemake.params['solver_name'], + cluster_param.get("algorithm", "hac"), + cluster_param.get("feature", None), aggregation_strategies, ) busmaps.append(cluster_map) From 216a02fba12fe445e36fae1c019820715379c437 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 19 May 2023 13:42:43 +0000 Subject: [PATCH 56/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_electricity.smk | 4 +- scripts/simplify_network.py | 90 +++++++++++++++++++++++++++---------- 2 files changed, 69 insertions(+), 25 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index f33d1686..45436c5d 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -321,7 +321,9 @@ rule simplify_network: renewable=config["renewable"], length_factor=config["lines"]["length_factor"], p_max_pu=config["links"].get("p_max_pu", 1.0), - exclude_carriers=config["clustering"]["simplify_network"].get("exclude_carriers", []), + exclude_carriers=config["clustering"]["simplify_network"].get( + "exclude_carriers", [] + ), focus_weights=config.get("focus_weights", None), solver_name=config["solving"]["solver"]["name"], input: diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 91dde6b1..689e8084 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -172,10 +172,18 @@ def _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_ def _compute_connection_costs_to_bus( - n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link=None, buses=None + n, + busmap, + costs, + renewable_param, + length_factor_param, + connection_costs_per_link=None, + buses=None, ): if connection_costs_per_link is None: - connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param) + connection_costs_per_link = _prepare_connection_costs_per_link( + n, costs, renewable_param, length_factor_param + ) if buses is None: buses = busmap.index[busmap.index != busmap.values] @@ -265,7 +273,16 @@ def _aggregate_and_move_components( n.mremove(c, df.index[df.bus0.isin(buses_to_del) | df.bus1.isin(buses_to_del)]) -def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_param, exclude_carriers_param, output, aggregation_strategies=dict()): +def simplify_links( + n, + costs, + renewable_param, + length_factor_param, + p_max_pu_param, + exclude_carriers_param, + output, + aggregation_strategies=dict(), +): ## Complex multi-node links are folded into end-points logger.info("Simplifying connected link components") @@ -315,7 +332,9 @@ def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_para busmap = n.buses.index.to_series() - connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param) + connection_costs_per_link = _prepare_connection_costs_per_link( + n, costs, renewable_param, length_factor_param + ) connection_costs_to_bus = pd.DataFrame( 0.0, index=n.buses.index, columns=list(connection_costs_per_link) ) @@ -333,7 +352,13 @@ def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_para ) busmap.loc[buses] = b[np.r_[0, m.argmin(axis=0), 1]] connection_costs_to_bus.loc[buses] += _compute_connection_costs_to_bus( - n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link, buses + n, + busmap, + costs, + renewable_param, + length_factor_param, + connection_costs_per_link, + buses, ) all_links = [i for _, i in sum(links, [])] @@ -387,7 +412,16 @@ def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_para return n, busmap -def remove_stubs(n, costs, renewable_param, length_factor_param, clustering_param, exclude_carriers_param, output, aggregation_strategies=dict()): +def remove_stubs( + n, + costs, + renewable_param, + length_factor_param, + clustering_param, + exclude_carriers_param, + output, + aggregation_strategies=dict(), +): logger.info("Removing stubs") across_borders = clustering_param["simplify_network"].get( @@ -396,12 +430,12 @@ def remove_stubs(n, costs, renewable_param, length_factor_param, clustering_para matching_attrs = [] if across_borders else ["country"] busmap = busmap_by_stubs(n, matching_attrs) - connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, renewable_param, length_factor_param) - - exclude_carriers = clustering_param["simplify_network"].get( - "exclude_carriers", [] + connection_costs_to_bus = _compute_connection_costs_to_bus( + n, busmap, costs, renewable_param, length_factor_param ) + exclude_carriers = clustering_param["simplify_network"].get("exclude_carriers", []) + _aggregate_and_move_components( n, busmap, @@ -468,7 +502,14 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): def cluster( - n, n_clusters, focus_weights_param, renewable_param, solver_name_param, algorithm="hac", feature=None, aggregation_strategies=dict() + n, + n_clusters, + focus_weights_param, + renewable_param, + solver_name_param, + algorithm="hac", + feature=None, + aggregation_strategies=dict(), ): logger.info(f"Clustering to {n_clusters} buses") @@ -524,13 +565,14 @@ if __name__ == "__main__": ) n, simplify_links_map = simplify_links( - n, technology_costs, - snakemake.params['renewable'], - snakemake.params['length_factor'], - snakemake.params['p_max_pu'], - snakemake.params['exclude_carriers'], - snakemake.output, - aggregation_strategies + n, + technology_costs, + snakemake.params["renewable"], + snakemake.params["length_factor"], + snakemake.params["p_max_pu"], + snakemake.params["exclude_carriers"], + snakemake.output, + aggregation_strategies, ) busmaps = [trafo_map, simplify_links_map] @@ -540,10 +582,10 @@ if __name__ == "__main__": n, stub_map = remove_stubs( n, technology_costs, - snakemake.params['renewable'], - snakemake.params['length_factor'], + snakemake.params["renewable"], + snakemake.params["length_factor"], snakemake.params["clustering"], - snakemake.params['exclude_carriers'], + snakemake.params["exclude_carriers"], snakemake.output, aggregation_strategies=aggregation_strategies, ) @@ -579,9 +621,9 @@ if __name__ == "__main__": n, cluster_map = cluster( n, int(snakemake.wildcards.simpl), - snakemake.params['focus_weights'], - snakemake.params['renewable'], - snakemake.params['solver_name'], + snakemake.params["focus_weights"], + snakemake.params["renewable"], + snakemake.params["solver_name"], cluster_param.get("algorithm", "hac"), cluster_param.get("feature", None), aggregation_strategies, From d2f6138ca564d3286fcd8f3f93e954d22b4d16d7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 21:56:04 +0000 Subject: [PATCH 57/78] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/docformatter: v1.6.5 → v1.7.1](https://github.com/PyCQA/docformatter/compare/v1.6.5...v1.7.1) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48915f4c..34d13746 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: # Make docstrings PEP 257 compliant - repo: https://github.com/PyCQA/docformatter - rev: v1.6.5 + rev: v1.7.1 hooks: - id: docformatter args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] From 0f09545d9626b7e64c883a721f592847c5fd4367 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 21:56:29 +0000 Subject: [PATCH 58/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/_helpers.py | 9 +++++---- scripts/build_retro_cost.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index a67fb105..8086c1b5 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -385,10 +385,11 @@ def mock_snakemake(rulename, configfiles=[], **wildcards): def override_component_attrs(directory): - """Tell PyPSA that links can have multiple outputs by - overriding the component_attrs. This can be done for - as many buses as you need with format busi for i = 2,3,4,5,.... - See https://pypsa.org/doc/components.html#link-with-multiple-outputs-or-inputs + """ + Tell PyPSA that links can have multiple outputs by overriding the + component_attrs. This can be done for as many buses as you need with format + busi for i = 2,3,4,5,.... See https://pypsa.org/doc/components.html#link- + with-multiple-outputs-or-inputs. Parameters ---------- diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index 9dbfc375..63ae0743 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -698,8 +698,8 @@ def get_solar_gains_per_year(window_area): def map_to_lstrength(l_strength, df): """ - renames column names from a pandas dataframe to map tabula retrofitting - strengths [2 = moderate, 3 = ambitious] to l_strength + Renames column names from a pandas dataframe to map tabula retrofitting + strengths [2 = moderate, 3 = ambitious] to l_strength. """ middle = len(l_strength) // 2 map_to_l = pd.MultiIndex.from_arrays( From 312dd81f216f2ab55ee2eb8b13c5a54532fe9e06 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Wed, 24 May 2023 11:13:37 +0200 Subject: [PATCH 59/78] Fix bug with underground H2 storage creation For some small model regions, none of the H2 cavern types specified in the configuration might actually be available, in which case the line `h2_caverns = h2_caverns[cavern_types].sum(axis=1)` throws an error. --- scripts/prepare_sector_network.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 012c9714..57deb5fd 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1105,7 +1105,11 @@ def add_storage_and_grids(n, costs): cavern_types = snakemake.config["sector"]["hydrogen_underground_storage_locations"] h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0) - if not h2_caverns.empty and options["hydrogen_underground_storage"]: + if ( + not h2_caverns.empty + and options["hydrogen_underground_storage"] + and set(cavern_types).intersection(h2_caverns.columns) + ): h2_caverns = h2_caverns[cavern_types].sum(axis=1) # only use sites with at least 2 TWh potential From dc043d8aeff58455ad0e339485de375e8afb8d02 Mon Sep 17 00:00:00 2001 From: lucie_rc <104382956+LucieRC@users.noreply.github.com> Date: Wed, 24 May 2023 15:46:53 +0300 Subject: [PATCH 60/78] Update Discord link README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 02cc3163..79cdfa65 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ repository](https://doi.org/10.5281/zenodo.3601881). We strongly welcome anyone interested in contributing to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on GitHub. - In case of code-related **questions**, please post on [stack overflow](https://stackoverflow.com/questions/tagged/pypsa). - For non-programming related and more general questions please refer to the [mailing list](https://groups.google.com/group/pypsa). -- To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb). +- To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.com/invite/AnuJBk23FU). - For **bugs and feature requests**, please use the [PyPSA-Eur Github Issues page](https://github.com/PyPSA/pypsa-eur/issues). # Licence From 1c73bb03426beaaaedc10df7ee83d8f036395a4d Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Sat, 27 May 2023 12:22:53 +0200 Subject: [PATCH 61/78] add params for solve_network --- rules/solve_electricity.smk | 6 ++++++ rules/solve_myopic.smk | 6 ++++++ rules/solve_overnight.smk | 6 ++++++ scripts/solve_network.py | 38 ++++++++++++++++++------------------- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 2e9c4b44..f0ce55c8 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -6,6 +6,12 @@ rule solve_network: params: solving=config["solving"], + config_parts={ + "foresight":config["foresight"], + "planning_horizons":config["scenario"]["planning_horizons"], + "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), + "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] + }, input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index ec7638cf..c682f780 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -84,6 +84,12 @@ ruleorder: add_existing_baseyear > add_brownfield rule solve_sector_network_myopic: params: solving=config["solving"], + config_parts={ + "foresight":config["foresight"], + "planning_horizons":config["scenario"]["planning_horizons"], + "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), + "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] + }, input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index b657eb2b..050372bd 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -6,6 +6,12 @@ rule solve_sector_network: params: solving=config["solving"], + config_parts={ + "foresight":config["foresight"], + "planning_horizons":config["scenario"]["planning_horizons"], + "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), + "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] + }, input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 180c83c6..d615fc85 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -44,14 +44,14 @@ pypsa.pf.logger.setLevel(logging.WARNING) from pypsa.descriptors import get_switchable_as_dense as get_as_dense -def add_land_use_constraint(n, config): +def add_land_use_constraint(n, param, config): if "m" in snakemake.wildcards.clusters: - _add_land_use_constraint_m(n, config) + _add_land_use_constraint_m(n, param, config) else: - _add_land_use_constraint(n, config) + _add_land_use_constraint(n, param) -def _add_land_use_constraint(n, config): +def _add_land_use_constraint(n, param): # warning: this will miss existing offwind which is not classed AC-DC and has carrier 'offwind' for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]: @@ -80,10 +80,10 @@ def _add_land_use_constraint(n, config): n.generators.p_nom_max.clip(lower=0, inplace=True) -def _add_land_use_constraint_m(n, config): +def _add_land_use_constraint_m(n, param, config): # if generators clustering is lower than network clustering, land_use accounting is at generators clusters - planning_horizons = config["scenario"]["planning_horizons"] + planning_horizons = param["planning_horizons"] grouping_years = config["existing_capacities"]["grouping_years"] current_horizon = snakemake.wildcards.planning_horizons @@ -141,7 +141,7 @@ def add_co2_sequestration_limit(n, limit=200): ) -def prepare_network(n, solve_opts=None, config=None): +def prepare_network(n, solve_opts=None, config=None, param=None): if "clip_p_max_pu" in solve_opts: for df in ( n.generators_t.p_max_pu, @@ -191,11 +191,11 @@ def prepare_network(n, solve_opts=None, config=None): n.set_snapshots(n.snapshots[:nhours]) n.snapshot_weightings[:] = 8760.0 / nhours - if config["foresight"] == "myopic": - add_land_use_constraint(n, config) + if param["foresight"] == "myopic": + add_land_use_constraint(n, param, config) if n.stores.carrier.eq("co2 stored").any(): - limit = config["sector"].get("co2_sequestration_potential", 200) + limit = param["co2_sequestration_potential"] add_co2_sequestration_limit(n, limit=limit) return n @@ -222,7 +222,7 @@ def add_CCL_constraints(n, config): agg_p_nom_limits: data/agg_p_nom_minmax.csv """ agg_p_nom_minmax = pd.read_csv( - config["electricity"]["agg_p_nom_limits"], index_col=[0, 1] + config['electricity']["agg_p_nom_limits"], index_col=[0, 1] ) logger.info("Adding generation capacity constraints per carrier and country") p_nom = n.model["Generator-p_nom"] @@ -370,7 +370,7 @@ def add_SAFE_constraints(n, config): Which sets a reserve margin of 10% above the peak demand. """ peakdemand = n.loads_t.p_set.sum(axis=1).max() - margin = 1.0 + config["electricity"]["SAFE_reservemargin"] + margin = 1.0 + config['electricity']["SAFE_reservemargin"] reserve_margin = peakdemand * margin # TODO: do not take this from the plotting config! conv_techs = config["plotting"]["conv_techs"] @@ -590,13 +590,13 @@ def extra_functionality(n, snapshots): add_pipe_retrofit_constraint(n) -def solve_network(n, config, opts="", **kwargs): - set_of_options = config["solving"]["solver"]["options"] +def solve_network(n, config, solving_param, opts="", **kwargs): + set_of_options = solving_param["solver"]["options"] solver_options = ( - config["solving"]["solver_options"][set_of_options] if set_of_options else {} + solving_param["solver_options"][set_of_options] if set_of_options else {} ) - solver_name = config["solving"]["solver"]["name"] - cf_solving = config["solving"]["options"] + solver_name = solving_param["solver"]["name"] + cf_solving = solving_param["options"] track_iterations = cf_solving.get("track_iterations", False) min_iterations = cf_solving.get("min_iterations", 4) max_iterations = cf_solving.get("max_iterations", 6) @@ -675,10 +675,10 @@ if __name__ == "__main__": else: n = pypsa.Network(snakemake.input.network) - n = prepare_network(n, solve_opts, config=snakemake.config) + n = prepare_network(n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"]) n = solve_network( - n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + n, config=snakemake.config, solving_param=snakemake.params["solving"], opts=opts, log_fn=snakemake.log.solver ) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) From ead87afce1ab0d7179b9012f88df1089e609bc61 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 27 May 2023 10:23:28 +0000 Subject: [PATCH 62/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/solve_electricity.smk | 14 +++++++++----- rules/solve_myopic.smk | 14 +++++++++----- rules/solve_overnight.smk | 14 +++++++++----- scripts/solve_network.py | 14 ++++++++++---- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index f0ce55c8..30520dc1 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -7,11 +7,15 @@ rule solve_network: params: solving=config["solving"], config_parts={ - "foresight":config["foresight"], - "planning_horizons":config["scenario"]["planning_horizons"], - "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), - "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] - }, + "foresight": config["foresight"], + "planning_horizons": config["scenario"]["planning_horizons"], + "co2_sequestration_potential": config["sector"].get( + "co2_sequestration_potential", 200 + ), + "H2_retrofit_capacity_per_CH4": config["sector"][ + "H2_retrofit_capacity_per_CH4" + ], + }, input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index c682f780..22555f75 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -85,11 +85,15 @@ rule solve_sector_network_myopic: params: solving=config["solving"], config_parts={ - "foresight":config["foresight"], - "planning_horizons":config["scenario"]["planning_horizons"], - "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), - "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] - }, + "foresight": config["foresight"], + "planning_horizons": config["scenario"]["planning_horizons"], + "co2_sequestration_potential": config["sector"].get( + "co2_sequestration_potential", 200 + ), + "H2_retrofit_capacity_per_CH4": config["sector"][ + "H2_retrofit_capacity_per_CH4" + ], + }, input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 050372bd..a523f132 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -7,11 +7,15 @@ rule solve_sector_network: params: solving=config["solving"], config_parts={ - "foresight":config["foresight"], - "planning_horizons":config["scenario"]["planning_horizons"], - "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), - "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] - }, + "foresight": config["foresight"], + "planning_horizons": config["scenario"]["planning_horizons"], + "co2_sequestration_potential": config["sector"].get( + "co2_sequestration_potential", 200 + ), + "H2_retrofit_capacity_per_CH4": config["sector"][ + "H2_retrofit_capacity_per_CH4" + ], + }, input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/solve_network.py b/scripts/solve_network.py index d615fc85..6e07e340 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -222,7 +222,7 @@ def add_CCL_constraints(n, config): agg_p_nom_limits: data/agg_p_nom_minmax.csv """ agg_p_nom_minmax = pd.read_csv( - config['electricity']["agg_p_nom_limits"], index_col=[0, 1] + config["electricity"]["agg_p_nom_limits"], index_col=[0, 1] ) logger.info("Adding generation capacity constraints per carrier and country") p_nom = n.model["Generator-p_nom"] @@ -370,7 +370,7 @@ def add_SAFE_constraints(n, config): Which sets a reserve margin of 10% above the peak demand. """ peakdemand = n.loads_t.p_set.sum(axis=1).max() - margin = 1.0 + config['electricity']["SAFE_reservemargin"] + margin = 1.0 + config["electricity"]["SAFE_reservemargin"] reserve_margin = peakdemand * margin # TODO: do not take this from the plotting config! conv_techs = config["plotting"]["conv_techs"] @@ -675,10 +675,16 @@ if __name__ == "__main__": else: n = pypsa.Network(snakemake.input.network) - n = prepare_network(n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"]) + n = prepare_network( + n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"] + ) n = solve_network( - n, config=snakemake.config, solving_param=snakemake.params["solving"], opts=opts, log_fn=snakemake.log.solver + n, + config=snakemake.config, + solving_param=snakemake.params["solving"], + opts=opts, + log_fn=snakemake.log.solver, ) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) From e581ca930c24de2c3e3fa4d0d6bdac0fbfa06b30 Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Sat, 27 May 2023 15:50:37 +0200 Subject: [PATCH 63/78] made params more specific --- rules/build_electricity.smk | 29 +++++--- rules/build_sector.smk | 72 ++++++++++++++++---- rules/postprocess.smk | 3 +- rules/solve_electricity.smk | 2 +- rules/solve_myopic.smk | 7 +- scripts/add_brownfield.py | 6 +- scripts/add_electricity.py | 11 ++- scripts/add_existing_baseyear.py | 2 +- scripts/add_extra_components.py | 23 +++---- scripts/build_cop_profiles.py | 2 +- scripts/build_cutout.py | 2 +- scripts/build_hydro_profile.py | 2 +- scripts/build_industrial_distribution_key.py | 2 +- scripts/build_industry_sector_ratios.py | 2 +- scripts/build_powerplants.py | 4 +- scripts/build_retro_cost.py | 2 +- scripts/build_sequestration_potentials.py | 2 +- scripts/cluster_network.py | 8 +-- scripts/plot_summary.py | 4 +- scripts/prepare_network.py | 10 +-- scripts/prepare_sector_network.py | 14 ++-- scripts/simplify_network.py | 2 +- scripts/solve_operations_network.py | 2 +- 23 files changed, 131 insertions(+), 82 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 45436c5d..9b3a03fb 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -39,7 +39,8 @@ rule build_electricity_demand: rule build_powerplants: params: - electricity=config["electricity"], + powerplants_filter=config["electricity"]["powerplants_filter"], + custom_powerplants=config["electricity"]["custom_powerplants"], countries=config["countries"], input: base_network=RESOURCES + "networks/base.nc", @@ -138,7 +139,7 @@ if config["enable"].get("build_cutout", False): rule build_cutout: params: snapshots=config["snapshots"], - atlite=config["atlite"], + cutouts=config["atlite"]["cutouts"], input: regions_onshore=RESOURCES + "regions_onshore.geojson", regions_offshore=RESOURCES + "regions_offshore.geojson", @@ -252,6 +253,7 @@ rule build_renewable_profiles: rule build_hydro_profile: params: + hydro=config["renewable"]["hydro"], countries=config["countries"], renewable=config["renewable"], input: @@ -272,8 +274,8 @@ rule build_hydro_profile: rule add_electricity: params: - lines=config["lines"], - load=config["load"], + length_factor=config["lines"]["length_factor"], + scaling_factor=config["load"]["scaling_factor"], countries=config["countries"], renewable=config["renewable"], electricity=config["electricity"], @@ -316,7 +318,7 @@ rule add_electricity: rule simplify_network: params: clustering=config["clustering"], - electricity=config["electricity"], + max_hours=config["electricity"]["max_hours"], costs=config["costs"], renewable=config["renewable"], length_factor=config["lines"]["length_factor"], @@ -352,10 +354,11 @@ rule simplify_network: rule cluster_network: params: - solving=config["solving"], - electricity=config["electricity"], + solver_name=config["solving"]["solver"]["name"], + max_hours=config["electricity"]["max_hours"], + conventional_carriers=config["electricity"].get("conventional_carriers", []), costs=config["costs"], - lines=config["lines"], + length_factor=config["lines"]["length_factor"], renewable=config["renewable"], clustering=config["clustering"], custom_busmap=config["enable"].get("custom_busmap", False), @@ -392,7 +395,8 @@ rule cluster_network: rule add_extra_components: params: costs=config["costs"], - electricity=config["electricity"], + ext_carriers=config["electricity"]["extendable_carriers"], + max_hours=config["electricity"]["max_hours"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", tech_costs=COSTS, @@ -414,9 +418,12 @@ rule add_extra_components: rule prepare_network: params: links=config["links"], - solving=config["solving"], lines=config["lines"], - electricity=config["electricity"], + solver_name=config["solving"]["solver"]["name"], + co2base=config["electricity"]["co2base"], + co2limit=config["electricity"]["co2limit"], + gaslimit=config["electricity"].get("gaslimit"), + max_hours=config["electricity"]["max_hours"], costs=config["costs"], input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 9faae4b9..da8a60f9 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -186,7 +186,7 @@ rule build_temperature_profiles: rule build_cop_profiles: params: - sector=config["sector"], + heat_pump_sink_T=config["sector"]["heat_pump_sink_T"], input: temp_soil_total=RESOURCES + "temp_soil_total_elec_s{simpl}_{clusters}.nc", temp_soil_rural=RESOURCES + "temp_soil_rural_elec_s{simpl}_{clusters}.nc", @@ -330,7 +330,7 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_sequestration_potentials: params: - sector=config["sector"], + co2seq_potential=config["sector"]["regional_co2_sequestration_potential"], input: sequestration_potential=HTTP.remote( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", @@ -405,8 +405,27 @@ rule build_ammonia_production: rule build_industry_sector_ratios: params: - industry=config["industry"], - sector=config["sector"], + industry={ + "H2_DRI":config["industry"]["H2_DRI"], + "elec_DRI":config["industry"]["elec_DRI"], + "HVC_production_today":config["industry"]["HVC_production_today"], + "petrochemical_process_emissions":config["industry"]["petrochemical_process_emissions"], + "NH3_process_emissions":config["industry"]["NH3_process_emissions"], + "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"], + "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"], + "chlorine_production_today":config["industry"]["chlorine_production_today"], + "MWh_H2_per_tCl":config["industry"]["MWh_H2_per_tCl"], + "MWh_elec_per_tCl":config["industry"]["MWh_elec_per_tCl"], + "methanol_production_today":config["industry"]["methanol_production_today"], + "MWh_CH4_per_tMeOH":config["industry"]["MWh_CH4_per_tMeOH"], + "MWh_elec_per_tMeOH":config["industry"]["MWh_elec_per_tMeOH"], + "MWh_elec_per_tHVC_mechanical_recycling":config["industry"]["MWh_elec_per_tHVC_mechanical_recycling"], + "MWh_elec_per_tHVC_chemical_recycling":config["industry"]["MWh_elec_per_tHVC_chemical_recycling"], + "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"], + "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"], + "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"] + }, + sector_amonia=config["sector"].get("ammonia", False), input: ammonia_production=RESOURCES + "ammonia_production.csv", idees="data/jrc-idees-2015", @@ -427,7 +446,12 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: params: - industry=config["industry"], + industry={ + "reference_year":config["industry"]["reference_year"], + "HVC_production_today":config["industry"]["HVC_production_today"], + "chlorine_production_today":config["industry"]["chlorine_production_today"], + "methanol_production_today":config["industry"]["methanol_production_today"] + }, countries=config["countries"], input: ammonia_production=RESOURCES + "ammonia_production.csv", @@ -451,7 +475,14 @@ rule build_industrial_production_per_country: rule build_industrial_production_per_country_tomorrow: params: - industry=config["industry"], + industry={ + "St_primary_fraction":config["industry"]["St_primary_fraction"], + "DRI_fraction":config["industry"]["DRI_fraction"], + "Al_primary_fraction":config["industry"]["Al_primary_fraction"], + "HVC_mechanical_recycling_fraction":config["industry"]["HVC_mechanical_recycling_fraction"], + "HVC_chemical_recycling_fraction":config["industry"]["HVC_chemical_recycling_fraction"], + "HVC_primary_fraction":config["industry"]["HVC_primary_fraction"] + }, input: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -477,7 +508,7 @@ rule build_industrial_production_per_country_tomorrow: rule build_industrial_distribution_key: params: - industry=config["industry"], + hotmaps_locate_missing=config["industry"].get("hotmaps_locate_missing", False), countries=config["countries"], input: regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -555,7 +586,12 @@ rule build_industrial_energy_demand_per_node: rule build_industrial_energy_demand_per_country_today: params: countries=config["countries"], - industry=config["industry"], + industry={ + "reference_year":config["industry"].get("reference_year", 2015), + "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"], + "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"], + "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"] + }, input: jrc="data/jrc-idees-2015", ammonia_production=RESOURCES + "ammonia_production.csv", @@ -603,7 +639,7 @@ if config["sector"]["retrofitting"]["retro_endogen"]: rule build_retro_cost: params: - sector=config["sector"], + retrofitting=config["sector"]["retrofitting"], countries=config["countries"], input: building_stock="data/retro/data_building_stock.csv", @@ -705,17 +741,23 @@ rule build_transport_demand: rule prepare_sector_network: params: co2_budget=config["co2_budget"], - solving=config["solving"], - existing_capacities=config["existing_capacities"], + solver_name=config["solving"]["solver"]["name"], + conventional_carriers=config["existing_capacities"]["conventional_carriers"], foresight=config["foresight"], costs=config["costs"], sector=config["sector"], - industry=config["industry"], + industry={ + "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"], + "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"], + "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"], + "MWh_NH3_per_MWh_H2_cracker":config["industry"]["MWh_NH3_per_MWh_H2_cracker"] + }, pypsa_eur=config["pypsa_eur"], - lines=config["lines"], - scenario=config["scenario"], + length_factor=config["lines"]["length_factor"], + planning_horizons=config["scenario"]["planning_horizons"], countries=config["countries"], - energy=config["energy"], + emissions_scope=config["energy"]["emissions"], + report_year=config["energy"]["eurostat_report_year"], RDIR=RDIR, input: **build_retro_cost_output, diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 9eb04ef5..ac80cd10 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -122,7 +122,8 @@ rule make_summary: rule plot_summary: params: countries=config["countries"], - scenario=config["scenario"], + planning_horizons=config["scenario"]["planning_horizons"], + sector_opts=config["scenario"]["sector_opts"], plotting=config["plotting"], RDIR=RDIR, input: diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 30520dc1..892415c4 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -41,7 +41,7 @@ rule solve_network: rule solve_operations_network: params: - solving=config["solving"], + options=config["solving"]["options"], input: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 22555f75..8ec1c4a8 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -5,7 +5,7 @@ rule add_existing_baseyear: params: - scenario=config["scenario"], + baseyear=config["scenario"]["planning_horizons"][0], sector=config["sector"], existing_capacities=config["existing_capacities"], costs=config["costs"], @@ -48,8 +48,9 @@ rule add_existing_baseyear: rule add_brownfield: params: - sector=config["sector"], - existing_capacities=config["existing_capacities"], + H2_retrofit=config["sector"]["H2_retrofit"], + H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"], + threshold_capacity=config["existing_capacities"]["threshold_capacity"], input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 9330953b..48dffbb9 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -49,7 +49,7 @@ def add_brownfield(n, n_p, year): ) ] - threshold = snakemake.params["existing_capacities"]["threshold_capacity"] + threshold = snakemake.params["threshold_capacity"] if not chp_heat.empty: threshold_chp_heat = ( @@ -87,7 +87,7 @@ def add_brownfield(n, n_p, year): # deal with gas network pipe_carrier = ["gas pipeline"] - if snakemake.params["sector"]["H2_retrofit"]: + if snakemake.params["H2_retrofit"]: # drop capacities of previous year to avoid duplicating to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year) n.mremove("Link", n.links.loc[to_drop].index) @@ -98,7 +98,7 @@ def add_brownfield(n, n_p, year): & (n.links.build_year != year) ].index gas_pipes_i = n.links[n.links.carrier.isin(pipe_carrier)].index - CH4_per_H2 = 1 / snakemake.params["sector"]["H2_retrofit_capacity_per_CH4"] + CH4_per_H2 = 1 / snakemake.params["H2_retrofit_capacity_per_CH4"] fr = "H2 pipeline retrofitted" to = "gas pipeline" # today's pipe capacity diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 69d91b87..dc5fb7cc 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -137,7 +137,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers): n.import_components_from_dataframe(emissions, "Carrier") -def load_costs(tech_costs, params, elec_params, Nyears=1.0): +def load_costs(tech_costs, params, max_hours, Nyears=1.0): # set all asset costs and other parameters costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index() @@ -180,7 +180,6 @@ def load_costs(tech_costs, params, elec_params, Nyears=1.0): dict(capital_cost=capital_cost, marginal_cost=0.0, co2_emissions=0.0) ) - max_hours = elec_params["max_hours"] costs.loc["battery"] = costs_for_storage( costs.loc["battery storage"], costs.loc["battery inverter"], @@ -728,7 +727,7 @@ if __name__ == "__main__": costs = load_costs( snakemake.input.tech_costs, snakemake.params["costs"], - snakemake.params["electricity"], + snakemake.params["electricity"]["max_hours"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) @@ -759,10 +758,10 @@ if __name__ == "__main__": snakemake.input.load, snakemake.input.nuts3_shapes, snakemake.params["countries"], - snakemake.params["load"]["scaling_factor"], + snakemake.params["scaling_factor"], ) - update_transmission_costs(n, costs, snakemake.params["lines"]["length_factor"]) + update_transmission_costs(n, costs, snakemake.params["length_factor"]) conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") @@ -783,7 +782,7 @@ if __name__ == "__main__": snakemake.input, renewable_carriers, extendable_carriers, - snakemake.params["lines"]["length_factor"], + snakemake.params["length_factor"], ) if "hydro" in renewable_carriers: diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index a4d3748b..bc1d3a05 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -615,7 +615,7 @@ if __name__ == "__main__": options = snakemake.params["sector"] opts = snakemake.wildcards.sector_opts.split("-") - baseyear = snakemake.params["scenario"]["planning_horizons"][0] + baseyear = snakemake.params["baseyear"] overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 08178c0a..ce4e533e 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -67,9 +67,8 @@ idx = pd.IndexSlice logger = logging.getLogger(__name__) -def attach_storageunits(n, costs, elec_opts): - carriers = elec_opts["extendable_carriers"]["StorageUnit"] - max_hours = elec_opts["max_hours"] +def attach_storageunits(n, costs, ext_carriers, max_hours): + carriers = ext_carriers["StorageUnit"] _add_missing_carriers_from_costs(n, costs, carriers) @@ -99,8 +98,8 @@ def attach_storageunits(n, costs, elec_opts): ) -def attach_stores(n, costs, elec_opts): - carriers = elec_opts["extendable_carriers"]["Store"] +def attach_stores(n, costs, ext_carriers): + carriers = ext_carriers["Store"] _add_missing_carriers_from_costs(n, costs, carriers) @@ -187,8 +186,7 @@ def attach_stores(n, costs, elec_opts): ) -def attach_hydrogen_pipelines(n, costs, elec_opts): - ext_carriers = elec_opts["extendable_carriers"] +def attach_hydrogen_pipelines(n, costs, ext_carriers): as_stores = ext_carriers.get("Store", []) if "H2 pipeline" not in ext_carriers.get("Link", []): @@ -235,16 +233,17 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) - elec_config = snakemake.params["electricity"] + ext_carriers = snakemake.params["ext_carriers"] + max_hours = snakemake.params["max_hours"] Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( - snakemake.input.tech_costs, snakemake.params["costs"], elec_config, Nyears + snakemake.input.tech_costs, snakemake.params["costs"], max_hours, Nyears ) - attach_storageunits(n, costs, elec_config) - attach_stores(n, costs, elec_config) - attach_hydrogen_pipelines(n, costs, elec_config) + attach_storageunits(n, costs, ext_carriers, max_hours) + attach_stores(n, costs, ext_carriers) + attach_hydrogen_pipelines(n, costs, ext_carriers) add_nice_carrier_names(n, snakemake.config) diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index 7128ec0d..983eda2d 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -39,7 +39,7 @@ if __name__ == "__main__": for source in ["air", "soil"]: source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"]) - delta_T = snakemake.params["sector"]["heat_pump_sink_T"] - source_T + delta_T = snakemake.params["heat_pump_sink_T"] - source_T cop = coefficient_of_performance(delta_T, source) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 2f61f017..798588d9 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -106,7 +106,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5") configure_logging(snakemake) - cutout_params = snakemake.params["atlite"]["cutouts"][snakemake.wildcards.cutout] + cutout_params = snakemake.params["cutouts"][snakemake.wildcards.cutout] snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) time = [snapshots[0], snapshots[-1]] diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index dd686be3..26bf31c6 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -130,7 +130,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) - params_hydro = snakemake.params["renewable"]["hydro"] + params_hydro = snakemake.params["hydro"] cutout = atlite.Cutout(snakemake.input.cutout) countries = snakemake.params["countries"] diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 9a513673..a51ff8b2 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -73,7 +73,7 @@ def prepare_hotmaps_database(regions): df[["srid", "coordinates"]] = df.geom.str.split(";", expand=True) - if snakemake.params["industry"].get("hotmaps_locate_missing", False): + if snakemake.params["hotmaps_locate_missing"]: df = locate_missing_industrial_sites(df) # remove those sites without valid locations diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index 2ec007a9..fa9e5c18 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -439,7 +439,7 @@ def chemicals_industry(): sector = "Ammonia" df[sector] = 0.0 - if snakemake.params["sector"].get("ammonia", False): + if snakemake.params["sector_amonia"]: df.loc["ammonia", sector] = params["MWh_NH3_per_tNH3"] else: df.loc["hydrogen", sector] = params["MWh_H2_per_tNH3_electrolysis"] diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index bd0ee74e..6edd4ac4 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -134,12 +134,12 @@ if __name__ == "__main__": ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")') ppl = pd.concat([ppl, opsd]) - ppl_query = snakemake.params["electricity"]["powerplants_filter"] + ppl_query = snakemake.params["powerplants_filter"] if isinstance(ppl_query, str): ppl.query(ppl_query, inplace=True) # add carriers from own powerplant files: - custom_ppl_query = snakemake.params["electricity"]["custom_powerplants"] + custom_ppl_query = snakemake.params["custom_powerplants"] ppl = add_custom_powerplants( ppl, snakemake.input.custom_powerplants, custom_ppl_query ) diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index c52d4eb6..8473be7a 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -1040,7 +1040,7 @@ if __name__ == "__main__": # ******** config ********************************************************* - retro_opts = snakemake.params["sector"]["retrofitting"] + retro_opts = snakemake.params["retrofitting"] interest_rate = retro_opts["interest_rate"] annualise_cost = retro_opts["annualise_cost"] # annualise the investment costs tax_weighting = retro_opts[ diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 5c388b2e..9d26b0b9 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -41,7 +41,7 @@ if __name__ == "__main__": "build_sequestration_potentials", simpl="", clusters="181" ) - cf = snakemake.params["sector"]["regional_co2_sequestration_potential"] + cf = snakemake.params["co2seq_potential"] gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index d217e5ed..3cfbd214 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -479,7 +479,7 @@ if __name__ == "__main__": if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) conventional = set( - snakemake.params["electricity"].get("conventional_carriers", []) + snakemake.params["conventional_carriers"] ) aggregate_carriers = conventional.intersection(aggregate_carriers) elif snakemake.wildcards.clusters == "all": @@ -495,13 +495,13 @@ if __name__ == "__main__": n, busmap, linemap, linemap, pd.Series(dtype="O") ) else: - line_length_factor = snakemake.params["lines"]["length_factor"] + line_length_factor = snakemake.params["length_factor"] Nyears = n.snapshot_weightings.objective.sum() / 8760 hvac_overhead_cost = load_costs( snakemake.input.tech_costs, snakemake.params["costs"], - snakemake.params["electricity"], + snakemake.params["max_hours"], Nyears, ).at["HVAC overhead", "capital_cost"] @@ -539,7 +539,7 @@ if __name__ == "__main__": aggregate_carriers, line_length_factor, aggregation_strategies, - snakemake.params["solving"]["solver"]["name"], + snakemake.params["solver_name"], cluster_config.get("algorithm", "hac"), cluster_config.get("feature", "solar+onwind-time"), hvac_overhead_cost, diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 4aa37de5..36f75207 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -455,7 +455,7 @@ def plot_carbon_budget_distribution(input_eurostat): ax1 = plt.subplot(gs1[0, 0]) ax1.set_ylabel("CO$_2$ emissions (Gt per year)", fontsize=22) ax1.set_ylim([0, 5]) - ax1.set_xlim([1990, snakemake.params["scenario"]["planning_horizons"][-1] + 1]) + ax1.set_xlim([1990, snakemake.params["planning_horizons"][-1] + 1]) path_cb = "results/" + snakemake.params.RDIR + "/csvs/" countries = snakemake.params["countries"] @@ -555,7 +555,7 @@ if __name__ == "__main__": plot_balances() - for sector_opts in snakemake.params["scenario"]["sector_opts"]: + for sector_opts in snakemake.params["sector_opts"]: opts = sector_opts.split("-") for o in opts: if "cb" in o: diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 46e4e74a..51777ef5 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -254,7 +254,7 @@ if __name__ == "__main__": costs = load_costs( snakemake.input.tech_costs, snakemake.params["costs"], - snakemake.params["electricity"], + snakemake.params["max_hours"], Nyears, ) @@ -269,7 +269,7 @@ if __name__ == "__main__": for o in opts: m = re.match(r"^\d+seg$", o, re.IGNORECASE) if m is not None: - solver_name = snakemake.params["solving"]["solver"]["name"] + solver_name = snakemake.params["solver_name"] n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) break @@ -277,11 +277,11 @@ if __name__ == "__main__": if "Co2L" in o: m = re.findall("[0-9]*\.?[0-9]+$", o) if len(m) > 0: - co2limit = float(m[0]) * snakemake.params["electricity"]["co2base"] + co2limit = float(m[0]) * snakemake.params["co2base"] add_co2limit(n, co2limit, Nyears) logger.info("Setting CO2 limit according to wildcard value.") else: - add_co2limit(n, snakemake.params["electricity"]["co2limit"], Nyears) + add_co2limit(n, snakemake.params["co2limit"], Nyears) logger.info("Setting CO2 limit according to config value.") break @@ -293,7 +293,7 @@ if __name__ == "__main__": add_gaslimit(n, limit, Nyears) logger.info("Setting gas usage limit according to wildcard value.") else: - add_gaslimit(n, snakemake.params["electricity"].get("gaslimit"), Nyears) + add_gaslimit(n, snakemake.params["gaslimit"], Nyears) logger.info("Setting gas usage limit according to config value.") break diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 6a71b1e2..54ab1f94 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -252,7 +252,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries, input_eurostat, opts, emissions_scope, report_year, year=2018 ) - planning_horizons = snakemake.params["scenario"]["planning_horizons"] + planning_horizons = snakemake.params["planning_horizons"] t_0 = planning_horizons[0] if "be" in o: @@ -391,7 +391,7 @@ def update_wind_solar_costs(n, costs): with xr.open_dataset(profile) as ds: underwater_fraction = ds["underwater_fraction"].to_pandas() connection_cost = ( - snakemake.params["lines"]["length_factor"] + snakemake.params["length_factor"] * ds["average_distance"].to_pandas() * ( underwater_fraction @@ -3300,7 +3300,7 @@ if __name__ == "__main__": if snakemake.params["foresight"] == "myopic": add_lifetime_wind_solar(n, costs) - conventional = snakemake.params["existing_capacities"]["conventional_carriers"] + conventional = snakemake.params["conventional_carriers"] for carrier in conventional: add_carrier_buses(n, carrier) @@ -3365,7 +3365,7 @@ if __name__ == "__main__": if options["allam_cycle"]: add_allam(n, costs) - solver_name = snakemake.params["solving"]["solver"]["name"] + solver_name = snakemake.params["solver_name"] n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" @@ -3376,8 +3376,8 @@ if __name__ == "__main__": limit_type = "carbon budget" fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): - emissions_scope = snakemake.params["energy"]["emissions"] - report_year = snakemake.params["energy"]["eurostat_report_year"] + emissions_scope = snakemake.params["emissions_scope"] + report_year = snakemake.params["report_year"] build_carbon_budget( o, snakemake.input.eurostat, fn, emissions_scope, report_year ) @@ -3413,7 +3413,7 @@ if __name__ == "__main__": add_electricity_grid_connection(n, costs) first_year_myopic = (snakemake.params["foresight"] == "myopic") and ( - snakemake.params["scenario"]["planning_horizons"][0] == investment_year + snakemake.params["planning_horizons"][0] == investment_year ) if options.get("cluster_heat_buses", False) and not first_year_myopic: diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 689e8084..83e932a3 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -560,7 +560,7 @@ if __name__ == "__main__": technology_costs = load_costs( snakemake.input.tech_costs, snakemake.params["costs"], - snakemake.params["electricity"], + snakemake.params["max_hours"], Nyears, ) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index c268e2ee..421bc515 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -41,7 +41,7 @@ if __name__ == "__main__": opts = (snakemake.wildcards.opts + "-" + snakemake.wildcards.sector_opts).split("-") opts = [o for o in opts if o != ""] - solve_opts = snakemake.params["solving"]["options"] + solve_opts = snakemake.params["options"] np.random.seed(solve_opts.get("seed", 123)) From f3e8fe23122bb47e2d8dfb4c8264e920007ee20b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 27 May 2023 13:50:58 +0000 Subject: [PATCH 64/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_sector.smk | 100 ++++++++++++++++++++++++------------- scripts/cluster_network.py | 4 +- 2 files changed, 65 insertions(+), 39 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index da8a60f9..2d6c8263 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -406,24 +406,38 @@ rule build_ammonia_production: rule build_industry_sector_ratios: params: industry={ - "H2_DRI":config["industry"]["H2_DRI"], - "elec_DRI":config["industry"]["elec_DRI"], - "HVC_production_today":config["industry"]["HVC_production_today"], - "petrochemical_process_emissions":config["industry"]["petrochemical_process_emissions"], - "NH3_process_emissions":config["industry"]["NH3_process_emissions"], - "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"], - "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"], - "chlorine_production_today":config["industry"]["chlorine_production_today"], - "MWh_H2_per_tCl":config["industry"]["MWh_H2_per_tCl"], - "MWh_elec_per_tCl":config["industry"]["MWh_elec_per_tCl"], - "methanol_production_today":config["industry"]["methanol_production_today"], - "MWh_CH4_per_tMeOH":config["industry"]["MWh_CH4_per_tMeOH"], - "MWh_elec_per_tMeOH":config["industry"]["MWh_elec_per_tMeOH"], - "MWh_elec_per_tHVC_mechanical_recycling":config["industry"]["MWh_elec_per_tHVC_mechanical_recycling"], - "MWh_elec_per_tHVC_chemical_recycling":config["industry"]["MWh_elec_per_tHVC_chemical_recycling"], - "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"], - "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"], - "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"] + "H2_DRI": config["industry"]["H2_DRI"], + "elec_DRI": config["industry"]["elec_DRI"], + "HVC_production_today": config["industry"]["HVC_production_today"], + "petrochemical_process_emissions": config["industry"][ + "petrochemical_process_emissions" + ], + "NH3_process_emissions": config["industry"]["NH3_process_emissions"], + "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"], + "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"], + "chlorine_production_today": config["industry"][ + "chlorine_production_today" + ], + "MWh_H2_per_tCl": config["industry"]["MWh_H2_per_tCl"], + "MWh_elec_per_tCl": config["industry"]["MWh_elec_per_tCl"], + "methanol_production_today": config["industry"][ + "methanol_production_today" + ], + "MWh_CH4_per_tMeOH": config["industry"]["MWh_CH4_per_tMeOH"], + "MWh_elec_per_tMeOH": config["industry"]["MWh_elec_per_tMeOH"], + "MWh_elec_per_tHVC_mechanical_recycling": config["industry"][ + "MWh_elec_per_tHVC_mechanical_recycling" + ], + "MWh_elec_per_tHVC_chemical_recycling": config["industry"][ + "MWh_elec_per_tHVC_chemical_recycling" + ], + "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], + "MWh_H2_per_tNH3_electrolysis": config["industry"][ + "MWh_H2_per_tNH3_electrolysis" + ], + "MWh_elec_per_tNH3_electrolysis": config["industry"][ + "MWh_elec_per_tNH3_electrolysis" + ], }, sector_amonia=config["sector"].get("ammonia", False), input: @@ -447,10 +461,14 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: params: industry={ - "reference_year":config["industry"]["reference_year"], - "HVC_production_today":config["industry"]["HVC_production_today"], - "chlorine_production_today":config["industry"]["chlorine_production_today"], - "methanol_production_today":config["industry"]["methanol_production_today"] + "reference_year": config["industry"]["reference_year"], + "HVC_production_today": config["industry"]["HVC_production_today"], + "chlorine_production_today": config["industry"][ + "chlorine_production_today" + ], + "methanol_production_today": config["industry"][ + "methanol_production_today" + ], }, countries=config["countries"], input: @@ -476,12 +494,16 @@ rule build_industrial_production_per_country: rule build_industrial_production_per_country_tomorrow: params: industry={ - "St_primary_fraction":config["industry"]["St_primary_fraction"], - "DRI_fraction":config["industry"]["DRI_fraction"], - "Al_primary_fraction":config["industry"]["Al_primary_fraction"], - "HVC_mechanical_recycling_fraction":config["industry"]["HVC_mechanical_recycling_fraction"], - "HVC_chemical_recycling_fraction":config["industry"]["HVC_chemical_recycling_fraction"], - "HVC_primary_fraction":config["industry"]["HVC_primary_fraction"] + "St_primary_fraction": config["industry"]["St_primary_fraction"], + "DRI_fraction": config["industry"]["DRI_fraction"], + "Al_primary_fraction": config["industry"]["Al_primary_fraction"], + "HVC_mechanical_recycling_fraction": config["industry"][ + "HVC_mechanical_recycling_fraction" + ], + "HVC_chemical_recycling_fraction": config["industry"][ + "HVC_chemical_recycling_fraction" + ], + "HVC_primary_fraction": config["industry"]["HVC_primary_fraction"], }, input: industrial_production_per_country=RESOURCES @@ -587,10 +609,10 @@ rule build_industrial_energy_demand_per_country_today: params: countries=config["countries"], industry={ - "reference_year":config["industry"].get("reference_year", 2015), - "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"], - "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"], - "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"] + "reference_year": config["industry"].get("reference_year", 2015), + "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"], + "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"], + "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], }, input: jrc="data/jrc-idees-2015", @@ -747,10 +769,16 @@ rule prepare_sector_network: costs=config["costs"], sector=config["sector"], industry={ - "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"], - "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"], - "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"], - "MWh_NH3_per_MWh_H2_cracker":config["industry"]["MWh_NH3_per_MWh_H2_cracker"] + "MWh_elec_per_tNH3_electrolysis": config["industry"][ + "MWh_elec_per_tNH3_electrolysis" + ], + "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], + "MWh_H2_per_tNH3_electrolysis": config["industry"][ + "MWh_H2_per_tNH3_electrolysis" + ], + "MWh_NH3_per_MWh_H2_cracker": config["industry"][ + "MWh_NH3_per_MWh_H2_cracker" + ], }, pypsa_eur=config["pypsa_eur"], length_factor=config["lines"]["length_factor"], diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 3cfbd214..b86bb225 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -478,9 +478,7 @@ if __name__ == "__main__": aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers) if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) - conventional = set( - snakemake.params["conventional_carriers"] - ) + conventional = set(snakemake.params["conventional_carriers"]) aggregate_carriers = conventional.intersection(aggregate_carriers) elif snakemake.wildcards.clusters == "all": n_clusters = len(n.buses) From b67a00903af6b5d55019da50bd845a217e07d628 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 31 May 2023 10:54:27 +0200 Subject: [PATCH 65/78] using higher keys for industries --- rules/build_electricity.smk | 1 - rules/build_sector.smk | 79 +++---------------------------------- 2 files changed, 5 insertions(+), 75 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 9b3a03fb..34276ba4 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -255,7 +255,6 @@ rule build_hydro_profile: params: hydro=config["renewable"]["hydro"], countries=config["countries"], - renewable=config["renewable"], input: country_shapes=RESOURCES + "country_shapes.geojson", eia_hydro_generation="data/eia_hydro_annual_generation.csv", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 2d6c8263..3714cbad 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -405,40 +405,7 @@ rule build_ammonia_production: rule build_industry_sector_ratios: params: - industry={ - "H2_DRI": config["industry"]["H2_DRI"], - "elec_DRI": config["industry"]["elec_DRI"], - "HVC_production_today": config["industry"]["HVC_production_today"], - "petrochemical_process_emissions": config["industry"][ - "petrochemical_process_emissions" - ], - "NH3_process_emissions": config["industry"]["NH3_process_emissions"], - "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"], - "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"], - "chlorine_production_today": config["industry"][ - "chlorine_production_today" - ], - "MWh_H2_per_tCl": config["industry"]["MWh_H2_per_tCl"], - "MWh_elec_per_tCl": config["industry"]["MWh_elec_per_tCl"], - "methanol_production_today": config["industry"][ - "methanol_production_today" - ], - "MWh_CH4_per_tMeOH": config["industry"]["MWh_CH4_per_tMeOH"], - "MWh_elec_per_tMeOH": config["industry"]["MWh_elec_per_tMeOH"], - "MWh_elec_per_tHVC_mechanical_recycling": config["industry"][ - "MWh_elec_per_tHVC_mechanical_recycling" - ], - "MWh_elec_per_tHVC_chemical_recycling": config["industry"][ - "MWh_elec_per_tHVC_chemical_recycling" - ], - "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], - "MWh_H2_per_tNH3_electrolysis": config["industry"][ - "MWh_H2_per_tNH3_electrolysis" - ], - "MWh_elec_per_tNH3_electrolysis": config["industry"][ - "MWh_elec_per_tNH3_electrolysis" - ], - }, + industry=config["industry"], sector_amonia=config["sector"].get("ammonia", False), input: ammonia_production=RESOURCES + "ammonia_production.csv", @@ -460,16 +427,7 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: params: - industry={ - "reference_year": config["industry"]["reference_year"], - "HVC_production_today": config["industry"]["HVC_production_today"], - "chlorine_production_today": config["industry"][ - "chlorine_production_today" - ], - "methanol_production_today": config["industry"][ - "methanol_production_today" - ], - }, + industry=config["industry"], countries=config["countries"], input: ammonia_production=RESOURCES + "ammonia_production.csv", @@ -493,18 +451,7 @@ rule build_industrial_production_per_country: rule build_industrial_production_per_country_tomorrow: params: - industry={ - "St_primary_fraction": config["industry"]["St_primary_fraction"], - "DRI_fraction": config["industry"]["DRI_fraction"], - "Al_primary_fraction": config["industry"]["Al_primary_fraction"], - "HVC_mechanical_recycling_fraction": config["industry"][ - "HVC_mechanical_recycling_fraction" - ], - "HVC_chemical_recycling_fraction": config["industry"][ - "HVC_chemical_recycling_fraction" - ], - "HVC_primary_fraction": config["industry"]["HVC_primary_fraction"], - }, + industry=config["industry"], input: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -608,12 +555,7 @@ rule build_industrial_energy_demand_per_node: rule build_industrial_energy_demand_per_country_today: params: countries=config["countries"], - industry={ - "reference_year": config["industry"].get("reference_year", 2015), - "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"], - "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"], - "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], - }, + industry=config["industry"], input: jrc="data/jrc-idees-2015", ammonia_production=RESOURCES + "ammonia_production.csv", @@ -768,18 +710,7 @@ rule prepare_sector_network: foresight=config["foresight"], costs=config["costs"], sector=config["sector"], - industry={ - "MWh_elec_per_tNH3_electrolysis": config["industry"][ - "MWh_elec_per_tNH3_electrolysis" - ], - "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], - "MWh_H2_per_tNH3_electrolysis": config["industry"][ - "MWh_H2_per_tNH3_electrolysis" - ], - "MWh_NH3_per_MWh_H2_cracker": config["industry"][ - "MWh_NH3_per_MWh_H2_cracker" - ], - }, + industry=config["industry"], pypsa_eur=config["pypsa_eur"], length_factor=config["lines"]["length_factor"], planning_horizons=config["scenario"]["planning_horizons"], From 83ff639ad57f40f7a61027951843cd7e4b334642 Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Fri, 2 Jun 2023 12:52:49 +0200 Subject: [PATCH 66/78] amend solve_network --- rules/solve_electricity.smk | 13 +++---------- rules/solve_myopic.smk | 13 +++---------- rules/solve_overnight.smk | 13 +++---------- scripts/solve_network.py | 25 +++++++++++++++---------- 4 files changed, 24 insertions(+), 40 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 892415c4..e7b264f7 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -6,16 +6,9 @@ rule solve_network: params: solving=config["solving"], - config_parts={ - "foresight": config["foresight"], - "planning_horizons": config["scenario"]["planning_horizons"], - "co2_sequestration_potential": config["sector"].get( - "co2_sequestration_potential", 200 - ), - "H2_retrofit_capacity_per_CH4": config["sector"][ - "H2_retrofit_capacity_per_CH4" - ], - }, + foresight=config["foresight"], + planning_horizons=config["scenario"]["planning_horizons"], + co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 8ec1c4a8..39567b92 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -85,16 +85,9 @@ ruleorder: add_existing_baseyear > add_brownfield rule solve_sector_network_myopic: params: solving=config["solving"], - config_parts={ - "foresight": config["foresight"], - "planning_horizons": config["scenario"]["planning_horizons"], - "co2_sequestration_potential": config["sector"].get( - "co2_sequestration_potential", 200 - ), - "H2_retrofit_capacity_per_CH4": config["sector"][ - "H2_retrofit_capacity_per_CH4" - ], - }, + foresight=config["foresight"], + planning_horizons=config["scenario"]["planning_horizons"], + co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index a523f132..71ef075e 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -6,16 +6,9 @@ rule solve_sector_network: params: solving=config["solving"], - config_parts={ - "foresight": config["foresight"], - "planning_horizons": config["scenario"]["planning_horizons"], - "co2_sequestration_potential": config["sector"].get( - "co2_sequestration_potential", 200 - ), - "H2_retrofit_capacity_per_CH4": config["sector"][ - "H2_retrofit_capacity_per_CH4" - ], - }, + foresight=config["foresight"], + planning_horizons=config["scenario"]["planning_horizons"], + co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 6e07e340..90ff3a2e 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -44,14 +44,14 @@ pypsa.pf.logger.setLevel(logging.WARNING) from pypsa.descriptors import get_switchable_as_dense as get_as_dense -def add_land_use_constraint(n, param, config): +def add_land_use_constraint(n, planning_horizons_param, config): if "m" in snakemake.wildcards.clusters: - _add_land_use_constraint_m(n, param, config) + _add_land_use_constraint_m(n, planning_horizons_param, config) else: - _add_land_use_constraint(n, param) + _add_land_use_constraint(n) -def _add_land_use_constraint(n, param): +def _add_land_use_constraint(n): # warning: this will miss existing offwind which is not classed AC-DC and has carrier 'offwind' for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]: @@ -80,7 +80,7 @@ def _add_land_use_constraint(n, param): n.generators.p_nom_max.clip(lower=0, inplace=True) -def _add_land_use_constraint_m(n, param, config): +def _add_land_use_constraint_m(n, planning_horizons_param, config): # if generators clustering is lower than network clustering, land_use accounting is at generators clusters planning_horizons = param["planning_horizons"] @@ -141,7 +141,7 @@ def add_co2_sequestration_limit(n, limit=200): ) -def prepare_network(n, solve_opts=None, config=None, param=None): +def prepare_network(n, solve_opts=None, config=None, foresight_param=None, planning_horizons_param=None, co2_sequestration_potential=None): if "clip_p_max_pu" in solve_opts: for df in ( n.generators_t.p_max_pu, @@ -191,11 +191,11 @@ def prepare_network(n, solve_opts=None, config=None, param=None): n.set_snapshots(n.snapshots[:nhours]) n.snapshot_weightings[:] = 8760.0 / nhours - if param["foresight"] == "myopic": - add_land_use_constraint(n, param, config) + if foresight_param == "myopic": + add_land_use_constraint(n, planning_horizons_param, config) if n.stores.carrier.eq("co2 stored").any(): - limit = param["co2_sequestration_potential"] + limit = co2_sequestration_potential_param add_co2_sequestration_limit(n, limit=limit) return n @@ -676,7 +676,12 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.network) n = prepare_network( - n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"] + n, + solve_opts, + config=snakemake.config, + foresight_param=snakemake.params["foresight"], + planning_horizons_param=snakemake.params["planning_horizons"], + co2_sequestration_potential=snakemake.params["co2_sequestration_potential"] ) n = solve_network( From 0f5bb8b893afc4955f608d9a206035f63e2bb6fe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Jun 2023 10:53:09 +0000 Subject: [PATCH 67/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/solve_electricity.smk | 4 +++- rules/solve_myopic.smk | 4 +++- rules/solve_overnight.smk | 4 +++- scripts/solve_network.py | 11 +++++++++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index e7b264f7..f73a5b0c 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -8,7 +8,9 @@ rule solve_network: solving=config["solving"], foresight=config["foresight"], planning_horizons=config["scenario"]["planning_horizons"], - co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), + co2_sequestration_potential=config["sector"].get( + "co2_sequestration_potential", 200 + ), input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 39567b92..086375c2 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -87,7 +87,9 @@ rule solve_sector_network_myopic: solving=config["solving"], foresight=config["foresight"], planning_horizons=config["scenario"]["planning_horizons"], - co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), + co2_sequestration_potential=config["sector"].get( + "co2_sequestration_potential", 200 + ), input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 71ef075e..fc430f4d 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -8,7 +8,9 @@ rule solve_sector_network: solving=config["solving"], foresight=config["foresight"], planning_horizons=config["scenario"]["planning_horizons"], - co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), + co2_sequestration_potential=config["sector"].get( + "co2_sequestration_potential", 200 + ), input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 90ff3a2e..ab1175ab 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -141,7 +141,14 @@ def add_co2_sequestration_limit(n, limit=200): ) -def prepare_network(n, solve_opts=None, config=None, foresight_param=None, planning_horizons_param=None, co2_sequestration_potential=None): +def prepare_network( + n, + solve_opts=None, + config=None, + foresight_param=None, + planning_horizons_param=None, + co2_sequestration_potential=None, +): if "clip_p_max_pu" in solve_opts: for df in ( n.generators_t.p_max_pu, @@ -681,7 +688,7 @@ if __name__ == "__main__": config=snakemake.config, foresight_param=snakemake.params["foresight"], planning_horizons_param=snakemake.params["planning_horizons"], - co2_sequestration_potential=snakemake.params["co2_sequestration_potential"] + co2_sequestration_potential=snakemake.params["co2_sequestration_potential"], ) n = solve_network( From 8dbcca560cd47d58d098dd91e751efd45314b8e7 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Sat, 3 Jun 2023 12:09:51 +0200 Subject: [PATCH 68/78] minor fixes in solve_network.py --- scripts/solve_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ab1175ab..ea5446ad 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -147,7 +147,7 @@ def prepare_network( config=None, foresight_param=None, planning_horizons_param=None, - co2_sequestration_potential=None, + co2_sequestration_potential_param=None, ): if "clip_p_max_pu" in solve_opts: for df in ( @@ -688,7 +688,7 @@ if __name__ == "__main__": config=snakemake.config, foresight_param=snakemake.params["foresight"], planning_horizons_param=snakemake.params["planning_horizons"], - co2_sequestration_potential=snakemake.params["co2_sequestration_potential"], + co2_sequestration_potential_param=snakemake.params["co2_sequestration_potential"], ) n = solve_network( From d6057cb92be6fdb07d02da23e13035b90a57d1fc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 3 Jun 2023 10:10:06 +0000 Subject: [PATCH 69/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/solve_network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ea5446ad..41679a03 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -688,7 +688,9 @@ if __name__ == "__main__": config=snakemake.config, foresight_param=snakemake.params["foresight"], planning_horizons_param=snakemake.params["planning_horizons"], - co2_sequestration_potential_param=snakemake.params["co2_sequestration_potential"], + co2_sequestration_potential_param=snakemake.params[ + "co2_sequestration_potential" + ], ) n = solve_network( From 176bebdef0f12de8c639c83e70a9b949ae5c318e Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Fri, 9 Jun 2023 17:18:25 +0200 Subject: [PATCH 70/78] Update cluster_network to avoid adding deleted links in clustered network --- scripts/cluster_network.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 7572d3b3..d5d27ef7 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -424,7 +424,10 @@ def clustering_for_n_clusters( n.links.eval("underwater_fraction * length").div(nc.links.length).dropna() ) nc.links["capital_cost"] = nc.links["capital_cost"].add( - (nc.links.length - n.links.length).clip(lower=0).mul(extended_link_costs), + (nc.links.length - n.links.length) + .clip(lower=0) + .mul(extended_link_costs) + .dropna(), fill_value=0, ) From e44f8d66cdb2abb62a9ef59f1141b1895ce8d167 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 22:22:08 +0000 Subject: [PATCH 71/78] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/docformatter: v1.7.1 → v1.7.2](https://github.com/PyCQA/docformatter/compare/v1.7.1...v1.7.2) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34d13746..826a4819 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: # Make docstrings PEP 257 compliant - repo: https://github.com/PyCQA/docformatter - rev: v1.7.1 + rev: v1.7.2 hooks: - id: docformatter args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] From 12b93562453ccce90df248c2686040c984a1612b Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Wed, 14 Jun 2023 11:03:07 +0200 Subject: [PATCH 72/78] add info to release_notes --- doc/release_notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 3af16477..8f26ac02 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,7 @@ Release Notes Upcoming Release ================ +* ``param:`` section in rule definition are added to track changed settings in ``config.yaml``. The goal is to automatically re-execute rules whose parameters have changed. See `Non-file parameters for rules `_ in the snakemake documentation. * **Important:** The configuration files are now located in the ``config`` directory. This counts for ``config.default.yaml``, ``config.yaml`` as well as the test configuration files which are now located in ``config/test``. Config files that are still in the root directory will be ignored. From 45cac01ea3c49529c4b68714e783bedb60565f00 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 14 Jun 2023 13:28:20 +0200 Subject: [PATCH 73/78] adjust param accessing in cluster, simplify and build_elec script --- rules/build_electricity.smk | 24 ++--- scripts/add_electricity.py | 40 ++++----- scripts/build_renewable_profiles.py | 2 +- scripts/cluster_network.py | 56 +++++------- scripts/simplify_network.py | 131 ++++++++++++---------------- 5 files changed, 109 insertions(+), 144 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 34276ba4..fd66cc04 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -316,16 +316,14 @@ rule add_electricity: rule simplify_network: params: - clustering=config["clustering"], - max_hours=config["electricity"]["max_hours"], + simplify_network=config["clustering"]["simplify_network"], + aggregation_strategies=config["clustering"].get("aggregation_strategies", {}), + focus_weights=config.get("focus_weights", None), + renewable_carriers=config["electricity"]["renewable_carriers"], costs=config["costs"], - renewable=config["renewable"], + max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], p_max_pu=config["links"].get("p_max_pu", 1.0), - exclude_carriers=config["clustering"]["simplify_network"].get( - "exclude_carriers", [] - ), - focus_weights=config.get("focus_weights", None), solver_name=config["solving"]["solver"]["name"], input: network=RESOURCES + "networks/elec.nc", @@ -353,14 +351,16 @@ rule simplify_network: rule cluster_network: params: - solver_name=config["solving"]["solver"]["name"], - max_hours=config["electricity"]["max_hours"], + cluster_network=config["clustering"]["cluster_network"], + aggregation_strategies=config["clustering"].get("aggregation_strategies", {}), + custom_busmap=config["enable"].get("custom_busmap", False), + focus_weights=config.get("focus_weights", None), + renewable_carriers=config["electricity"]["renewable_carriers"], conventional_carriers=config["electricity"].get("conventional_carriers", []), costs=config["costs"], + max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], - renewable=config["renewable"], - clustering=config["clustering"], - custom_busmap=config["enable"].get("custom_busmap", False), + solver_name=config["solving"]["solver"]["name"], input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index dc5fb7cc..34147af3 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -726,23 +726,23 @@ if __name__ == "__main__": costs = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["electricity"]["max_hours"], + snakemake.params.costs, + snakemake.params.electricity["max_hours"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in snakemake.params["electricity"]: - renewable_carriers = set(snakemake.params["electricity"]["renewable_carriers"]) + if "renewable_carriers" in snakemake.params.electricity: + renewable_carriers = set(snakemake.params.electricity["renewable_carriers"]) else: logger.warning( "Missing key `renewable_carriers` under config entry `electricity`. " "In future versions, this will raise an error. " "Falling back to carriers listed under `renewable`." ) - renewable_carriers = snakemake.params["renewable"] + renewable_carriers = snakemake.params.renewable - extendable_carriers = snakemake.params["electricity"]["extendable_carriers"] + extendable_carriers = snakemake.params.electricity["extendable_carriers"] if not (set(renewable_carriers) & set(extendable_carriers["Generator"])): logger.warning( "No renewables found in config entry `extendable_carriers`. " @@ -750,18 +750,18 @@ if __name__ == "__main__": "Falling back to all renewables." ) - conventional_carriers = snakemake.params["electricity"]["conventional_carriers"] + conventional_carriers = snakemake.params.electricity["conventional_carriers"] attach_load( n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.params["countries"], - snakemake.params["scaling_factor"], + snakemake.params.countries, + snakemake.params.scaling_factor, ) - update_transmission_costs(n, costs, snakemake.params["length_factor"]) + update_transmission_costs(n, costs, snakemake.params.length_factor) conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") @@ -772,7 +772,7 @@ if __name__ == "__main__": ppl, conventional_carriers, extendable_carriers, - snakemake.params["conventional"], + snakemake.params.conventional, conventional_inputs, ) @@ -782,11 +782,11 @@ if __name__ == "__main__": snakemake.input, renewable_carriers, extendable_carriers, - snakemake.params["length_factor"], + snakemake.params.length_factor, ) if "hydro" in renewable_carriers: - para = snakemake.params["renewable"]["hydro"] + para = snakemake.params.renewable["hydro"] attach_hydro( n, costs, @@ -797,7 +797,7 @@ if __name__ == "__main__": **para, ) - if "estimate_renewable_capacities" not in snakemake.params["electricity"]: + if "estimate_renewable_capacities" not in snakemake.params.electricity: logger.warning( "Missing key `estimate_renewable_capacities` under config entry `electricity`. " "In future versions, this will raise an error. " @@ -805,18 +805,18 @@ if __name__ == "__main__": ) if ( "estimate_renewable_capacities_from_capacity_stats" - in snakemake.params["electricity"] + in snakemake.params.electricity ): estimate_renewable_caps = { "enable": True, - **snakemake.params["electricity"][ + **snakemake.params.electricity[ "estimate_renewable_capacities_from_capacity_stats" ], } else: estimate_renewable_caps = {"enable": False} else: - estimate_renewable_caps = snakemake.params["electricity"][ + estimate_renewable_caps = snakemake.params.electricity[ "estimate_renewable_capacities" ] if "enable" not in estimate_renewable_caps: @@ -832,18 +832,18 @@ if __name__ == "__main__": "Falling back to whether `renewable_capacities_from_opsd` is non-empty." ) from_opsd = bool( - snakemake.params["electricity"].get("renewable_capacities_from_opsd", False) + snakemake.params.electricity.get("renewable_capacities_from_opsd", False) ) estimate_renewable_caps["from_opsd"] = from_opsd if estimate_renewable_caps["enable"]: if estimate_renewable_caps["from_opsd"]: - tech_map = snakemake.params["electricity"]["estimate_renewable_capacities"][ + tech_map = snakemake.params.electricity["estimate_renewable_capacities"][ "technology_mapping" ] attach_OPSD_renewables(n, tech_map) estimate_renewable_capacities( - n, snakemake.params["electricity"], snakemake.params["countries"] + n, snakemake.params.electricity, snakemake.params.countries ) update_p_nom_max(n) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 91463c23..9e90d42b 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -309,7 +309,7 @@ if __name__ == "__main__": p_nom_max = capacities / max_cap_factor else: raise AssertionError( - 'params key `potential` should be one of "simple" ' + 'Config key `potential` should be one of "simple" ' f'(default) or "conservative", not "{p_nom_max_meth}"' ) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index db0fb2c2..52685af2 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -186,7 +186,7 @@ def get_feature_for_hac(n, buses_i=None, feature=None): if "offwind" in carriers: carriers.remove("offwind") carriers = np.append( - carriers, network.generators.carrier.filter(like="offwind").unique() + carriers, n.generators.carrier.filter(like="offwind").unique() ) if feature.split("-")[1] == "cap": @@ -463,26 +463,18 @@ if __name__ == "__main__": snakemake = mock_snakemake("cluster_network", simpl="", clusters="5") configure_logging(snakemake) + params = snakemake.params + solver_name = snakemake.config["solving"]["solver"]["name"] + n = pypsa.Network(snakemake.input.network) - focus_weights = snakemake.config.get("focus_weights", None) - - renewable_carriers = pd.Index( - [ - tech - for tech in n.generators.carrier.unique() - if tech in snakemake.params["renewable"] - ] - ) - - exclude_carriers = snakemake.params["clustering"]["cluster_network"].get( - "exclude_carriers", [] - ) + exclude_carriers = params.cluster_network["exclude_carriers"] aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers) if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) - conventional = set(snakemake.params["conventional_carriers"]) - aggregate_carriers = conventional.intersection(aggregate_carriers) + aggregate_carriers = set(params.conventional_carriers).intersection( + aggregate_carriers + ) elif snakemake.wildcards.clusters == "all": n_clusters = len(n.buses) else: @@ -496,13 +488,12 @@ if __name__ == "__main__": n, busmap, linemap, linemap, pd.Series(dtype="O") ) else: - line_length_factor = snakemake.params["length_factor"] Nyears = n.snapshot_weightings.objective.sum() / 8760 hvac_overhead_cost = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["max_hours"], + params.costs, + params.max_hours, Nyears, ).at["HVAC overhead", "capital_cost"] @@ -513,16 +504,16 @@ if __name__ == "__main__": ).all() or x.isnull().all(), "The `potential` configuration option must agree for all renewable carriers, for now!" return v - aggregation_strategies = snakemake.params["clustering"].get( - "aggregation_strategies", {} - ) # translate str entries of aggregation_strategies to pd.Series functions: aggregation_strategies = { - p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()} - for p in aggregation_strategies.keys() + p: { + k: getattr(pd.Series, v) + for k, v in params.aggregation_strategies[p].items() + } + for p in params.aggregation_strategies.keys() } - custom_busmap = snakemake.params["custom_busmap"] + custom_busmap = params.custom_busmap if custom_busmap: custom_busmap = pd.read_csv( snakemake.input.custom_busmap, index_col=0, squeeze=True @@ -530,21 +521,18 @@ if __name__ == "__main__": custom_busmap.index = custom_busmap.index.astype(str) logger.info(f"Imported custom busmap from {snakemake.input.custom_busmap}") - cluster_config = snakemake.config.get("clustering", {}).get( - "cluster_network", {} - ) clustering = clustering_for_n_clusters( n, n_clusters, custom_busmap, aggregate_carriers, - line_length_factor, - aggregation_strategies, - snakemake.params["solver_name"], - cluster_config.get("algorithm", "hac"), - cluster_config.get("feature", "solar+onwind-time"), + params.length_factor, + params.aggregation_strategies, + solver_name, + params.cluster_network["algorithm"], + params.cluster_network["feature"], hvac_overhead_cost, - focus_weights, + params.focus_weights, ) update_p_nom_max(clustering.network) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 83e932a3..79161760 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -149,17 +149,17 @@ def simplify_network_to_380(n): return n, trafo_map -def _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param): +def _prepare_connection_costs_per_link(n, costs, renewable_carriers, length_factor): if n.links.empty: return {} connection_costs_per_link = {} - for tech in renewable_param: + for tech in renewable_carriers: if tech.startswith("offwind"): connection_costs_per_link[tech] = ( n.links.length - * length_factor_param + * length_factor * ( n.links.underwater_fraction * costs.at[tech + "-connection-submarine", "capital_cost"] @@ -175,14 +175,14 @@ def _compute_connection_costs_to_bus( n, busmap, costs, - renewable_param, - length_factor_param, + renewable_carriers, + length_factor, connection_costs_per_link=None, buses=None, ): if connection_costs_per_link is None: connection_costs_per_link = _prepare_connection_costs_per_link( - n, costs, renewable_param, length_factor_param + n, costs, renewable_carriers, length_factor ) if buses is None: @@ -276,10 +276,10 @@ def _aggregate_and_move_components( def simplify_links( n, costs, - renewable_param, - length_factor_param, - p_max_pu_param, - exclude_carriers_param, + renewables, + length_factor, + p_max_pu, + exclude_carriers, output, aggregation_strategies=dict(), ): @@ -333,7 +333,7 @@ def simplify_links( busmap = n.buses.index.to_series() connection_costs_per_link = _prepare_connection_costs_per_link( - n, costs, renewable_param, length_factor_param + n, costs, renewables, length_factor ) connection_costs_to_bus = pd.DataFrame( 0.0, index=n.buses.index, columns=list(connection_costs_per_link) @@ -355,8 +355,8 @@ def simplify_links( n, busmap, costs, - renewable_param, - length_factor_param, + renewables, + length_factor, connection_costs_per_link, buses, ) @@ -378,8 +378,8 @@ def simplify_links( / lengths.sum() * n.links.loc[all_links, "underwater_fraction"] ), - p_max_pu=p_max_pu_param, - p_min_pu=-p_max_pu_param, + p_max_pu=p_max_pu, + p_min_pu=-p_max_pu, underground=False, under_construction=False, ) @@ -407,7 +407,7 @@ def simplify_links( connection_costs_to_bus, output, aggregation_strategies=aggregation_strategies, - exclude_carriers=exclude_carriers_param, + exclude_carriers=exclude_carriers, ) return n, busmap @@ -415,34 +415,29 @@ def simplify_links( def remove_stubs( n, costs, - renewable_param, - length_factor_param, - clustering_param, - exclude_carriers_param, + renewable_carriers, + length_factor, + simplify_network, output, aggregation_strategies=dict(), ): logger.info("Removing stubs") - across_borders = clustering_param["simplify_network"].get( - "remove_stubs_across_borders", True - ) + across_borders = simplify_network["remove_stubs_across_borders"] matching_attrs = [] if across_borders else ["country"] busmap = busmap_by_stubs(n, matching_attrs) connection_costs_to_bus = _compute_connection_costs_to_bus( - n, busmap, costs, renewable_param, length_factor_param + n, busmap, costs, renewable_carriers, length_factor ) - exclude_carriers = clustering_param["simplify_network"].get("exclude_carriers", []) - _aggregate_and_move_components( n, busmap, connection_costs_to_bus, output, aggregation_strategies=aggregation_strategies, - exclude_carriers=exclude_carriers, + exclude_carriers=simplify_network["exclude_carriers"], ) return n, busmap @@ -504,32 +499,23 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): def cluster( n, n_clusters, - focus_weights_param, - renewable_param, - solver_name_param, + focus_weights, + solver_name, algorithm="hac", feature=None, aggregation_strategies=dict(), ): logger.info(f"Clustering to {n_clusters} buses") - renewable_carriers = pd.Index( - [ - tech - for tech in n.generators.carrier.unique() - if tech.split("-", 2)[0] in renewable_param - ] - ) - clustering = clustering_for_n_clusters( n, n_clusters, custom_busmap=False, aggregation_strategies=aggregation_strategies, - solver_name=solver_name_param, + solver_name=solver_name, algorithm=algorithm, feature=feature, - focus_weights=focus_weights_param, + focus_weights=focus_weights, ) return clustering.network, clustering.busmap @@ -542,77 +528,69 @@ if __name__ == "__main__": snakemake = mock_snakemake("simplify_network", simpl="") configure_logging(snakemake) - n = pypsa.Network(snakemake.input.network) + params = snakemake.params + solver_name = snakemake.config["solving"]["solver"]["name"] + + n = pypsa.Network(snakemake.input.network) + Nyears = n.snapshot_weightings.objective.sum() / 8760 - aggregation_strategies = snakemake.params["clustering"].get( - "aggregation_strategies", {} - ) # translate str entries of aggregation_strategies to pd.Series functions: aggregation_strategies = { - p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()} - for p in aggregation_strategies.keys() + p: { + k: getattr(pd.Series, v) + for k, v in params.aggregation_strategies[p].items() + } + for p in params.aggregation_strategies.keys() } n, trafo_map = simplify_network_to_380(n) - Nyears = n.snapshot_weightings.objective.sum() / 8760 - technology_costs = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["max_hours"], + params.costs, + params.max_hours, Nyears, ) n, simplify_links_map = simplify_links( n, technology_costs, - snakemake.params["renewable"], - snakemake.params["length_factor"], - snakemake.params["p_max_pu"], - snakemake.params["exclude_carriers"], + params.renewable_carriers, + params.length_factor, + params.p_max_pu, + params.simplify_network["exclude_carriers"], snakemake.output, aggregation_strategies, ) busmaps = [trafo_map, simplify_links_map] - cluster_param = snakemake.params["clustering"]["simplify_network"] - if cluster_param.get("remove_stubs", True): + if params.simplify_network["remove_stubs"]: n, stub_map = remove_stubs( n, technology_costs, - snakemake.params["renewable"], - snakemake.params["length_factor"], - snakemake.params["clustering"], - snakemake.params["exclude_carriers"], + params.renewable_carriers, + params.length_factor, + params.simplify_network, snakemake.output, aggregation_strategies=aggregation_strategies, ) busmaps.append(stub_map) - if cluster_param.get("to_substations", False): + if params.simplify_network["to_substations"]: n, substation_map = aggregate_to_substations(n, aggregation_strategies) busmaps.append(substation_map) # treatment of outliers (nodes without a profile for considered carrier): # all nodes that have no profile of the given carrier are being aggregated to closest neighbor - if ( - snakemake.config.get("clustering", {}) - .get("cluster_network", {}) - .get("algorithm", "hac") - == "hac" - or cluster_param.get("algorithm", "hac") == "hac" - ): - carriers = ( - cluster_param.get("feature", "solar+onwind-time").split("-")[0].split("+") - ) + if params.simplify_network["algorithm"] == "hac": + carriers = params.simplify_network["feature"].split("-")[0].split("+") for carrier in carriers: buses_i = list( set(n.buses.index) - set(n.generators.query("carrier == @carrier").bus) ) logger.info( - f"clustering preparaton (hac): aggregating {len(buses_i)} buses of type {carrier}." + f"clustering preparation (hac): aggregating {len(buses_i)} buses of type {carrier}." ) n, busmap_hac = aggregate_to_substations(n, aggregation_strategies, buses_i) busmaps.append(busmap_hac) @@ -621,11 +599,10 @@ if __name__ == "__main__": n, cluster_map = cluster( n, int(snakemake.wildcards.simpl), - snakemake.params["focus_weights"], - snakemake.params["renewable"], - snakemake.params["solver_name"], - cluster_param.get("algorithm", "hac"), - cluster_param.get("feature", None), + params.focus_weights, + solver_name, + params.simplify_network["algorithm"], + params.simplify_network["feature"], aggregation_strategies, ) busmaps.append(cluster_map) From 1d10073514a66f09e6821364f03a57f99cff2352 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 15 Jun 2023 18:52:25 +0200 Subject: [PATCH 74/78] review params implementation; reproduce objective values in CI --- config/test/config.myopic.yaml | 8 ++ config/test/config.overnight.yaml | 8 ++ rules/build_electricity.smk | 2 +- rules/build_sector.smk | 2 +- scripts/add_brownfield.py | 6 +- scripts/add_electricity.py | 93 +++++-------------- scripts/add_existing_baseyear.py | 20 ++-- scripts/add_extra_components.py | 26 +++--- scripts/build_ammonia_production.py | 2 +- scripts/build_biomass_potentials.py | 2 +- scripts/build_bus_regions.py | 2 +- scripts/build_cop_profiles.py | 2 +- scripts/build_cutout.py | 4 +- scripts/build_electricity_demand.py | 12 +-- scripts/build_energy_totals.py | 8 +- scripts/build_heat_demand.py | 2 +- scripts/build_hydro_profile.py | 4 +- scripts/build_industrial_distribution_key.py | 4 +- ...ustrial_energy_demand_per_country_today.py | 4 +- ...build_industrial_production_per_country.py | 6 +- ...ustrial_production_per_country_tomorrow.py | 2 +- scripts/build_industry_sector_ratios.py | 4 +- scripts/build_powerplants.py | 6 +- scripts/build_renewable_profiles.py | 2 +- scripts/build_retro_cost.py | 4 +- scripts/build_sequestration_potentials.py | 2 +- scripts/build_shapes.py | 6 +- scripts/build_solar_thermal_profiles.py | 4 +- scripts/build_temperature_profiles.py | 2 +- scripts/build_transport_demand.py | 4 +- scripts/make_summary.py | 20 ++-- scripts/plot_network.py | 16 ++-- scripts/plot_summary.py | 24 ++--- scripts/prepare_network.py | 20 ++-- scripts/prepare_sector_network.py | 40 ++++---- scripts/retrieve_databundle.py | 2 +- scripts/solve_network.py | 42 ++++----- scripts/solve_operations_network.py | 2 +- 38 files changed, 188 insertions(+), 231 deletions(-) diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index efa03136..0bb85ec6 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -31,6 +31,14 @@ snapshots: end: "2013-03-08" electricity: + co2limit: 100.e+6 + + extendable_carriers: + Generator: [OCGT] + StorageUnit: [battery] + Store: [H2] + Link: [H2 pipeline] + renewable_carriers: [solar, onwind, offwind-ac, offwind-dc] atlite: diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index fb468ded..a2a0f5a4 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -28,6 +28,14 @@ snapshots: end: "2013-03-08" electricity: + co2limit: 100.e+6 + + extendable_carriers: + Generator: [OCGT] + StorageUnit: [battery] + Store: [H2] + Link: [H2 pipeline] + renewable_carriers: [solar, onwind, offwind-ac, offwind-dc] atlite: diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index fd66cc04..9d4315df 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -394,7 +394,7 @@ rule cluster_network: rule add_extra_components: params: costs=config["costs"], - ext_carriers=config["electricity"]["extendable_carriers"], + extendable_carriers=config["electricity"]["extendable_carriers"], max_hours=config["electricity"]["max_hours"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 3714cbad..e15394ff 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -406,7 +406,7 @@ rule build_ammonia_production: rule build_industry_sector_ratios: params: industry=config["industry"], - sector_amonia=config["sector"].get("ammonia", False), + ammonia=config["sector"].get("ammonia", False), input: ammonia_production=RESOURCES + "ammonia_production.csv", idees="data/jrc-idees-2015", diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 48dffbb9..b33d2f19 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -49,7 +49,7 @@ def add_brownfield(n, n_p, year): ) ] - threshold = snakemake.params["threshold_capacity"] + threshold = snakemake.params.threshold_capacity if not chp_heat.empty: threshold_chp_heat = ( @@ -87,7 +87,7 @@ def add_brownfield(n, n_p, year): # deal with gas network pipe_carrier = ["gas pipeline"] - if snakemake.params["H2_retrofit"]: + if snakemake.params.H2_retrofit: # drop capacities of previous year to avoid duplicating to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year) n.mremove("Link", n.links.loc[to_drop].index) @@ -98,7 +98,7 @@ def add_brownfield(n, n_p, year): & (n.links.build_year != year) ].index gas_pipes_i = n.links[n.links.carrier.isin(pipe_carrier)].index - CH4_per_H2 = 1 / snakemake.params["H2_retrofit_capacity_per_CH4"] + CH4_per_H2 = 1 / snakemake.params.H2_retrofit_capacity_per_CH4 fr = "H2 pipeline retrofitted" to = "gas pipeline" # today's pipe capacity diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 34147af3..d98dc767 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -652,13 +652,7 @@ def attach_OPSD_renewables(n, tech_map): n.generators.p_nom_min.update(gens.bus.map(caps).dropna()) -def estimate_renewable_capacities(n, electricity_params, countries): - year = electricity_params["estimate_renewable_capacities"]["year"] - tech_map = electricity_params["estimate_renewable_capacities"]["technology_mapping"] - expansion_limit = electricity_params["estimate_renewable_capacities"][ - "expansion_limit" - ] - +def estimate_renewable_capacities(n, year, tech_map, expansion_limit, countries): if not len(countries) or not len(tech_map): return @@ -721,48 +715,42 @@ if __name__ == "__main__": snakemake = mock_snakemake("add_electricity") configure_logging(snakemake) + params = snakemake.params + n = pypsa.Network(snakemake.input.base_network) Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( snakemake.input.tech_costs, - snakemake.params.costs, - snakemake.params.electricity["max_hours"], + params.costs, + params.electricity["max_hours"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in snakemake.params.electricity: - renewable_carriers = set(snakemake.params.electricity["renewable_carriers"]) + if "renewable_carriers" in params.electricity: + renewable_carriers = set(params.electricity["renewable_carriers"]) else: logger.warning( "Missing key `renewable_carriers` under config entry `electricity`. " "In future versions, this will raise an error. " "Falling back to carriers listed under `renewable`." ) - renewable_carriers = snakemake.params.renewable - - extendable_carriers = snakemake.params.electricity["extendable_carriers"] - if not (set(renewable_carriers) & set(extendable_carriers["Generator"])): - logger.warning( - "No renewables found in config entry `extendable_carriers`. " - "In future versions, these have to be explicitly listed. " - "Falling back to all renewables." - ) - - conventional_carriers = snakemake.params.electricity["conventional_carriers"] + renewable_carriers = params.renewable attach_load( n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.params.countries, - snakemake.params.scaling_factor, + params.countries, + params.scaling_factor, ) - update_transmission_costs(n, costs, snakemake.params.length_factor) + update_transmission_costs(n, costs, params.length_factor) + extendable_carriers = params.electricity["extendable_carriers"] + conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") } @@ -772,7 +760,7 @@ if __name__ == "__main__": ppl, conventional_carriers, extendable_carriers, - snakemake.params.conventional, + params.conventional, conventional_inputs, ) @@ -782,11 +770,11 @@ if __name__ == "__main__": snakemake.input, renewable_carriers, extendable_carriers, - snakemake.params.length_factor, + params.length_factor, ) if "hydro" in renewable_carriers: - para = snakemake.params.renewable["hydro"] + para = params.renewable["hydro"] attach_hydro( n, costs, @@ -797,53 +785,16 @@ if __name__ == "__main__": **para, ) - if "estimate_renewable_capacities" not in snakemake.params.electricity: - logger.warning( - "Missing key `estimate_renewable_capacities` under config entry `electricity`. " - "In future versions, this will raise an error. " - "Falling back to whether ``estimate_renewable_capacities_from_capacity_stats`` is in the config." - ) - if ( - "estimate_renewable_capacities_from_capacity_stats" - in snakemake.params.electricity - ): - estimate_renewable_caps = { - "enable": True, - **snakemake.params.electricity[ - "estimate_renewable_capacities_from_capacity_stats" - ], - } - else: - estimate_renewable_caps = {"enable": False} - else: - estimate_renewable_caps = snakemake.params.electricity[ - "estimate_renewable_capacities" - ] - if "enable" not in estimate_renewable_caps: - logger.warning( - "Missing key `enable` under config entry `estimate_renewable_capacities`. " - "In future versions, this will raise an error. Falling back to False." - ) - estimate_renewable_caps = {"enable": False} - if "from_opsd" not in estimate_renewable_caps: - logger.warning( - "Missing key `from_opsd` under config entry `estimate_renewable_capacities`. " - "In future versions, this will raise an error. " - "Falling back to whether `renewable_capacities_from_opsd` is non-empty." - ) - from_opsd = bool( - snakemake.params.electricity.get("renewable_capacities_from_opsd", False) - ) - estimate_renewable_caps["from_opsd"] = from_opsd - + estimate_renewable_caps = params.electricity["estimate_renewable_capacities"] if estimate_renewable_caps["enable"]: + tech_map = estimate_renewable_caps["technology_mapping"] + expansion_limit = estimate_renewable_caps["expansion_limit"] + year = estimate_renewable_caps["year"] + if estimate_renewable_caps["from_opsd"]: - tech_map = snakemake.params.electricity["estimate_renewable_capacities"][ - "technology_mapping" - ] attach_OPSD_renewables(n, tech_map) estimate_renewable_capacities( - n, snakemake.params.electricity, snakemake.params.countries + n, year, tech_map, expansion_limit, params.countries ) update_p_nom_max(n) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index bc1d3a05..52cb585e 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -157,7 +157,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas # Fill missing DateOut dateout = ( df_agg.loc[biomass_i, "DateIn"] - + snakemake.params["costs"]["fill_values"]["lifetime"] + + snakemake.params.costs["fill_values"]["lifetime"] ) df_agg.loc[biomass_i, "DateOut"] = df_agg.loc[biomass_i, "DateOut"].fillna(dateout) @@ -218,7 +218,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas capacity = df.loc[grouping_year, generator] capacity = capacity[~capacity.isna()] capacity = capacity[ - capacity > snakemake.params["existing_capacities"]["threshold_capacity"] + capacity > snakemake.params.existing_capacities["threshold_capacity"] ] suffix = "-ac" if generator == "offwind" else "" name_suffix = f" {generator}{suffix}-{grouping_year}" @@ -582,7 +582,7 @@ def add_heating_capacities_installed_before_baseyear( ) # delete links with capacities below threshold - threshold = snakemake.params["existing_capacities"]["threshold_capacity"] + threshold = snakemake.params.existing_capacities["threshold_capacity"] n.mremove( "Link", [ @@ -612,10 +612,10 @@ if __name__ == "__main__": update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) - options = snakemake.params["sector"] + options = snakemake.params.sector opts = snakemake.wildcards.sector_opts.split("-") - baseyear = snakemake.params["baseyear"] + baseyear = snakemake.params.baseyear overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) @@ -626,14 +626,12 @@ if __name__ == "__main__": Nyears = n.snapshot_weightings.generators.sum() / 8760.0 costs = prepare_costs( snakemake.input.costs, - snakemake.params["costs"], + snakemake.params.costs, Nyears, ) - grouping_years_power = snakemake.params["existing_capacities"][ - "grouping_years_power" - ] - grouping_years_heat = snakemake.params["existing_capacities"]["grouping_years_heat"] + grouping_years_power = snakemake.params.existing_capacities["grouping_years_power"] + grouping_years_heat = snakemake.params.existing_capacities["grouping_years_heat"] add_power_capacities_installed_before_baseyear( n, grouping_years_power, costs, baseyear ) @@ -650,7 +648,7 @@ if __name__ == "__main__": .to_pandas() .reindex(index=n.snapshots) ) - default_lifetime = snakemake.params["costs"]["fill_values"]["lifetime"] + default_lifetime = snakemake.params.costs["fill_values"]["lifetime"] add_heating_capacities_installed_before_baseyear( n, baseyear, diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index ce4e533e..f034a7f2 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -67,8 +67,8 @@ idx = pd.IndexSlice logger = logging.getLogger(__name__) -def attach_storageunits(n, costs, ext_carriers, max_hours): - carriers = ext_carriers["StorageUnit"] +def attach_storageunits(n, costs, extendable_carriers, max_hours): + carriers = extendable_carriers["StorageUnit"] _add_missing_carriers_from_costs(n, costs, carriers) @@ -98,8 +98,8 @@ def attach_storageunits(n, costs, ext_carriers, max_hours): ) -def attach_stores(n, costs, ext_carriers): - carriers = ext_carriers["Store"] +def attach_stores(n, costs, extendable_carriers): + carriers = extendable_carriers["Store"] _add_missing_carriers_from_costs(n, costs, carriers) @@ -186,10 +186,10 @@ def attach_stores(n, costs, ext_carriers): ) -def attach_hydrogen_pipelines(n, costs, ext_carriers): - as_stores = ext_carriers.get("Store", []) +def attach_hydrogen_pipelines(n, costs, extendable_carriers): + as_stores = extendable_carriers.get("Store", []) - if "H2 pipeline" not in ext_carriers.get("Link", []): + if "H2 pipeline" not in extendable_carriers.get("Link", []): return assert "H2" in as_stores, ( @@ -233,17 +233,17 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) - ext_carriers = snakemake.params["ext_carriers"] - max_hours = snakemake.params["max_hours"] + extendable_carriers = snakemake.params.extendable_carriers + max_hours = snakemake.params.max_hours Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( - snakemake.input.tech_costs, snakemake.params["costs"], max_hours, Nyears + snakemake.input.tech_costs, snakemake.params.costs, max_hours, Nyears ) - attach_storageunits(n, costs, ext_carriers, max_hours) - attach_stores(n, costs, ext_carriers) - attach_hydrogen_pipelines(n, costs, ext_carriers) + attach_storageunits(n, costs, extendable_carriers, max_hours) + attach_stores(n, costs, extendable_carriers) + attach_hydrogen_pipelines(n, costs, extendable_carriers) add_nice_carrier_names(n, snakemake.config) diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index 6f03324f..1bcdf9ae 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -30,7 +30,7 @@ if __name__ == "__main__": ammonia.index = cc.convert(ammonia.index, to="iso2") years = [str(i) for i in range(2013, 2018)] - countries = ammonia.index.intersection(snakemake.params["countries"]) + countries = ammonia.index.intersection(snakemake.params.countries) ammonia = ammonia.loc[countries, years].astype(float) # convert from ktonN to ktonNH3 diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 0b423ad5..d200a78e 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -210,7 +210,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_biomass_potentials", simpl="", clusters="5") - params = snakemake.params["biomass"] + params = snakemake.params.biomass year = params["year"] scenario = params["scenario"] diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index e9378792..a6500bb0 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -116,7 +116,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_bus_regions") configure_logging(snakemake) - countries = snakemake.params["countries"] + countries = snakemake.params.countries n = pypsa.Network(snakemake.input.base_network) diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index 983eda2d..4b1d952e 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -39,7 +39,7 @@ if __name__ == "__main__": for source in ["air", "soil"]: source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"]) - delta_T = snakemake.params["heat_pump_sink_T"] - source_T + delta_T = snakemake.params.heat_pump_sink_T - source_T cop = coefficient_of_performance(delta_T, source) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 798588d9..9a7f9e00 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -106,9 +106,9 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5") configure_logging(snakemake) - cutout_params = snakemake.params["cutouts"][snakemake.wildcards.cutout] + cutout_params = snakemake.params.cutouts[snakemake.wildcards.cutout] - snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) time = [snapshots[0], snapshots[-1]] cutout_params["time"] = slice(*cutout_params.get("time", time)) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index 4ef56d1d..ba1fb881 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -279,16 +279,16 @@ if __name__ == "__main__": configure_logging(snakemake) - powerstatistics = snakemake.params["load"]["power_statistics"] - interpolate_limit = snakemake.params["load"]["interpolate_limit"] - countries = snakemake.params["countries"] - snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) + powerstatistics = snakemake.params.load["power_statistics"] + interpolate_limit = snakemake.params.load["interpolate_limit"] + countries = snakemake.params.countries + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) years = slice(snapshots[0], snapshots[-1]) - time_shift = snakemake.params["load"]["time_shift_for_large_gaps"] + time_shift = snakemake.params.load["time_shift_for_large_gaps"] load = load_timeseries(snakemake.input[0], years, countries, powerstatistics) - if snakemake.params["load"]["manual_adjustments"]: + if snakemake.params.load["manual_adjustments"]: load = manual_adjustment(load, snakemake.input[0], powerstatistics) logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.") diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 3e3cb485..891c4e2a 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -737,16 +737,16 @@ if __name__ == "__main__": logging.basicConfig(level=snakemake.config["logging"]["level"]) - params = snakemake.params["energy"] + params = snakemake.params.energy nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index") population = nuts3["pop"].groupby(nuts3.country).sum() - countries = snakemake.params["countries"] + countries = snakemake.params.countries idees_countries = pd.Index(countries).intersection(eu28) data_year = params["energy_totals_year"] - report_year = snakemake.params["energy"]["eurostat_report_year"] + report_year = snakemake.params.energy["eurostat_report_year"] input_eurostat = snakemake.input.eurostat eurostat = build_eurostat(input_eurostat, countries, report_year, data_year) swiss = build_swiss(data_year) @@ -756,7 +756,7 @@ if __name__ == "__main__": energy.to_csv(snakemake.output.energy_name) base_year_emissions = params["base_emissions_year"] - emissions_scope = snakemake.params["energy"]["emissions"] + emissions_scope = snakemake.params.energy["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope) eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, base_year_emissions diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 655df28f..73494260 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -27,7 +27,7 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - time = pd.date_range(freq="h", **snakemake.params["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params.snapshots) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 26bf31c6..bed666f2 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -130,10 +130,10 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) - params_hydro = snakemake.params["hydro"] + params_hydro = snakemake.params.hydro cutout = atlite.Cutout(snakemake.input.cutout) - countries = snakemake.params["countries"] + countries = snakemake.params.countries country_shapes = ( gpd.read_file(snakemake.input.country_shapes) .set_index("name")["geometry"] diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index a51ff8b2..e93e43c2 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -73,7 +73,7 @@ def prepare_hotmaps_database(regions): df[["srid", "coordinates"]] = df.geom.str.split(";", expand=True) - if snakemake.params["hotmaps_locate_missing"]: + if snakemake.params.hotmaps_locate_missing: df = locate_missing_industrial_sites(df) # remove those sites without valid locations @@ -143,7 +143,7 @@ if __name__ == "__main__": logging.basicConfig(level=snakemake.config["logging"]["level"]) - countries = snakemake.params["countries"] + countries = snakemake.params.countries regions = gpd.read_file(snakemake.input.regions_onshore).set_index("name") diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 9f8c47d0..9ca0d003 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -178,9 +178,9 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today") - params = snakemake.params["industry"] + params = snakemake.params.industry year = params.get("reference_year", 2015) - countries = pd.Index(snakemake.params["countries"]) + countries = pd.Index(snakemake.params.countries) demand = industrial_energy_demand(countries.intersection(eu28), year) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 889c9ecd..74cb1949 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -279,11 +279,11 @@ if __name__ == "__main__": logging.basicConfig(level=snakemake.config["logging"]["level"]) - countries = snakemake.params["countries"] + countries = snakemake.params.countries - year = snakemake.params["industry"]["reference_year"] + year = snakemake.params.industry["reference_year"] - params = snakemake.params["industry"] + params = snakemake.params.industry jrc_dir = snakemake.input.jrc eurostat_dir = snakemake.input.eurostat diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py index 609170aa..ffed5195 100644 --- a/scripts/build_industrial_production_per_country_tomorrow.py +++ b/scripts/build_industrial_production_per_country_tomorrow.py @@ -15,7 +15,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow") - params = snakemake.params["industry"] + params = snakemake.params.industry investment_year = int(snakemake.wildcards.planning_horizons) diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index fa9e5c18..45705002 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -439,7 +439,7 @@ def chemicals_industry(): sector = "Ammonia" df[sector] = 0.0 - if snakemake.params["sector_amonia"]: + if snakemake.params.ammonia: df.loc["ammonia", sector] = params["MWh_NH3_per_tNH3"] else: df.loc["hydrogen", sector] = params["MWh_H2_per_tNH3_electrolysis"] @@ -1468,7 +1468,7 @@ if __name__ == "__main__": # TODO make params option year = 2015 - params = snakemake.params["industry"] + params = snakemake.params.industry df = pd.concat( [ diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 6edd4ac4..7f396e1e 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -115,7 +115,7 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) - countries = snakemake.params["countries"] + countries = snakemake.params.countries ppl = ( pm.powerplants(from_url=True) @@ -134,12 +134,12 @@ if __name__ == "__main__": ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")') ppl = pd.concat([ppl, opsd]) - ppl_query = snakemake.params["powerplants_filter"] + ppl_query = snakemake.params.powerplants_filter if isinstance(ppl_query, str): ppl.query(ppl_query, inplace=True) # add carriers from own powerplant files: - custom_ppl_query = snakemake.params["custom_powerplants"] + custom_ppl_query = snakemake.params.custom_powerplants ppl = add_custom_powerplants( ppl, snakemake.input.custom_powerplants, custom_ppl_query ) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 9e90d42b..9f1f1a51 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -204,7 +204,7 @@ if __name__ == "__main__": nprocesses = int(snakemake.threads) noprogress = snakemake.config["run"].get("disable_progressbar", True) - params = snakemake.params["renewable"][snakemake.wildcards.technology] + params = snakemake.params.renewable[snakemake.wildcards.technology] resource = params["resource"] # pv panel params / wind turbine params correction_factor = params.get("correction_factor", 1.0) capacity_per_sqkm = params["capacity_per_sqkm"] diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index ac7eb4ae..c830415e 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -305,7 +305,7 @@ def prepare_building_stock_data(): u_values.set_index(["country_code", "subsector", "bage", "type"], inplace=True) # only take in config.yaml specified countries into account - countries = snakemake.params["countries"] + countries = snakemake.params.countries area_tot = area_tot.loc[countries] return u_values, country_iso_dic, countries, area_tot, area @@ -1040,7 +1040,7 @@ if __name__ == "__main__": # ******** config ********************************************************* - retro_opts = snakemake.params["retrofitting"] + retro_opts = snakemake.params.retrofitting interest_rate = retro_opts["interest_rate"] annualise_cost = retro_opts["annualise_cost"] # annualise the investment costs tax_weighting = retro_opts[ diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 9d26b0b9..0e59e55b 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -41,7 +41,7 @@ if __name__ == "__main__": "build_sequestration_potentials", simpl="", clusters="181" ) - cf = snakemake.params["co2seq_potential"] + cf = snakemake.params.co2seq_potential gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 0c8b0a94..eb837409 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -255,13 +255,11 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_shapes") configure_logging(snakemake) - country_shapes = countries( - snakemake.input.naturalearth, snakemake.params["countries"] - ) + country_shapes = countries(snakemake.input.naturalearth, snakemake.params.countries) country_shapes.reset_index().to_file(snakemake.output.country_shapes) offshore_shapes = eez( - country_shapes, snakemake.input.eez, snakemake.params["countries"] + country_shapes, snakemake.input.eez, snakemake.params.countries ) offshore_shapes.reset_index().to_file(snakemake.output.offshore_shapes) diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index 180007b7..d285691a 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -27,9 +27,9 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - config = snakemake.params["solar_thermal"] + config = snakemake.params.solar_thermal - time = pd.date_range(freq="h", **snakemake.params["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params.snapshots) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index ee06aebb..9db37c25 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -27,7 +27,7 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - time = pd.date_range(freq="h", **snakemake.params["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params.snapshots) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index dc2b94b9..c5bf4632 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -175,9 +175,9 @@ if __name__ == "__main__": snakemake.input.pop_weighted_energy_totals, index_col=0 ) - options = snakemake.params["sector"] + options = snakemake.params.sector - snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"], tz="UTC") + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots, tz="UTC") nyears = len(snapshots) / 8760 diff --git a/scripts/make_summary.py b/scripts/make_summary.py index da0712d7..00fca2fd 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -198,7 +198,7 @@ def calculate_costs(n, label, costs): def calculate_cumulative_cost(): - planning_horizons = snakemake.params["scenario"]["planning_horizons"] + planning_horizons = snakemake.params.scenario["planning_horizons"] cumulative_cost = pd.DataFrame( index=df["costs"].sum().index, @@ -688,19 +688,19 @@ if __name__ == "__main__": (cluster, ll, opt + sector_opt, planning_horizon): "results/" + snakemake.params.RDIR + f"/postnetworks/elec_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc" - for simpl in snakemake.params["scenario"]["simpl"] - for cluster in snakemake.params["scenario"]["clusters"] - for opt in snakemake.params["scenario"]["opts"] - for sector_opt in snakemake.params["scenario"]["sector_opts"] - for ll in snakemake.params["scenario"]["ll"] - for planning_horizon in snakemake.params["scenario"]["planning_horizons"] + for simpl in snakemake.params.scenario["simpl"] + for cluster in snakemake.params.scenario["clusters"] + for opt in snakemake.params.scenario["opts"] + for sector_opt in snakemake.params.scenario["sector_opts"] + for ll in snakemake.params.scenario["ll"] + for planning_horizon in snakemake.params.scenario["planning_horizons"] } - Nyears = len(pd.date_range(freq="h", **snakemake.params["snapshots"])) / 8760 + Nyears = len(pd.date_range(freq="h", **snakemake.params.snapshots)) / 8760 costs_db = prepare_costs( snakemake.input.costs, - snakemake.params["costs"], + snakemake.params.costs, Nyears, ) @@ -710,7 +710,7 @@ if __name__ == "__main__": to_csv(df) - if snakemake.params["foresight"] == "myopic": + if snakemake.params.foresight == "myopic": cumulative_cost = calculate_cumulative_cost() cumulative_cost.to_csv( "results/" + snakemake.params.RDIR + "/csvs/cumulative_cost.csv" diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 399f46e8..184d86f0 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -70,7 +70,7 @@ def plot_map( transmission=False, with_legend=True, ): - tech_colors = snakemake.params["plotting"]["tech_colors"] + tech_colors = snakemake.params.plotting["tech_colors"] n = network.copy() assign_location(n) @@ -116,9 +116,7 @@ def plot_map( costs = costs.stack() # .sort_index() # hack because impossible to drop buses... - eu_location = snakemake.params["plotting"].get( - "eu_node_location", dict(x=-5.5, y=46) - ) + eu_location = snakemake.params.plotting.get("eu_node_location", dict(x=-5.5, y=46)) n.buses.loc["EU gas", "x"] = eu_location["x"] n.buses.loc["EU gas", "y"] = eu_location["y"] @@ -315,7 +313,7 @@ def plot_h2_map(network, regions): h2_new = n.links[n.links.carrier == "H2 pipeline"] h2_retro = n.links[n.links.carrier == "H2 pipeline retrofitted"] - if snakemake.params["foresight"] == "myopic": + if snakemake.params.foresight == "myopic": # sum capacitiy for pipelines from different investment periods h2_new = group_pipes(h2_new) @@ -558,7 +556,7 @@ def plot_ch4_map(network): link_widths_used = max_usage / linewidth_factor link_widths_used[max_usage < line_lower_threshold] = 0.0 - tech_colors = snakemake.params["plotting"]["tech_colors"] + tech_colors = snakemake.params.plotting["tech_colors"] pipe_colors = { "gas pipeline": "#f08080", @@ -700,7 +698,7 @@ def plot_map_without(network): # hack because impossible to drop buses... if "EU gas" in n.buses.index: - eu_location = snakemake.params["plotting"].get( + eu_location = snakemake.params.plotting.get( "eu_node_location", dict(x=-5.5, y=46) ) n.buses.loc["EU gas", "x"] = eu_location["x"] @@ -876,7 +874,7 @@ def plot_series(network, carrier="AC", name="test"): stacked=True, linewidth=0.0, color=[ - snakemake.params["plotting"]["tech_colors"][i.replace(suffix, "")] + snakemake.params.plotting["tech_colors"][i.replace(suffix, "")] for i in new_columns ], ) @@ -937,7 +935,7 @@ if __name__ == "__main__": regions = gpd.read_file(snakemake.input.regions).set_index("name") - map_opts = snakemake.params["plotting"]["map"] + map_opts = snakemake.params.plotting["map"] if map_opts["boundaries"] is None: map_opts["boundaries"] = regions.total_bounds[[0, 2, 1, 3]] + [-1, 1, -1, 1] diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 36f75207..e7de5473 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -142,7 +142,7 @@ def plot_costs(): df = df.groupby(df.index.map(rename_techs)).sum() - to_drop = df.index[df.max(axis=1) < snakemake.params["plotting"]["costs_threshold"]] + to_drop = df.index[df.max(axis=1) < snakemake.params.plotting["costs_threshold"]] logger.info( f"Dropping technology with costs below {snakemake.params['plotting']['costs_threshold']} EUR billion per year" @@ -165,7 +165,7 @@ def plot_costs(): kind="bar", ax=ax, stacked=True, - color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params.plotting["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -173,7 +173,7 @@ def plot_costs(): handles.reverse() labels.reverse() - ax.set_ylim([0, snakemake.params["plotting"]["costs_max"]]) + ax.set_ylim([0, snakemake.params.plotting["costs_max"]]) ax.set_ylabel("System Cost [EUR billion per year]") @@ -201,7 +201,7 @@ def plot_energy(): df = df.groupby(df.index.map(rename_techs)).sum() to_drop = df.index[ - df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] + df.abs().max(axis=1) < snakemake.params.plotting["energy_threshold"] ] logger.info( @@ -227,7 +227,7 @@ def plot_energy(): kind="bar", ax=ax, stacked=True, - color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params.plotting["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -237,8 +237,8 @@ def plot_energy(): ax.set_ylim( [ - snakemake.params["plotting"]["energy_min"], - snakemake.params["plotting"]["energy_max"], + snakemake.params.plotting["energy_min"], + snakemake.params.plotting["energy_max"], ] ) @@ -287,7 +287,7 @@ def plot_balances(): df = df.groupby(df.index.map(rename_techs)).sum() to_drop = df.index[ - df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] / 10 + df.abs().max(axis=1) < snakemake.params.plotting["energy_threshold"] / 10 ] if v[0] in co2_carriers: @@ -317,7 +317,7 @@ def plot_balances(): kind="bar", ax=ax, stacked=True, - color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params.plotting["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -455,10 +455,10 @@ def plot_carbon_budget_distribution(input_eurostat): ax1 = plt.subplot(gs1[0, 0]) ax1.set_ylabel("CO$_2$ emissions (Gt per year)", fontsize=22) ax1.set_ylim([0, 5]) - ax1.set_xlim([1990, snakemake.params["planning_horizons"][-1] + 1]) + ax1.set_xlim([1990, snakemake.params.planning_horizons[-1] + 1]) path_cb = "results/" + snakemake.params.RDIR + "/csvs/" - countries = snakemake.params["countries"] + countries = snakemake.params.countries e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990) CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0) @@ -555,7 +555,7 @@ if __name__ == "__main__": plot_balances() - for sector_opts in snakemake.params["sector_opts"]: + for sector_opts in snakemake.params.sector_opts: opts = sector_opts.split("-") for o in opts: if "cb" in o: diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 51777ef5..e48ee162 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -253,12 +253,12 @@ if __name__ == "__main__": Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["max_hours"], + snakemake.params.costs, + snakemake.params.max_hours, Nyears, ) - set_line_s_max_pu(n, snakemake.params["lines"]["s_max_pu"]) + set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"]) for o in opts: m = re.match(r"^\d+h$", o, re.IGNORECASE) @@ -269,7 +269,7 @@ if __name__ == "__main__": for o in opts: m = re.match(r"^\d+seg$", o, re.IGNORECASE) if m is not None: - solver_name = snakemake.params["solver_name"] + solver_name = snakemake.params.solver_name n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) break @@ -277,11 +277,11 @@ if __name__ == "__main__": if "Co2L" in o: m = re.findall("[0-9]*\.?[0-9]+$", o) if len(m) > 0: - co2limit = float(m[0]) * snakemake.params["co2base"] + co2limit = float(m[0]) * snakemake.params.co2base add_co2limit(n, co2limit, Nyears) logger.info("Setting CO2 limit according to wildcard value.") else: - add_co2limit(n, snakemake.params["co2limit"], Nyears) + add_co2limit(n, snakemake.params.co2limit, Nyears) logger.info("Setting CO2 limit according to config value.") break @@ -293,7 +293,7 @@ if __name__ == "__main__": add_gaslimit(n, limit, Nyears) logger.info("Setting gas usage limit according to wildcard value.") else: - add_gaslimit(n, snakemake.params["gaslimit"], Nyears) + add_gaslimit(n, snakemake.params.gaslimit, Nyears) logger.info("Setting gas usage limit according to config value.") break @@ -322,7 +322,7 @@ if __name__ == "__main__": add_emission_prices(n, dict(co2=float(m[0]))) else: logger.info("Setting emission prices according to config value.") - add_emission_prices(n, snakemake.params["costs"]["emission_prices"]) + add_emission_prices(n, snakemake.params.costs["emission_prices"]) break ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:] @@ -330,8 +330,8 @@ if __name__ == "__main__": set_line_nom_max( n, - s_nom_max_set=snakemake.params["lines"].get("s_nom_max,", np.inf), - p_nom_max_set=snakemake.params["links"].get("p_nom_max,", np.inf), + s_nom_max_set=snakemake.params.lines.get("s_nom_max,", np.inf), + p_nom_max_set=snakemake.params.links.get("p_nom_max,", np.inf), ) if "ATK" in opts: diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 8b8b7d80..77fab4f8 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -200,12 +200,12 @@ def co2_emissions_year( """ Calculate CO2 emissions in one specific year (e.g. 1990 or 2018). """ - emissions_scope = snakemake.params["energy"]["emissions"] + emissions_scope = snakemake.params.energy["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope) # TODO: read Eurostat data from year > 2014 # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK - report_year = snakemake.params["energy"]["eurostat_report_year"] + report_year = snakemake.params.energy["eurostat_report_year"] if year > 2014: eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, year=2014 @@ -241,7 +241,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): carbon_budget = float(o[o.find("cb") + 2 : o.find("ex")]) r = float(o[o.find("ex") + 2 :]) - countries = snakemake.params["countries"] + countries = snakemake.params.countries e_1990 = co2_emissions_year( countries, input_eurostat, opts, emissions_scope, report_year, year=1990 @@ -252,7 +252,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries, input_eurostat, opts, emissions_scope, report_year, year=2018 ) - planning_horizons = snakemake.params["planning_horizons"] + planning_horizons = snakemake.params.planning_horizons t_0 = planning_horizons[0] if "be" in o: @@ -391,7 +391,7 @@ def update_wind_solar_costs(n, costs): with xr.open_dataset(profile) as ds: underwater_fraction = ds["underwater_fraction"].to_pandas() connection_cost = ( - snakemake.params["length_factor"] + snakemake.params.length_factor * ds["average_distance"].to_pandas() * ( underwater_fraction @@ -483,8 +483,8 @@ def remove_elec_base_techs(n): batteries and H2) from base electricity-only network, since they're added here differently using links. """ - for c in n.iterate_components(snakemake.params["pypsa_eur"]): - to_keep = snakemake.params["pypsa_eur"][c.name] + for c in n.iterate_components(snakemake.params.pypsa_eur): + to_keep = snakemake.params.pypsa_eur[c.name] to_remove = pd.Index(c.df.carrier.unique()).symmetric_difference(to_keep) if to_remove.empty: continue @@ -674,7 +674,7 @@ def add_dac(n, costs): def add_co2limit(n, nyears=1.0, limit=0.0): logger.info(f"Adding CO2 budget limit as per unit of 1990 levels of {limit}") - countries = snakemake.params["countries"] + countries = snakemake.params.countries sectors = emission_sectors_from_opts(opts) @@ -787,7 +787,7 @@ def add_ammonia(n, costs): nodes = pop_layout.index - cf_industry = snakemake.params["industry"] + cf_industry = snakemake.params.industry n.add("Carrier", "NH3") @@ -1102,7 +1102,7 @@ def add_storage_and_grids(n, costs): lifetime=costs.at["OCGT", "lifetime"], ) - cavern_types = snakemake.params["sector"]["hydrogen_underground_storage_locations"] + cavern_types = snakemake.params.sector["hydrogen_underground_storage_locations"] h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0) if ( @@ -3274,7 +3274,7 @@ if __name__ == "__main__": update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) - options = snakemake.params["sector"] + options = snakemake.params.sector opts = snakemake.wildcards.sector_opts.split("-") @@ -3289,7 +3289,7 @@ if __name__ == "__main__": costs = prepare_costs( snakemake.input.costs, - snakemake.params["costs"], + snakemake.params.costs, nyears, ) @@ -3301,10 +3301,10 @@ if __name__ == "__main__": spatial = define_spatial(pop_layout.index, options) - if snakemake.params["foresight"] == "myopic": + if snakemake.params.foresight == "myopic": add_lifetime_wind_solar(n, costs) - conventional = snakemake.params["conventional_carriers"] + conventional = snakemake.params.conventional_carriers for carrier in conventional: add_carrier_buses(n, carrier) @@ -3369,19 +3369,19 @@ if __name__ == "__main__": if options["allam_cycle"]: add_allam(n, costs) - solver_name = snakemake.params["solver_name"] + solver_name = snakemake.params.solver_name n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" - limit = get(snakemake.params["co2_budget"], investment_year) + limit = get(snakemake.params.co2_budget, investment_year) for o in opts: if "cb" not in o: continue limit_type = "carbon budget" fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): - emissions_scope = snakemake.params["emissions_scope"] - report_year = snakemake.params["report_year"] + emissions_scope = snakemake.params.emissions_scope + report_year = snakemake.params.report_year build_carbon_budget( o, snakemake.input.eurostat, fn, emissions_scope, report_year ) @@ -3416,8 +3416,8 @@ if __name__ == "__main__": if options["electricity_grid_connection"]: add_electricity_grid_connection(n, costs) - first_year_myopic = (snakemake.params["foresight"] == "myopic") and ( - snakemake.params["planning_horizons"][0] == investment_year + first_year_myopic = (snakemake.params.foresight == "myopic") and ( + snakemake.params.planning_horizons[0] == investment_year ) if options.get("cluster_heat_buses", False) and not first_year_myopic: diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 2204ac36..c7053bed 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -53,7 +53,7 @@ if __name__ == "__main__": snakemake ) # TODO Make logging compatible with progressbar (see PR #102) - if snakemake.params["tutorial"]: + if snakemake.params.tutorial: url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz" else: url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz" diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 41679a03..d80e38a0 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -44,9 +44,9 @@ pypsa.pf.logger.setLevel(logging.WARNING) from pypsa.descriptors import get_switchable_as_dense as get_as_dense -def add_land_use_constraint(n, planning_horizons_param, config): +def add_land_use_constraint(n, planning_horizons, config): if "m" in snakemake.wildcards.clusters: - _add_land_use_constraint_m(n, planning_horizons_param, config) + _add_land_use_constraint_m(n, planning_horizons, config) else: _add_land_use_constraint(n) @@ -80,7 +80,7 @@ def _add_land_use_constraint(n): n.generators.p_nom_max.clip(lower=0, inplace=True) -def _add_land_use_constraint_m(n, planning_horizons_param, config): +def _add_land_use_constraint_m(n, planning_horizons, config): # if generators clustering is lower than network clustering, land_use accounting is at generators clusters planning_horizons = param["planning_horizons"] @@ -145,9 +145,9 @@ def prepare_network( n, solve_opts=None, config=None, - foresight_param=None, - planning_horizons_param=None, - co2_sequestration_potential_param=None, + foresight=None, + planning_horizons=None, + co2_sequestration_potential=None, ): if "clip_p_max_pu" in solve_opts: for df in ( @@ -198,11 +198,11 @@ def prepare_network( n.set_snapshots(n.snapshots[:nhours]) n.snapshot_weightings[:] = 8760.0 / nhours - if foresight_param == "myopic": - add_land_use_constraint(n, planning_horizons_param, config) + if foresight == "myopic": + add_land_use_constraint(n, planning_horizons, config) if n.stores.carrier.eq("co2 stored").any(): - limit = co2_sequestration_potential_param + limit = co2_sequestration_potential add_co2_sequestration_limit(n, limit=limit) return n @@ -597,13 +597,11 @@ def extra_functionality(n, snapshots): add_pipe_retrofit_constraint(n) -def solve_network(n, config, solving_param, opts="", **kwargs): - set_of_options = solving_param["solver"]["options"] - solver_options = ( - solving_param["solver_options"][set_of_options] if set_of_options else {} - ) - solver_name = solving_param["solver"]["name"] - cf_solving = solving_param["options"] +def solve_network(n, config, solving, opts="", **kwargs): + set_of_options = solving["solver"]["options"] + solver_options = solving["solver_options"][set_of_options] if set_of_options else {} + solver_name = solving["solver"]["name"] + cf_solving = solving["options"] track_iterations = cf_solving.get("track_iterations", False) min_iterations = cf_solving.get("min_iterations", 4) max_iterations = cf_solving.get("max_iterations", 6) @@ -672,7 +670,7 @@ if __name__ == "__main__": if "sector_opts" in snakemake.wildcards.keys(): opts += "-" + snakemake.wildcards.sector_opts opts = [o for o in opts.split("-") if o != ""] - solve_opts = snakemake.params["solving"]["options"] + solve_opts = snakemake.params.solving["options"] np.random.seed(solve_opts.get("seed", 123)) @@ -686,17 +684,15 @@ if __name__ == "__main__": n, solve_opts, config=snakemake.config, - foresight_param=snakemake.params["foresight"], - planning_horizons_param=snakemake.params["planning_horizons"], - co2_sequestration_potential_param=snakemake.params[ - "co2_sequestration_potential" - ], + foresight=snakemake.params.foresight, + planning_horizons=snakemake.params.planning_horizons, + co2_sequestration_potential=snakemake.params["co2_sequestration_potential"], ) n = solve_network( n, config=snakemake.config, - solving_param=snakemake.params["solving"], + solving=snakemake.params.solving, opts=opts, log_fn=snakemake.log.solver, ) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 421bc515..37e853e5 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -41,7 +41,7 @@ if __name__ == "__main__": opts = (snakemake.wildcards.opts + "-" + snakemake.wildcards.sector_opts).split("-") opts = [o for o in opts if o != ""] - solve_opts = snakemake.params["options"] + solve_opts = snakemake.params.options np.random.seed(solve_opts.get("seed", 123)) From 27eea273bd20559dcf02d32ed0638c2938fb4acf Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 15 Jun 2023 19:12:30 +0200 Subject: [PATCH 75/78] harmonize params names --- rules/build_electricity.smk | 9 +++------ rules/build_sector.smk | 7 ++++--- scripts/add_electricity.py | 21 ++++++--------------- scripts/build_sequestration_potentials.py | 2 +- scripts/prepare_network.py | 2 +- scripts/prepare_sector_network.py | 4 ++-- 6 files changed, 17 insertions(+), 28 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 9d4315df..44ebf404 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -320,11 +320,10 @@ rule simplify_network: aggregation_strategies=config["clustering"].get("aggregation_strategies", {}), focus_weights=config.get("focus_weights", None), renewable_carriers=config["electricity"]["renewable_carriers"], - costs=config["costs"], max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], p_max_pu=config["links"].get("p_max_pu", 1.0), - solver_name=config["solving"]["solver"]["name"], + costs=config["costs"], input: network=RESOURCES + "networks/elec.nc", tech_costs=COSTS, @@ -357,10 +356,9 @@ rule cluster_network: focus_weights=config.get("focus_weights", None), renewable_carriers=config["electricity"]["renewable_carriers"], conventional_carriers=config["electricity"].get("conventional_carriers", []), - costs=config["costs"], max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], - solver_name=config["solving"]["solver"]["name"], + costs=config["costs"], input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", @@ -393,9 +391,9 @@ rule cluster_network: rule add_extra_components: params: - costs=config["costs"], extendable_carriers=config["electricity"]["extendable_carriers"], max_hours=config["electricity"]["max_hours"], + costs=config["costs"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", tech_costs=COSTS, @@ -418,7 +416,6 @@ rule prepare_network: params: links=config["links"], lines=config["lines"], - solver_name=config["solving"]["solver"]["name"], co2base=config["electricity"]["co2base"], co2limit=config["electricity"]["co2limit"], gaslimit=config["electricity"].get("gaslimit"), diff --git a/rules/build_sector.smk b/rules/build_sector.smk index e15394ff..1c3507fd 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -330,7 +330,9 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_sequestration_potentials: params: - co2seq_potential=config["sector"]["regional_co2_sequestration_potential"], + sequestration_potential=config["sector"][ + "regional_co2_sequestration_potential" + ], input: sequestration_potential=HTTP.remote( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", @@ -705,7 +707,6 @@ rule build_transport_demand: rule prepare_sector_network: params: co2_budget=config["co2_budget"], - solver_name=config["solving"]["solver"]["name"], conventional_carriers=config["existing_capacities"]["conventional_carriers"], foresight=config["foresight"], costs=config["costs"], @@ -716,7 +717,7 @@ rule prepare_sector_network: planning_horizons=config["scenario"]["planning_horizons"], countries=config["countries"], emissions_scope=config["energy"]["emissions"], - report_year=config["energy"]["eurostat_report_year"], + eurostat_report_year=config["energy"]["eurostat_report_year"], RDIR=RDIR, input: **build_retro_cost_output, diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index d98dc767..e2f7fb65 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -137,7 +137,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers): n.import_components_from_dataframe(emissions, "Carrier") -def load_costs(tech_costs, params, max_hours, Nyears=1.0): +def load_costs(tech_costs, costs, max_hours, Nyears=1.0): # set all asset costs and other parameters costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index() @@ -145,7 +145,7 @@ def load_costs(tech_costs, params, max_hours, Nyears=1.0): costs.loc[costs.unit.str.contains("/kW"), "value"] *= 1e3 costs.unit = costs.unit.str.replace("/kW", "/MW") - fill_values = params["fill_values"] + fill_values = costs["fill_values"] costs = costs.value.unstack().fillna(fill_values) costs["capital_cost"] = ( @@ -168,8 +168,8 @@ def load_costs(tech_costs, params, max_hours, Nyears=1.0): costs.at["CCGT", "co2_emissions"] = costs.at["gas", "co2_emissions"] costs.at["solar", "capital_cost"] = ( - params["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] - + (1 - params["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] + costs["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] + + (1 - costs["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] ) def costs_for_storage(store, link1, link2=None, max_hours=1.0): @@ -193,7 +193,7 @@ def load_costs(tech_costs, params, max_hours, Nyears=1.0): ) for attr in ("marginal_cost", "capital_cost"): - overwrites = params.get(attr) + overwrites = costs.get(attr) if overwrites is not None: overwrites = pd.Series(overwrites) costs.loc[overwrites.index, attr] = overwrites @@ -728,16 +728,6 @@ if __name__ == "__main__": ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in params.electricity: - renewable_carriers = set(params.electricity["renewable_carriers"]) - else: - logger.warning( - "Missing key `renewable_carriers` under config entry `electricity`. " - "In future versions, this will raise an error. " - "Falling back to carriers listed under `renewable`." - ) - renewable_carriers = params.renewable - attach_load( n, snakemake.input.regions, @@ -749,6 +739,7 @@ if __name__ == "__main__": update_transmission_costs(n, costs, params.length_factor) + renewable_carriers = set(params.electricity["renewable_carriers"]) extendable_carriers = params.electricity["extendable_carriers"] conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 0e59e55b..e19a96da 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -41,7 +41,7 @@ if __name__ == "__main__": "build_sequestration_potentials", simpl="", clusters="181" ) - cf = snakemake.params.co2seq_potential + cf = snakemake.params.sequestration_potential gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index e48ee162..adbfa36c 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -269,7 +269,7 @@ if __name__ == "__main__": for o in opts: m = re.match(r"^\d+seg$", o, re.IGNORECASE) if m is not None: - solver_name = snakemake.params.solver_name + solver_name = snakemake.config["solving"]["solver"]["name"] n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) break diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 77fab4f8..73adf522 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3369,7 +3369,7 @@ if __name__ == "__main__": if options["allam_cycle"]: add_allam(n, costs) - solver_name = snakemake.params.solver_name + solver_name = snakemake.config["solving"]["solver"]["name"] n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" @@ -3381,7 +3381,7 @@ if __name__ == "__main__": fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): emissions_scope = snakemake.params.emissions_scope - report_year = snakemake.params.report_year + report_year = snakemake.params.eurostat_report_year build_carbon_budget( o, snakemake.input.eurostat, fn, emissions_scope, report_year ) From a17647a07e3c5e93fcaa2a686e0f72c09ab0dbfc Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 15 Jun 2023 19:35:41 +0200 Subject: [PATCH 76/78] add_electricity: fix cost ref --- scripts/add_electricity.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index e2f7fb65..594988bd 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -137,7 +137,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers): n.import_components_from_dataframe(emissions, "Carrier") -def load_costs(tech_costs, costs, max_hours, Nyears=1.0): +def load_costs(tech_costs, config, max_hours, Nyears=1.0): # set all asset costs and other parameters costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index() @@ -145,7 +145,7 @@ def load_costs(tech_costs, costs, max_hours, Nyears=1.0): costs.loc[costs.unit.str.contains("/kW"), "value"] *= 1e3 costs.unit = costs.unit.str.replace("/kW", "/MW") - fill_values = costs["fill_values"] + fill_values = config["fill_values"] costs = costs.value.unstack().fillna(fill_values) costs["capital_cost"] = ( @@ -168,8 +168,8 @@ def load_costs(tech_costs, costs, max_hours, Nyears=1.0): costs.at["CCGT", "co2_emissions"] = costs.at["gas", "co2_emissions"] costs.at["solar", "capital_cost"] = ( - costs["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] - + (1 - costs["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] + config["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] + + (1 - config["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] ) def costs_for_storage(store, link1, link2=None, max_hours=1.0): @@ -193,7 +193,7 @@ def load_costs(tech_costs, costs, max_hours, Nyears=1.0): ) for attr in ("marginal_cost", "capital_cost"): - overwrites = costs.get(attr) + overwrites = config.get(attr) if overwrites is not None: overwrites = pd.Series(overwrites) costs.loc[overwrites.index, attr] = overwrites From 284e3f8e70c4c8f44346bb519967d82256d5b3b3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 22:43:01 +0000 Subject: [PATCH 77/78] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/codespell-project/codespell: v2.2.4 → v2.2.5](https://github.com/codespell-project/codespell/compare/v2.2.4...v2.2.5) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 826a4819..5acfbf42 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,7 +30,7 @@ repos: # Find common spelling mistakes in comments and docstrings - repo: https://github.com/codespell-project/codespell - rev: v2.2.4 + rev: v2.2.5 hooks: - id: codespell args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore'] # Ignore capital case words, e.g. country codes From cec407e656bfa7c684b821d0e714ba03111158f4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 22:19:08 +0000 Subject: [PATCH 78/78] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/docformatter: v1.7.2 → v1.7.3](https://github.com/PyCQA/docformatter/compare/v1.7.2...v1.7.3) - [github.com/fsfe/reuse-tool: v1.1.2 → v2.0.0](https://github.com/fsfe/reuse-tool/compare/v1.1.2...v2.0.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5acfbf42..7bb35b53 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: # Make docstrings PEP 257 compliant - repo: https://github.com/PyCQA/docformatter - rev: v1.7.2 + rev: v1.7.3 hooks: - id: docformatter args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] @@ -87,6 +87,6 @@ repos: # Check for FSFE REUSE compliance (licensing) - repo: https://github.com/fsfe/reuse-tool - rev: v1.1.2 + rev: v2.0.0 hooks: - id: reuse