diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 02d360d3..a38e6800 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -87,6 +87,6 @@ repos: # Check for FSFE REUSE compliance (licensing) - repo: https://github.com/fsfe/reuse-tool - rev: v2.1.0 + rev: v3.0.1 hooks: - id: reuse diff --git a/Snakefile b/Snakefile index 7c16ff9f..14c9e821 100644 --- a/Snakefile +++ b/Snakefile @@ -13,9 +13,10 @@ from snakemake.utils import min_version min_version("7.7") - -if not exists("config/config.yaml") and exists("config/config.default.yaml"): - copyfile("config/config.default.yaml", "config/config.yaml") +conf_file = os.path.join(workflow.current_basedir, "config/config.yaml") +conf_default_file = os.path.join(workflow.current_basedir, "config/config.default.yaml") +if not exists(conf_file) and exists(conf_default_file): + copyfile(conf_default_file, conf_file) configfile: "config/config.yaml" diff --git a/config/config.default.yaml b/config/config.default.yaml index 6349f43e..51080862 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -44,7 +44,7 @@ scenario: opts: - '' sector_opts: - - Co2L0-3H-T-H-B-I-A-solar+p3-dist1 + - Co2L0-3H-T-H-B-I-A-dist1 planning_horizons: # - 2020 # - 2030 @@ -687,6 +687,7 @@ solving: rolling_horizon: false seed: 123 custom_extra_functionality: "../data/custom_extra_functionality.py" + # io_api: "direct" # Increases performance but only supported for the highs and gurobi solvers # options that go into the optimize function track_iterations: false min_iterations: 4 @@ -772,6 +773,13 @@ plotting: color_geomap: ocean: white land: white + projection: + name: "EqualEarth" + # See https://scitools.org.uk/cartopy/docs/latest/reference/projections.html for alternatives, for example: + # name: "LambertConformal" + # central_longitude: 10. + # central_latitude: 50. + # standard_parallels: [35, 65] eu_node_location: x: -5.5 y: 46. diff --git a/config/config.perfect.yaml b/config/config.perfect.yaml index 4dde8db9..ff531303 100644 --- a/config/config.perfect.yaml +++ b/config/config.perfect.yaml @@ -19,9 +19,9 @@ scenario: opts: - '' sector_opts: - - 1p5-4380H-T-H-B-I-A-solar+p3-dist1 - - 1p7-4380H-T-H-B-I-A-solar+p3-dist1 - - 2p0-4380H-T-H-B-I-A-solar+p3-dist1 + - 1p5-4380H-T-H-B-I-A-dist1 + - 1p7-4380H-T-H-B-I-A-dist1 + - 2p0-4380H-T-H-B-I-A-dist1 planning_horizons: - 2020 - 2025 diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index d566c6cb..2dab7b04 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -18,7 +18,7 @@ scenario: clusters: - 5 sector_opts: - - 24H-T-H-B-I-A-solar+p3-dist1 + - 24H-T-H-B-I-A-dist1 planning_horizons: - 2030 - 2040 diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index a2a0f5a4..6d1900cf 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -17,7 +17,7 @@ scenario: clusters: - 5 sector_opts: - - CO2L0-24H-T-H-B-I-A-solar+p3-dist1 + - CO2L0-24H-T-H-B-I-A-dist1 planning_horizons: - 2030 diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index 49886b26..f20a2c9f 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -18,7 +18,7 @@ scenario: clusters: - 5 sector_opts: - - 8760H-T-H-B-I-A-solar+p3-dist1 + - 8760H-T-H-B-I-A-dist1 planning_horizons: - 2030 - 2040 diff --git a/doc/configtables/plotting.csv b/doc/configtables/plotting.csv index ed5d9c9f..82fc203c 100644 --- a/doc/configtables/plotting.csv +++ b/doc/configtables/plotting.csv @@ -1,6 +1,9 @@ ,Unit,Values,Description map,,, -- boundaries,°,"[x1,x2,y1,y2]",Boundaries of the map plots in degrees latitude (y) and longitude (x) +projection,,,, +-- name,--,"Valid Cartopy projection name","See https://scitools.org.uk/cartopy/docs/latest/reference/projections.html for list of available projections." +-- args,--,--,"Other entries under 'projection' are passed as keyword arguments to the projection constructor, e.g. ``central_longitude: 10.``." costs_max,bn Euro,float,Upper y-axis limit in cost bar plots. costs_threshold,bn Euro,float,Threshold below which technologies will not be shown in cost bar plots. energy_max,TWh,float,Upper y-axis limit in energy bar plots. diff --git a/doc/configtables/snapshots.csv b/doc/configtables/snapshots.csv index 4a3e1212..0226a9aa 100644 --- a/doc/configtables/snapshots.csv +++ b/doc/configtables/snapshots.csv @@ -2,5 +2,5 @@ start,--,str or datetime-like; e.g. YYYY-MM-DD,Left bound of date range end,--,str or datetime-like; e.g. YYYY-MM-DD,Right bound of date range inclusive,--,"One of {'neither', 'both', ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``both`` or neither side ``None``." -resolution ,--,"{false,``nH``; i.e. ``2H``-``6H``}",Resample the time-resolution by averaging over every ``n`` snapshots -segmentation,--,"{false,``n``; e.g. ``4380``}","Apply time series segmentation with `tsam `_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load." +resolution ,--,"{false,``nH``; i.e. ``2H``-``6H``}","Resample the time-resolution by averaging over every ``n`` snapshots in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks." +segmentation,--,"{false,``n``; e.g. ``4380``}","Apply time series segmentation with `tsam `_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks." diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv index 6eff10ae..7189399b 100644 --- a/doc/configtables/solving.csv +++ b/doc/configtables/solving.csv @@ -7,6 +7,7 @@ options,,, -- rolling_horizon,bool,"{'true','false'}","Whether to optimize the network in a rolling horizon manner, where the snapshot range is split into slices of size `horizon` which are solved consecutively." -- seed,--,int,Random seed for increased deterministic behaviour. -- custom_extra_functionality,--,str,Path to a Python file with custom extra functionality code to be injected into the solving rules of the workflow relative to ``rules`` directory. +-- io_api,string,"{'lp','mps','direct'}",Passed to linopy and determines the API used to communicate with the solver. With the ``'lp'`` and ``'mps'`` options linopy passes a file to the solver; with the ``'direct'`` option (only supported for HIGHS and Gurobi) linopy uses an in-memory python API resulting in better performance. -- track_iterations,bool,"{'true','false'}",Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration) -- min_iterations,--,int,Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run. -- max_iterations,--,int,Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run. diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 1a0013d5..83e14eab 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -48,6 +48,15 @@ Upcoming Release * Bugfix: Correctly read out number of solver threads from configuration file. +* Air-sourced heat pumps can now also be built in rural areas. Previously, only + ground-sourced heat pumps were considered for this category. + +* Bugfix: Correctly read out number of solver threads from configuration file. + +* Add support for the linopy ``io_api`` option; set to ``"direct"`` to increase model reading and writing performance for the highs and gurobi solvers. + +* Add the option to customise map projection in plotting config. + PyPSA-Eur 0.9.0 (5th January 2024) ================================== diff --git a/envs/environment.yaml b/envs/environment.yaml index 535acbdb..6ff4b7f1 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -26,7 +26,7 @@ dependencies: - yaml - pytables - lxml -- powerplantmatching>=0.5.5 +- powerplantmatching>=0.5.5,!=0.5.9 - numpy - pandas>=2.1 - geopandas>=0.11.0 diff --git a/envs/retrieve.yaml b/envs/retrieve.yaml new file mode 100644 index 00000000..b5db795d --- /dev/null +++ b/envs/retrieve.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +name: pypsa-eur-retrieve +channels: +- conda-forge +- bioconda +dependencies: +- python>=3.8 +- snakemake-minimal>=7.7.0,<8.0.0 +- pandas>=2.1 +- tqdm diff --git a/rules/common.smk b/rules/common.smk index 1654180f..5aa7ae53 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -43,7 +43,7 @@ def memory(w): def input_custom_extra_functionality(w): path = config["solving"]["options"].get("custom_extra_functionality", False) if path: - return workflow.source_path(path) + return os.path.join(os.path.dirname(workflow.snakefile), path) return [] diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 5e1e3e59..05bbefd8 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -37,7 +37,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", mem_mb=1000, retries: 2 conda: - "../envs/environment.yaml" + "../envs/retrieve.yaml" script: "../scripts/retrieve_databundle.py" @@ -55,7 +55,7 @@ if config["enable"].get("retrieve_irena"): mem_mb=1000, retries: 2 conda: - "../envs/environment.yaml" + "../envs/retrieve.yaml" script: "../scripts/retrieve_irena.py" @@ -157,7 +157,7 @@ if config["enable"]["retrieve"] and config["enable"].get( LOGS + "retrieve_sector_databundle.log", retries: 2 conda: - "../envs/environment.yaml" + "../envs/retrieve.yaml" script: "../scripts/retrieve_sector_databundle.py" @@ -180,7 +180,7 @@ if config["enable"]["retrieve"]: LOGS + "retrieve_gas_infrastructure_data.log", retries: 2 conda: - "../envs/environment.yaml" + "../envs/retrieve.yaml" script: "../scripts/retrieve_gas_infrastructure_data.py" @@ -316,7 +316,7 @@ if config["enable"]["retrieve"]: layer_path = ( f"/vsizip/{params.folder}/WDPA_{bYYYY}_Public_shp_{i}.zip" ) - print(f"Adding layer {i+1} of 3 to combined output file.") + print(f"Adding layer {i + 1} of 3 to combined output file.") shell("ogr2ogr -f gpkg -update -append {output.gpkg} {layer_path}") rule download_wdpa_marine: @@ -340,7 +340,7 @@ if config["enable"]["retrieve"]: for i in range(3): # vsizip is special driver for directly working with zipped shapefiles in ogr2ogr layer_path = f"/vsizip/{params.folder}/WDPA_WDOECM_{bYYYY}_Public_marine_shp_{i}.zip" - print(f"Adding layer {i+1} of 3 to combined output file.") + print(f"Adding layer {i + 1} of 3 to combined output file.") shell("ogr2ogr -f gpkg -update -append {output.gpkg} {layer_path}") @@ -376,6 +376,6 @@ if config["enable"]["retrieve"]: mem_mb=5000, retries: 2 conda: - "../envs/environment.yaml" + "../envs/retrieve.yaml" script: "../scripts/retrieve_monthly_fuel_prices.py" diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index ac58136a..00a729b0 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -153,7 +153,7 @@ if __name__ == "__main__": clusters="37", opts="", ll="v1.0", - sector_opts="168H-T-H-B-I-solar+p3-dist1", + sector_opts="168H-T-H-B-I-dist1", planning_horizons=2030, ) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index c67d5f8b..46d4bc31 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -561,7 +561,7 @@ if __name__ == "__main__": clusters="37", ll="v1.0", opts="", - sector_opts="1p7-4380H-T-H-B-I-A-solar+p3-dist1", + sector_opts="1p7-4380H-T-H-B-I-A-dist1", planning_horizons=2020, ) diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index b8da5012..bc4a6e14 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -75,10 +75,10 @@ def build_clustered_gas_network(df, bus_regions, length_factor=1.25): return df -def reindex_pipes(df): +def reindex_pipes(df, prefix="gas pipeline"): def make_index(x): connector = " <-> " if x.bidirectional else " -> " - return "gas pipeline " + x.bus0 + connector + x.bus1 + return prefix + " " + x.bus0 + connector + x.bus1 df.index = df.apply(make_index, axis=1) diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 6a3783e7..13736d01 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -170,7 +170,7 @@ def plot_map( line_widths = line_widths.replace(line_lower_threshold, 0) link_widths = link_widths.replace(line_lower_threshold, 0) - fig, ax = plt.subplots(subplot_kw={"projection": ccrs.EqualEarth()}) + fig, ax = plt.subplots(subplot_kw={"projection": proj}) fig.set_size_inches(7, 6) n.plot( @@ -358,7 +358,6 @@ def plot_h2_map(network, regions): n.links.bus0 = n.links.bus0.str.replace(" H2", "") n.links.bus1 = n.links.bus1.str.replace(" H2", "") - proj = ccrs.EqualEarth() regions = regions.to_crs(proj.proj4_init) fig, ax = plt.subplots(figsize=(7, 6), subplot_kw={"projection": proj}) @@ -568,7 +567,7 @@ def plot_ch4_map(network): "biogas": "seagreen", } - fig, ax = plt.subplots(figsize=(7, 6), subplot_kw={"projection": ccrs.EqualEarth()}) + fig, ax = plt.subplots(figsize=(7, 6), subplot_kw={"projection": proj}) n.plot( bus_sizes=bus_sizes, @@ -679,7 +678,7 @@ def plot_map_without(network): # Drop non-electric buses so they don't clutter the plot n.buses.drop(n.buses.index[n.buses.carrier != "AC"], inplace=True) - fig, ax = plt.subplots(figsize=(7, 6), subplot_kw={"projection": ccrs.EqualEarth()}) + fig, ax = plt.subplots(figsize=(7, 6), subplot_kw={"projection": proj}) # PDF has minimum width, so set these to zero line_lower_threshold = 200.0 @@ -993,7 +992,7 @@ def plot_map_perfect( link_widths[link_widths > line_upper_threshold] = line_upper_threshold for year in costs.columns: - fig, ax = plt.subplots(subplot_kw={"projection": ccrs.PlateCarree()}) + fig, ax = plt.subplots(subplot_kw={"projection": proj}) fig.set_size_inches(7, 6) fig.suptitle(year) @@ -1068,7 +1067,7 @@ if __name__ == "__main__": opts="", clusters="37", ll="v1.0", - sector_opts="4380H-T-H-B-I-A-solar+p3-dist1", + sector_opts="4380H-T-H-B-I-A-dist1", ) logging.basicConfig(level=snakemake.config["logging"]["level"]) @@ -1082,6 +1081,10 @@ if __name__ == "__main__": if map_opts["boundaries"] is None: map_opts["boundaries"] = regions.total_bounds[[0, 2, 1, 3]] + [-1, 1, -1, 1] + proj_kwargs = snakemake.params.plotting.get("projection", dict(name="EqualEarth")) + proj_func = getattr(ccrs, proj_kwargs.pop("name")) + proj = proj_func(**proj_kwargs) + if snakemake.params["foresight"] == "perfect": plot_map_perfect( n, diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index b2ec0892..92bf726c 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -282,7 +282,10 @@ def plot_balances(): # remove trailing link ports df.index = [ i[:-1] - if ((i not in ["co2", "NH3", "H2"]) and (i[-1:] in ["0", "1", "2", "3"])) + if ( + (i not in ["co2", "NH3", "H2"]) + and (i[-1:] in ["0", "1", "2", "3", "4"]) + ) else i for i in df.index ] diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 1c3a0ebe..f5ae3919 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -503,7 +503,7 @@ if __name__ == "__main__": opts="", clusters="37", ll="v1.5", - sector_opts="1p7-4380H-T-H-B-I-A-solar+p3-dist1", + sector_opts="1p7-4380H-T-H-B-I-A-dist1", ) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index ba92e137..22fb1bcc 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1321,7 +1321,7 @@ def add_storage_and_grids(n, costs): h2_pipes["p_nom"] = 0.0 - if "custom_h2_pipelines" in snakemake.input: + if snakemake.input.get("custom_h2_pipelines"): fn = snakemake.input.custom_h2_pipelines custom_pipes = pd.read_csv(fn, index_col=0) @@ -1795,28 +1795,29 @@ def add_heat(n, costs): ## Add heat pumps - heat_pump_type = "air" if "urban" in name else "ground" + heat_pump_types = ["air"] if "urban" in name else ["ground", "air"] - costs_name = f"{name_type} {heat_pump_type}-sourced heat pump" - efficiency = ( - cop[heat_pump_type][nodes] - if options["time_dep_hp_cop"] - else costs.at[costs_name, "efficiency"] - ) + for heat_pump_type in heat_pump_types: + costs_name = f"{name_type} {heat_pump_type}-sourced heat pump" + efficiency = ( + cop[heat_pump_type][nodes] + if options["time_dep_hp_cop"] + else costs.at[costs_name, "efficiency"] + ) - n.madd( - "Link", - nodes, - suffix=f" {name} {heat_pump_type} heat pump", - bus0=nodes, - bus1=nodes + f" {name} heat", - carrier=f"{name} {heat_pump_type} heat pump", - efficiency=efficiency, - capital_cost=costs.at[costs_name, "efficiency"] - * costs.at[costs_name, "fixed"], - p_nom_extendable=True, - lifetime=costs.at[costs_name, "lifetime"], - ) + n.madd( + "Link", + nodes, + suffix=f" {name} {heat_pump_type} heat pump", + bus0=nodes, + bus1=nodes + f" {name} heat", + carrier=f"{name} {heat_pump_type} heat pump", + efficiency=efficiency, + capital_cost=costs.at[costs_name, "efficiency"] + * costs.at[costs_name, "fixed"], + p_nom_extendable=True, + lifetime=costs.at[costs_name, "lifetime"], + ) if options["tes"]: n.add("Carrier", name + " water tanks") @@ -3568,7 +3569,7 @@ if __name__ == "__main__": opts="", clusters="37", ll="v1.0", - sector_opts="CO2L0-24H-T-H-B-I-A-solar+p3-dist1", + sector_opts="CO2L0-24H-T-H-B-I-A-dist1", planning_horizons="2030", ) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 1c37bfd2..9fc41555 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -890,6 +890,7 @@ def solve_network(n, config, solving, opts="", **kwargs): "linearized_unit_commitment", False ) kwargs["assign_all_duals"] = cf_solving.get("assign_all_duals", False) + kwargs["io_api"] = cf_solving.get("io_api", None) if kwargs["solver_name"] == "gurobi": logging.getLogger("gurobipy").setLevel(logging.CRITICAL) @@ -943,7 +944,7 @@ if __name__ == "__main__": opts="", clusters="37", ll="v1.0", - sector_opts="CO2L0-1H-T-H-B-I-A-solar+p3-dist1", + sector_opts="CO2L0-1H-T-H-B-I-A-dist1", planning_horizons="2030", ) configure_logging(snakemake)