Merge pull request #662 from PyPSA/remove-vresutils

Remove vresutils

doc/release_notes.rst

@@ -23,6 +23,8 @@ Upcoming Release
   hydrogen fuel cell. Add switches for both re-electrification options under
   ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``.
 
+* Remove ``vresutils`` dependency.
+
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================
 

doc/requirements.txt

@@ -9,7 +9,6 @@ sphinxcontrib-bibtex
 myst-parser  # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820
 
 pypsa
-vresutils>=0.3.1
 powerplantmatching>=0.5.5
 atlite>=0.2.9
 dask[distributed]

envs/environment.fixed.yaml

@@ -226,7 +226,7 @@ dependencies:
 - nspr=4.35
 - nss=3.88
 - numexpr=2.8.3
-- numpy=1.23.5
+- numpy=1.24
 - openjdk=17.0.3
 - openjpeg=2.5.0
 - openpyxl=3.1.0
@@ -378,4 +378,3 @@ dependencies:
   - highspy==1.5.0.dev0
   - pybind11==2.10.3
   - tsam==2.2.2
-  - vresutils==0.3.1

envs/environment.yaml

@@ -25,7 +25,7 @@ dependencies:
 - pytables
 - lxml
 - powerplantmatching>=0.5.5
-- numpy<1.24
+- numpy
 - pandas>=1.4
 - geopandas>=0.11.0
 - xarray
@@ -55,5 +55,4 @@ dependencies:
 - rasterio!=1.2.10
 
 - pip:
-  - vresutils>=0.3.1
   - tsam>=1.1.0

rules/solve_electricity.smk

@@ -14,8 +14,6 @@ rule solve_network:
         ),
         python=LOGS
         + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log",
-        memory=LOGS
-        + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log",
     benchmark:
         BENCHMARKS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"
     threads: 4
@@ -41,8 +39,6 @@ rule solve_operations_network:
         ),
         python=LOGS
         + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log",
-        memory=LOGS
-        + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log",
     benchmark:
         (
             BENCHMARKS

rules/solve_myopic.smk

@@ -90,8 +90,6 @@ rule solve_sector_network_myopic:
         + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
         python=LOGS
         + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
-        memory=LOGS
-        + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log",
     threads: 4
     resources:
         mem_mb=config["solving"]["mem"],

rules/solve_overnight.smk

@@ -21,8 +21,6 @@ rule solve_sector_network:
         + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
         python=LOGS
         + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
-        memory=LOGS
-        + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log",
     threads: config["solving"]["solver"].get("threads", 4)
     resources:
         mem_mb=config["solving"]["mem"],

scripts/add_electricity.py

@@ -85,16 +85,18 @@ It further adds extendable ``generators`` with **zero** capacity for
 """
 
 import logging
+from itertools import product
 
 import geopandas as gpd
 import numpy as np
 import pandas as pd
 import powerplantmatching as pm
 import pypsa
+import scipy.sparse as sparse
 import xarray as xr
 from _helpers import configure_logging, update_p_nom_max
 from powerplantmatching.export import map_country_bus
-from vresutils import transfer as vtransfer
+from shapely.prepared import prep
 
 idx = pd.IndexSlice
 
@@ -216,6 +218,21 @@ def load_powerplants(ppl_fn):
     )
 
 
+def shapes_to_shapes(orig, dest):
+    """
+    Adopted from vresutils.transfer.Shapes2Shapes()
+    """
+    orig_prepped = list(map(prep, orig))
+    transfer = sparse.lil_matrix((len(dest), len(orig)), dtype=float)
+
+    for i, j in product(range(len(dest)), range(len(orig))):
+        if orig_prepped[j].intersects(dest[i]):
+            area = orig[j].intersection(dest[i]).area
+            transfer[i, j] = area / dest[i].area
+
+    return transfer
+
+
 def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0):
     substation_lv_i = n.buses.index[n.buses["substation_lv"]]
     regions = gpd.read_file(regions).set_index("name").reindex(substation_lv_i)
@@ -232,9 +249,7 @@ def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0):
             return pd.DataFrame({group.index[0]: l})
         else:
             nuts3_cntry = nuts3.loc[nuts3.country == cntry]
-            transfer = vtransfer.Shapes2Shapes(
+            transfer = shapes_to_shapes(group, nuts3_cntry.geometry).T.tocsr()
-                group, nuts3_cntry.geometry, normed=False
-            ).T.tocsr()
             gdp_n = pd.Series(
                 transfer.dot(nuts3_cntry["gdp"].fillna(1.0).values), index=group.index
             )

scripts/prepare_sector_network.py

@@ -22,13 +22,13 @@ from _helpers import (
     override_component_attrs,
     update_config_with_sector_opts,
 )
+from add_electricity import calculate_annuity
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation
 from pypsa.geo import haversine_pts
 from pypsa.io import import_components_from_dataframe
 from scipy.stats import beta
-from vresutils.costdata import annuity
 
 logger = logging.getLogger(__name__)
 
@@ -742,7 +742,7 @@ def prepare_costs(cost_file, config, nyears):
     costs = costs.fillna(config["fill_values"])
 
     def annuity_factor(v):
-        return annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100
+        return calculate_annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100
 
     costs["fixed"] = [
         annuity_factor(v) * v["investment"] * nyears for i, v in costs.iterrows()
@@ -851,7 +851,7 @@ def add_wave(n, wave_cost_factor):
     capacity = pd.Series({"Attenuator": 750, "F2HB": 1000, "MultiPA": 600})
 
     # in EUR/MW
-    annuity_factor = annuity(25, 0.07) + 0.03
+    annuity_factor = calculate_annuity(25, 0.07) + 0.03
     costs = (
         1e6
         * wave_cost_factor

scripts/solve_network.py

@@ -38,7 +38,6 @@ from _helpers import (
     override_component_attrs,
     update_config_with_sector_opts,
 )
-from vresutils.benchmark import memory_logger
 
 logger = logging.getLogger(__name__)
 pypsa.pf.logger.setLevel(logging.WARNING)
@@ -667,23 +666,17 @@ if __name__ == "__main__":
 
     np.random.seed(solve_opts.get("seed", 123))
 
-    fn = getattr(snakemake.log, "memory", None)
+    if "overrides" in snakemake.input.keys():
-    with memory_logger(filename=fn, interval=30.0) as mem:
+        overrides = override_component_attrs(snakemake.input.overrides)
-        if "overrides" in snakemake.input.keys():
+        n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
-            overrides = override_component_attrs(snakemake.input.overrides)
+    else:
-            n = pypsa.Network(
+        n = pypsa.Network(snakemake.input.network)
-                snakemake.input.network, override_component_attrs=overrides
-            )
-        else:
-            n = pypsa.Network(snakemake.input.network)
 
-        n = prepare_network(n, solve_opts, config=snakemake.config)
+    n = prepare_network(n, solve_opts, config=snakemake.config)
 
-        n = solve_network(
+    n = solve_network(
-            n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver
+        n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver
-        )
+    )
 
-        n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
+    n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
-        n.export_to_netcdf(snakemake.output[0])
+    n.export_to_netcdf(snakemake.output[0])
-
-    logger.info("Maximum memory usage: {}".format(mem.mem_usage))

scripts/solve_operations_network.py

@@ -17,7 +17,6 @@ from _helpers import (
     update_config_with_sector_opts,
 )
 from solve_network import prepare_network, solve_network
-from vresutils.benchmark import memory_logger
 
 logger = logging.getLogger(__name__)
 
@@ -46,23 +45,17 @@ if __name__ == "__main__":
 
     np.random.seed(solve_opts.get("seed", 123))
 
-    fn = getattr(snakemake.log, "memory", None)
+    if "overrides" in snakemake.input:
-    with memory_logger(filename=fn, interval=30.0) as mem:
+        overrides = override_component_attrs(snakemake.input.overrides)
-        if "overrides" in snakemake.input:
+        n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
-            overrides = override_component_attrs(snakemake.input.overrides)
+    else:
-            n = pypsa.Network(
+        n = pypsa.Network(snakemake.input.network)
-                snakemake.input.network, override_component_attrs=overrides
-            )
-        else:
-            n = pypsa.Network(snakemake.input.network)
 
-        n.optimize.fix_optimal_capacities()
+    n.optimize.fix_optimal_capacities()
-        n = prepare_network(n, solve_opts, config=snakemake.config)
+    n = prepare_network(n, solve_opts, config=snakemake.config)
-        n = solve_network(
+    n = solve_network(
-            n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver
+        n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver
-        )
+    )
 
-        n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
+    n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
-        n.export_to_netcdf(snakemake.output[0])
+    n.export_to_netcdf(snakemake.output[0])
-
-    logger.info("Maximum memory usage: {}".format(mem.mem_usage))