From 52e4a1b285a16ebd487f599018b1b250368d7ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6rsch?= Date: Thu, 23 Jan 2020 09:55:59 +0100 Subject: [PATCH] environment: Free pyproj from version constraint (fixes #119) (#120) * environment: Free pyproj from version constraint (fixes #119) proj was constrained to 1.9.6, since cartopy was incompatible with recent proj versions. * build_renewable_profiles: Import gdal and geokit after forking GDAL sets up a shared context on module import which seems to contain a handle to the PROJ database for coordinate reference systems. By forking these handles seem to become invalidated or at least are not usable in parallel anymore. Instead importing gdal only after setting up the different processes fixes the database disk image is malformed proj error. --- environment.docs.yaml | 2 +- environment.yaml | 2 +- scripts/build_renewable_profiles.py | 14 +++++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/environment.docs.yaml b/environment.docs.yaml index e9062872..79031624 100644 --- a/environment.docs.yaml +++ b/environment.docs.yaml @@ -35,7 +35,7 @@ dependencies: # GIS dependencies have to come all from conda-forge - conda-forge::cartopy - conda-forge::fiona - - conda-forge::pyproj<=1.9.6 # until cartopy release with proj>=6.2 + - conda-forge::proj - conda-forge::pyshp - conda-forge::geopandas - conda-forge::rasterio diff --git a/environment.yaml b/environment.yaml index 2fe09f2b..df14f482 100644 --- a/environment.yaml +++ b/environment.yaml @@ -36,7 +36,7 @@ dependencies: # GIS dependencies have to come all from conda-forge - conda-forge::cartopy - conda-forge::fiona - - conda-forge::pyproj<=1.9.6 # until cartopy release with proj>=6.2 + - conda-forge::proj - conda-forge::pyshp - conda-forge::geopandas - conda-forge::rasterio diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 4d63ca9e..f8e83205 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -163,9 +163,6 @@ import xarray as xr import pandas as pd import multiprocessing as mp -import glaes as gl -import geokit as gk -from osgeo import gdal from scipy.sparse import csr_matrix, vstack from pypsa.geo import haversine @@ -176,6 +173,12 @@ import progressbar as pgb bounds = dx = dy = config = paths = gebco = clc = natura = None def init_globals(bounds_xXyY, n_dx, n_dy, n_config, n_paths): + # Late import so that the GDAL Context is only created in the new processes + global gl, gk, gdal + import glaes as gl + import geokit as gk + from osgeo import gdal as gdal + # global in each process of the multiprocessing.Pool global bounds, dx, dy, config, paths, gebco, clc, natura @@ -267,6 +270,11 @@ if __name__ == '__main__': # mp.set_start_method('spawn') with mp.Pool(initializer=init_globals, initargs=(bounds_xXyY, dx, dy, config, paths), maxtasksperchild=20, processes=snakemake.config['atlite'].get('nprocesses', 2)) as pool: + + # The GDAL library creates a GDAL context on module import, which may not be shared over multiple + # processes or the PROJ4 library has a hickup, so we import only after forking. + import geokit as gk + regions = gk.vector.extractFeatures(paths["regions"], onlyAttr=True) buses = pd.Index(regions['name'], name="bus") widgets = [