environment: Free pyproj from version constraint (fixes #119) (#120)

* environment: Free pyproj from version constraint (fixes #119)

proj was constrained to 1.9.6, since cartopy was incompatible with
recent proj versions.

* build_renewable_profiles: Import gdal and geokit after forking

GDAL sets up a shared context on module import which seems to contain a
handle to the PROJ database for coordinate reference systems. By forking
these handles seem to become invalidated or at least are not usable in parallel
anymore.

Instead importing gdal only after setting up the different processes fixes
the database disk image is malformed proj error.

environment.docs.yaml

@@ -35,7 +35,7 @@ dependencies:
   # GIS dependencies have to come all from conda-forge
   - conda-forge::cartopy
   - conda-forge::fiona
-  - conda-forge::pyproj<=1.9.6 # until cartopy release with proj>=6.2
+  - conda-forge::proj
   - conda-forge::pyshp
   - conda-forge::geopandas
   - conda-forge::rasterio

environment.yaml

@@ -36,7 +36,7 @@ dependencies:
   # GIS dependencies have to come all from conda-forge
   - conda-forge::cartopy
   - conda-forge::fiona
-  - conda-forge::pyproj<=1.9.6 # until cartopy release with proj>=6.2
+  - conda-forge::proj
   - conda-forge::pyshp
   - conda-forge::geopandas
   - conda-forge::rasterio

scripts/build_renewable_profiles.py

@@ -163,9 +163,6 @@ import xarray as xr
 import pandas as pd
 import multiprocessing as mp
 
-import glaes as gl
-import geokit as gk
-from osgeo import gdal
 from scipy.sparse import csr_matrix, vstack
 
 from pypsa.geo import haversine
@@ -176,6 +173,12 @@ import progressbar as pgb
 
 bounds = dx = dy = config = paths = gebco = clc = natura = None
 def init_globals(bounds_xXyY, n_dx, n_dy, n_config, n_paths):
+    # Late import so that the GDAL Context is only created in the new processes
+    global gl, gk, gdal
+    import glaes as gl
+    import geokit as gk
+    from osgeo import gdal as gdal
+
     # global in each process of the multiprocessing.Pool
     global bounds, dx, dy, config, paths, gebco, clc, natura
 
@@ -267,6 +270,11 @@ if __name__ == '__main__':
     # mp.set_start_method('spawn')
     with mp.Pool(initializer=init_globals, initargs=(bounds_xXyY, dx, dy, config, paths),
                  maxtasksperchild=20, processes=snakemake.config['atlite'].get('nprocesses', 2)) as pool:
+
+        # The GDAL library creates a GDAL context on module import, which may not be shared over multiple
+        # processes or the PROJ4 library has a hickup, so we import only after forking.
+        import geokit as gk
+
         regions = gk.vector.extractFeatures(paths["regions"], onlyAttr=True)
         buses = pd.Index(regions['name'], name="bus")
         widgets = [