allow to disable progressbar

Snakefile

@@ -25,7 +25,7 @@ CDIR = RDIR if not run.get("shared_cutouts") else ""
 
 LOGS = "logs/" + RDIR
 BENCHMARKS = "benchmarks/" + RDIR
-RESOURCES = "resources/" + RDIR if run.get("shared_resources") else "resources/"
+RESOURCES = "resources/" + RDIR if not run.get("shared_resources") else "resources/"
 RESULTS = "results/" + RDIR
 
 

config.default.yaml

@@ -11,6 +11,7 @@ logging:
 
 run:
   name: "" # use this to keep track of runs with different settings
+  disable_progressbar: false # set to true to disable the progressbar
   shared_resources: false # set to true to share the default resources across runs
   shared_cutouts: false # set to true to share the default cutout(s) across runs
 

scripts/_helpers.py

@@ -264,15 +264,18 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False):
     return costs
 
 
-def progress_retrieve(url, file):
-    with tqdm(unit="B", unit_scale=True, unit_divisor=1024, miniters=1) as t:
+def progress_retrieve(url, file, disable=False):
+    if disable:
+        urllib.request.urlretrieve(url, file)
+    else:
+        with tqdm(unit="B", unit_scale=True, unit_divisor=1024, miniters=1) as t:
 
-        def update_to(b=1, bsize=1, tsize=None):
-            if tsize is not None:
-                t.total = tsize
-            t.update(b * bsize - t.n)
+            def update_to(b=1, bsize=1, tsize=None):
+                if tsize is not None:
+                    t.total = tsize
+                t.update(b * bsize - t.n)
 
-        urllib.request.urlretrieve(url, file, reporthook=update_to)
+            urllib.request.urlretrieve(url, file, reporthook=update_to)
 
 
 def get_aggregation_strategies(aggregation_strategies):

scripts/build_energy_totals.py

@@ -398,6 +398,7 @@ def idees_per_country(ct, year, base_dir):
 
 def build_idees(countries, year):
     nprocesses = snakemake.threads
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
 
     func = partial(idees_per_country, year=year, base_dir=snakemake.input.idees)
     tqdm_kwargs = dict(
@@ -405,6 +406,7 @@ def build_idees(countries, year):
         unit=" country",
         total=len(countries),
         desc="Build from IDEES database",
+        disable=disable_progress,
     )
     with mute_print():
         with mp.Pool(processes=nprocesses) as pool:

scripts/build_industrial_energy_demand_per_country_today.py

@@ -179,6 +179,7 @@ def add_non_eu28_industrial_energy_demand(demand):
 
 def industrial_energy_demand(countries, year):
     nprocesses = snakemake.threads
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
     func = partial(
         industrial_energy_demand_per_country, year=year, jrc_dir=snakemake.input.jrc
     )
@@ -187,6 +188,7 @@ def industrial_energy_demand(countries, year):
         unit=" country",
         total=len(countries),
         desc="Build industrial energy demand",
+        disable=disable_progress,
     )
     with mp.Pool(processes=nprocesses) as pool:
         demand_l = list(tqdm(pool.imap(func, countries), **tqdm_kwargs))

scripts/build_industrial_production_per_country.py

@@ -245,7 +245,9 @@ def industry_production_per_country(country, year, eurostat_dir, jrc_dir):
 
 
 def industry_production(countries, year, eurostat_dir, jrc_dir):
-    nprocesses = 1  # snakemake.threads
+    nprocesses = snakemake.threads
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
+
     func = partial(
         industry_production_per_country,
         year=year,
@@ -257,6 +259,7 @@ def industry_production(countries, year, eurostat_dir, jrc_dir):
         unit=" country",
         total=len(countries),
         desc="Build industry production",
+        disable=disable_progress,
     )
     with mp.Pool(processes=nprocesses) as pool:
         demand_l = list(tqdm(pool.imap(func, countries), **tqdm_kwargs))

scripts/build_renewable_profiles.py

@@ -204,7 +204,7 @@ if __name__ == "__main__":
     configure_logging(snakemake)
 
     nprocesses = int(snakemake.threads)
-    noprogress = not snakemake.config["atlite"].get("show_progress", False)
+    noprogress = snakemake.config["run"].get("disable_progressbar", True)
     config = snakemake.config["renewable"][snakemake.wildcards.technology]
     resource = config["resource"]  # pv panel config / wind turbine config
     correction_factor = config.get("correction_factor", 1.0)
@@ -369,3 +369,4 @@ if __name__ == "__main__":
         ds["profile"] = ds["profile"].where(ds["profile"] >= min_p_max_pu, 0)
 
     ds.to_netcdf(snakemake.output.profile)
+    client.shutdown()

scripts/retrieve_databundle.py

@@ -64,7 +64,8 @@ if __name__ == "__main__":
     to_fn = Path(f"{rootpath}/data")
 
     logger.info(f"Downloading databundle from '{url}'.")
-    progress_retrieve(url, tarball_fn)
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
+    progress_retrieve(url, tarball_fn, disable=disable_progress)
 
     logger.info("Extracting databundle.")
     tarfile.open(tarball_fn).extractall(to_fn)

scripts/retrieve_gas_infrastructure_data.py

@@ -32,7 +32,8 @@ if __name__ == "__main__":
     to_fn = Path(f"{rootpath}/data/gas_network/scigrid-gas")
 
     logger.info(f"Downloading databundle from '{url}'.")
-    progress_retrieve(url, zip_fn)
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
+    progress_retrieve(url, zip_fn, disable=disable_progress)
 
     logger.info("Extracting databundle.")
     zipfile.ZipFile(zip_fn).extractall(to_fn)

scripts/retrieve_sector_databundle.py

@@ -29,7 +29,8 @@ if __name__ == "__main__":
     to_fn = Path("data")
 
     logger.info(f"Downloading databundle from '{url}'.")
-    progress_retrieve(url, tarball_fn)
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
+    progress_retrieve(url, tarball_fn, disable=disable_progress)
 
     logger.info("Extracting databundle.")
     tarfile.open(tarball_fn).extractall(to_fn)

test/config.myopic.yaml

@@ -2,8 +2,10 @@
 #
 # SPDX-License-Identifier: CC0-1.0
 
+
 run:
   name: "test-sector-myopic"
+  disable_progressbar: true
   shared_resources: true
   shared_cutouts: true
 

test/config.overnight.yaml

@@ -2,11 +2,14 @@
 #
 # SPDX-License-Identifier: CC0-1.0
 
+
 run:
   name: "test-sector-overnight"
+  disable_progressbar: true
   shared_resources: true
   shared_cutouts: true
 
+
 scenario:
   ll:
   - v1.5

test/config.test1.yaml

@@ -4,8 +4,10 @@
 
 tutorial: true
 
+
 run:
   name: "test-elec" # use this to keep track of runs with different settings
+  disable_progressbar: true
   shared_resources: true
   shared_cutouts: true