From 825d3e84d228468020fac806fcbca59b3d06ef13 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 22 Feb 2023 14:12:26 +0100 Subject: [PATCH 1/7] replace progressbar by tqdm --- envs/environment.yaml | 1 - scripts/_helpers.py | 21 ++++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 9325ac09..5ffe0d71 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -33,7 +33,6 @@ dependencies: - networkx - scipy - shapely>=2.0 -- progressbar2 - pyomo - matplotlib<3.6 - proj diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 5032cdf9..634cc59b 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -4,7 +4,8 @@ # SPDX-License-Identifier: MIT from pathlib import Path - +import urllib +from tqdm import tqdm import pandas as pd REGION_COLS = ["geometry", "name", "x", "y", "country"] @@ -251,16 +252,14 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False): def progress_retrieve(url, file): - import urllib - - from progressbar import ProgressBar - - pbar = ProgressBar(0, 100) - - def dlProgress(count, blockSize, totalSize): - pbar.update(int(count * blockSize * 100 / totalSize)) - - urllib.request.urlretrieve(url, file, reporthook=dlProgress) + + with tqdm(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, position=0, leave=True) as t: + def update_to(b=1, bsize=1, tsize=None): + if tsize is not None: + t.total = tsize + t.update(b * bsize - t.n) + + urllib.request.urlretrieve(url, file, reporthook=update_to) def get_aggregation_strategies(aggregation_strategies): From 64745e7ec2ef7008d128d9ef2234f5d78a2243b8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Feb 2023 13:21:26 +0000 Subject: [PATCH 2/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/_helpers.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 634cc59b..40404d0d 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -3,10 +3,11 @@ # # SPDX-License-Identifier: MIT -from pathlib import Path import urllib -from tqdm import tqdm +from pathlib import Path + import pandas as pd +from tqdm import tqdm REGION_COLS = ["geometry", "name", "x", "y", "country"] @@ -252,13 +253,15 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False): def progress_retrieve(url, file): - - with tqdm(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, position=0, leave=True) as t: + with tqdm( + unit="B", unit_scale=True, unit_divisor=1024, miniters=1, position=0, leave=True + ) as t: + def update_to(b=1, bsize=1, tsize=None): if tsize is not None: t.total = tsize t.update(b * bsize - t.n) - + urllib.request.urlretrieve(url, file, reporthook=update_to) From ef047c20fa8173b93776aa112d1c7d4e31e048a9 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 22 Feb 2023 14:38:53 +0100 Subject: [PATCH 3/7] ci: flush python terminal output --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 26111c94..778c8da3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,6 +20,7 @@ on: env: CACHE_NUMBER: 1 # Change this value to manually reset the environment cache + PYTHONUNBUFFERED: 1 # to flush progress bars jobs: build: From 44bb81af468960dde44099b22fccee4b628647dc Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 22 Feb 2023 14:40:02 +0100 Subject: [PATCH 4/7] helpers: undo position fixing in tqdm --- scripts/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 40404d0d..57b84691 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -254,7 +254,7 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False): def progress_retrieve(url, file): with tqdm( - unit="B", unit_scale=True, unit_divisor=1024, miniters=1, position=0, leave=True + unit="B", unit_scale=True, unit_divisor=1024, miniters=1 ) as t: def update_to(b=1, bsize=1, tsize=None): From b134a395b44d1023f1c9f5cf63e2feffa1faf9c0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Feb 2023 13:40:35 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/_helpers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 57b84691..c32baf11 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -253,9 +253,7 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False): def progress_retrieve(url, file): - with tqdm( - unit="B", unit_scale=True, unit_divisor=1024, miniters=1 - ) as t: + with tqdm(unit="B", unit_scale=True, unit_divisor=1024, miniters=1) as t: def update_to(b=1, bsize=1, tsize=None): if tsize is not None: From ef07ad0b973f1c8432b26739ee6938f89c9c7a14 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 22 Feb 2023 15:01:49 +0100 Subject: [PATCH 6/7] ci: revert python buffer --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 778c8da3..26111c94 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,7 +20,6 @@ on: env: CACHE_NUMBER: 1 # Change this value to manually reset the environment cache - PYTHONUNBUFFERED: 1 # to flush progress bars jobs: build: From 3642d5c834f0f8358828de7f73cf15c4a4926d44 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 22 Feb 2023 15:07:57 +0100 Subject: [PATCH 7/7] build_renewable_profiles: remove progressbar import --- scripts/build_renewable_profiles.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index c8a0fc42..a132115b 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -187,7 +187,6 @@ import time import atlite import geopandas as gpd import numpy as np -import progressbar as pgb import xarray as xr from _helpers import configure_logging from dask.distributed import Client, LocalCluster @@ -203,7 +202,6 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_renewable_profiles", technology="solar") configure_logging(snakemake) - pgb.streams.wrap_stderr() nprocesses = int(snakemake.threads) noprogress = not snakemake.config["atlite"].get("show_progress", False)