From 7b4480f5977375ff607124bdc4d1eafed129e90e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 Feb 2023 21:27:10 +0000 Subject: [PATCH 1/8] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/docformatter: v1.6.0.rc1 → v1.5.1](https://github.com/PyCQA/docformatter/compare/v1.6.0.rc1...v1.5.1) - [github.com/macisamuele/language-formatters-pre-commit-hooks: v2.6.0 → v2.7.0](https://github.com/macisamuele/language-formatters-pre-commit-hooks/compare/v2.6.0...v2.7.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 98c8e569..71b9ba03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: # Make docstrings PEP 257 compliant - repo: https://github.com/PyCQA/docformatter - rev: v1.6.0.rc1 + rev: v1.5.1 hooks: - id: docformatter args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] @@ -67,7 +67,7 @@ repos: # Do YAML formatting (before the linter checks it for misses) - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.6.0 + rev: v2.7.0 hooks: - id: pretty-format-yaml args: [--autofix, --indent, "2", --preserve-quotes] From 825d3e84d228468020fac806fcbca59b3d06ef13 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 22 Feb 2023 14:12:26 +0100 Subject: [PATCH 2/8] replace progressbar by tqdm --- envs/environment.yaml | 1 - scripts/_helpers.py | 21 ++++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 9325ac09..5ffe0d71 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -33,7 +33,6 @@ dependencies: - networkx - scipy - shapely>=2.0 -- progressbar2 - pyomo - matplotlib<3.6 - proj diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 5032cdf9..634cc59b 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -4,7 +4,8 @@ # SPDX-License-Identifier: MIT from pathlib import Path - +import urllib +from tqdm import tqdm import pandas as pd REGION_COLS = ["geometry", "name", "x", "y", "country"] @@ -251,16 +252,14 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False): def progress_retrieve(url, file): - import urllib - - from progressbar import ProgressBar - - pbar = ProgressBar(0, 100) - - def dlProgress(count, blockSize, totalSize): - pbar.update(int(count * blockSize * 100 / totalSize)) - - urllib.request.urlretrieve(url, file, reporthook=dlProgress) + + with tqdm(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, position=0, leave=True) as t: + def update_to(b=1, bsize=1, tsize=None): + if tsize is not None: + t.total = tsize + t.update(b * bsize - t.n) + + urllib.request.urlretrieve(url, file, reporthook=update_to) def get_aggregation_strategies(aggregation_strategies): From 64745e7ec2ef7008d128d9ef2234f5d78a2243b8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Feb 2023 13:21:26 +0000 Subject: [PATCH 3/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/_helpers.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 634cc59b..40404d0d 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -3,10 +3,11 @@ # # SPDX-License-Identifier: MIT -from pathlib import Path import urllib -from tqdm import tqdm +from pathlib import Path + import pandas as pd +from tqdm import tqdm REGION_COLS = ["geometry", "name", "x", "y", "country"] @@ -252,13 +253,15 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False): def progress_retrieve(url, file): - - with tqdm(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, position=0, leave=True) as t: + with tqdm( + unit="B", unit_scale=True, unit_divisor=1024, miniters=1, position=0, leave=True + ) as t: + def update_to(b=1, bsize=1, tsize=None): if tsize is not None: t.total = tsize t.update(b * bsize - t.n) - + urllib.request.urlretrieve(url, file, reporthook=update_to) From ef047c20fa8173b93776aa112d1c7d4e31e048a9 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 22 Feb 2023 14:38:53 +0100 Subject: [PATCH 4/8] ci: flush python terminal output --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 26111c94..778c8da3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,6 +20,7 @@ on: env: CACHE_NUMBER: 1 # Change this value to manually reset the environment cache + PYTHONUNBUFFERED: 1 # to flush progress bars jobs: build: From 44bb81af468960dde44099b22fccee4b628647dc Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 22 Feb 2023 14:40:02 +0100 Subject: [PATCH 5/8] helpers: undo position fixing in tqdm --- scripts/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 40404d0d..57b84691 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -254,7 +254,7 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False): def progress_retrieve(url, file): with tqdm( - unit="B", unit_scale=True, unit_divisor=1024, miniters=1, position=0, leave=True + unit="B", unit_scale=True, unit_divisor=1024, miniters=1 ) as t: def update_to(b=1, bsize=1, tsize=None): From b134a395b44d1023f1c9f5cf63e2feffa1faf9c0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Feb 2023 13:40:35 +0000 Subject: [PATCH 6/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/_helpers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 57b84691..c32baf11 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -253,9 +253,7 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False): def progress_retrieve(url, file): - with tqdm( - unit="B", unit_scale=True, unit_divisor=1024, miniters=1 - ) as t: + with tqdm(unit="B", unit_scale=True, unit_divisor=1024, miniters=1) as t: def update_to(b=1, bsize=1, tsize=None): if tsize is not None: From ef07ad0b973f1c8432b26739ee6938f89c9c7a14 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 22 Feb 2023 15:01:49 +0100 Subject: [PATCH 7/8] ci: revert python buffer --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 778c8da3..26111c94 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,7 +20,6 @@ on: env: CACHE_NUMBER: 1 # Change this value to manually reset the environment cache - PYTHONUNBUFFERED: 1 # to flush progress bars jobs: build: From 3642d5c834f0f8358828de7f73cf15c4a4926d44 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 22 Feb 2023 15:07:57 +0100 Subject: [PATCH 8/8] build_renewable_profiles: remove progressbar import --- scripts/build_renewable_profiles.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index c8a0fc42..a132115b 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -187,7 +187,6 @@ import time import atlite import geopandas as gpd import numpy as np -import progressbar as pgb import xarray as xr from _helpers import configure_logging from dask.distributed import Client, LocalCluster @@ -203,7 +202,6 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_renewable_profiles", technology="solar") configure_logging(snakemake) - pgb.streams.wrap_stderr() nprocesses = int(snakemake.threads) noprogress = not snakemake.config["atlite"].get("show_progress", False)