diff --git a/Snakefile b/Snakefile index 9064b6cb..e31670ee 100644 --- a/Snakefile +++ b/Snakefile @@ -25,7 +25,7 @@ CDIR = RDIR if not run.get("shared_cutouts") else "" LOGS = "logs/" + RDIR BENCHMARKS = "benchmarks/" + RDIR -RESOURCES = "resources/" + RDIR if run.get("shared_resources") else "resources/" +RESOURCES = "resources/" + RDIR if not run.get("shared_resources") else "resources/" RESULTS = "results/" + RDIR diff --git a/config.default.yaml b/config.default.yaml index cb9863f2..23e066b3 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -11,6 +11,7 @@ logging: run: name: "" # use this to keep track of runs with different settings + disable_progressbar: false # set to true to disable the progressbar shared_resources: false # set to true to share the default resources across runs shared_cutouts: false # set to true to share the default cutout(s) across runs diff --git a/scripts/_helpers.py b/scripts/_helpers.py index dd815c48..ceeb69f9 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -264,15 +264,18 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False): return costs -def progress_retrieve(url, file): - with tqdm(unit="B", unit_scale=True, unit_divisor=1024, miniters=1) as t: +def progress_retrieve(url, file, disable=False): + if disable: + urllib.request.urlretrieve(url, file) + else: + with tqdm(unit="B", unit_scale=True, unit_divisor=1024, miniters=1) as t: - def update_to(b=1, bsize=1, tsize=None): - if tsize is not None: - t.total = tsize - t.update(b * bsize - t.n) + def update_to(b=1, bsize=1, tsize=None): + if tsize is not None: + t.total = tsize + t.update(b * bsize - t.n) - urllib.request.urlretrieve(url, file, reporthook=update_to) + urllib.request.urlretrieve(url, file, reporthook=update_to) def get_aggregation_strategies(aggregation_strategies): diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 72e268f9..7e3da9ff 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -398,6 +398,7 @@ def idees_per_country(ct, year, base_dir): def build_idees(countries, year): nprocesses = snakemake.threads + disable_progress = snakemake.config["run"].get("disable_progressbar", False) func = partial(idees_per_country, year=year, base_dir=snakemake.input.idees) tqdm_kwargs = dict( @@ -405,6 +406,7 @@ def build_idees(countries, year): unit=" country", total=len(countries), desc="Build from IDEES database", + disable=disable_progress, ) with mute_print(): with mp.Pool(processes=nprocesses) as pool: diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index cfa1cbc2..fd99d9d6 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -179,6 +179,7 @@ def add_non_eu28_industrial_energy_demand(demand): def industrial_energy_demand(countries, year): nprocesses = snakemake.threads + disable_progress = snakemake.config["run"].get("disable_progressbar", False) func = partial( industrial_energy_demand_per_country, year=year, jrc_dir=snakemake.input.jrc ) @@ -187,6 +188,7 @@ def industrial_energy_demand(countries, year): unit=" country", total=len(countries), desc="Build industrial energy demand", + disable=disable_progress, ) with mp.Pool(processes=nprocesses) as pool: demand_l = list(tqdm(pool.imap(func, countries), **tqdm_kwargs)) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 39e73cd8..952c7382 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -245,7 +245,9 @@ def industry_production_per_country(country, year, eurostat_dir, jrc_dir): def industry_production(countries, year, eurostat_dir, jrc_dir): - nprocesses = 1 # snakemake.threads + nprocesses = snakemake.threads + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + func = partial( industry_production_per_country, year=year, @@ -257,6 +259,7 @@ def industry_production(countries, year, eurostat_dir, jrc_dir): unit=" country", total=len(countries), desc="Build industry production", + disable=disable_progress, ) with mp.Pool(processes=nprocesses) as pool: demand_l = list(tqdm(pool.imap(func, countries), **tqdm_kwargs)) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 29975197..c6e4271c 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -204,7 +204,7 @@ if __name__ == "__main__": configure_logging(snakemake) nprocesses = int(snakemake.threads) - noprogress = not snakemake.config["atlite"].get("show_progress", False) + noprogress = snakemake.config["run"].get("disable_progressbar", True) config = snakemake.config["renewable"][snakemake.wildcards.technology] resource = config["resource"] # pv panel config / wind turbine config correction_factor = config.get("correction_factor", 1.0) @@ -369,3 +369,4 @@ if __name__ == "__main__": ds["profile"] = ds["profile"].where(ds["profile"] >= min_p_max_pu, 0) ds.to_netcdf(snakemake.output.profile) + client.shutdown() diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index cba92f18..b3a6cb4e 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -64,7 +64,8 @@ if __name__ == "__main__": to_fn = Path(f"{rootpath}/data") logger.info(f"Downloading databundle from '{url}'.") - progress_retrieve(url, tarball_fn) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + progress_retrieve(url, tarball_fn, disable=disable_progress) logger.info("Extracting databundle.") tarfile.open(tarball_fn).extractall(to_fn) diff --git a/scripts/retrieve_gas_infrastructure_data.py b/scripts/retrieve_gas_infrastructure_data.py index 82d79caf..dda7bd8c 100644 --- a/scripts/retrieve_gas_infrastructure_data.py +++ b/scripts/retrieve_gas_infrastructure_data.py @@ -32,7 +32,8 @@ if __name__ == "__main__": to_fn = Path(f"{rootpath}/data/gas_network/scigrid-gas") logger.info(f"Downloading databundle from '{url}'.") - progress_retrieve(url, zip_fn) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + progress_retrieve(url, zip_fn, disable=disable_progress) logger.info("Extracting databundle.") zipfile.ZipFile(zip_fn).extractall(to_fn) diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py index 991271b4..0526d01e 100644 --- a/scripts/retrieve_sector_databundle.py +++ b/scripts/retrieve_sector_databundle.py @@ -29,7 +29,8 @@ if __name__ == "__main__": to_fn = Path("data") logger.info(f"Downloading databundle from '{url}'.") - progress_retrieve(url, tarball_fn) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + progress_retrieve(url, tarball_fn, disable=disable_progress) logger.info("Extracting databundle.") tarfile.open(tarball_fn).extractall(to_fn) diff --git a/test/config.myopic.yaml b/test/config.myopic.yaml index f215c63c..f51dcd77 100644 --- a/test/config.myopic.yaml +++ b/test/config.myopic.yaml @@ -2,8 +2,10 @@ # # SPDX-License-Identifier: CC0-1.0 + run: name: "test-sector-myopic" + disable_progressbar: true shared_resources: true shared_cutouts: true diff --git a/test/config.overnight.yaml b/test/config.overnight.yaml index a41c6b53..5130c28d 100644 --- a/test/config.overnight.yaml +++ b/test/config.overnight.yaml @@ -2,11 +2,14 @@ # # SPDX-License-Identifier: CC0-1.0 + run: name: "test-sector-overnight" + disable_progressbar: true shared_resources: true shared_cutouts: true + scenario: ll: - v1.5 diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 330b3f62..d95858de 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -4,8 +4,10 @@ tutorial: true + run: name: "test-elec" # use this to keep track of runs with different settings + disable_progressbar: true shared_resources: true shared_cutouts: true