From 25c5751565500aa6c7ea901f358a25186f6a734f Mon Sep 17 00:00:00 2001 From: euronion <42553970+euronion@users.noreply.github.com> Date: Thu, 21 Dec 2023 18:39:06 +0100 Subject: [PATCH 1/4] Check WDPA url also a month forward --- doc/release_notes.rst | 2 ++ rules/retrieve.smk | 35 ++++++++++++++++++++++------------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index ee5a954b..1c259cb5 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -31,6 +31,8 @@ Upcoming Release * Rule ``retrieve_irena`` get updated values for renewables capacities. +* Rule ``retrieve_wdpa`` updated to not only check for current and previous, but also potentially next months dataset availability. + * Split configuration to enable SMR and SMR CC. * The configuration setting for country focus weights when clustering the diff --git a/rules/retrieve.smk b/rules/retrieve.smk index ac89e360..b3969f41 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -246,22 +246,31 @@ if config["enable"]["retrieve"]: if config["enable"]["retrieve"]: - current_month = datetime.now().strftime("%b") - current_year = datetime.now().strftime("%Y") - bYYYY = f"{current_month}{current_year}" + + # Some logic to find the correct file URL + # Sometimes files are released delayed or ahead of schedule, check which file is currently available def check_file_exists(url): - response = requests.head(url) - return response.status_code == 200 + response = requests.head(url) + return response.status_code == 200 - url = f"https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{bYYYY}_Public.zip" + # Basic pattern where WDPA files can be found + url_pattern = "https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{bY}_Public.zip" - if not check_file_exists(url): - prev_month = (datetime.now() - timedelta(30)).strftime("%b") - bYYYY = f"{prev_month}{current_year}" - assert check_file_exists( - f"https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{bYYYY}_Public.zip" - ), "The file does not exist." + # 3-letter month + 4 digit year for current/previous/next month to test + current_monthyear = datetime.now().strftime("%b%Y") + prev_monthyear = (datetime.now() - timedelta(30)).strftime("%b%Y") + next_monthyear = (datetime.now() + timedelta(30)).strftime("%b%Y") + + # Test prioritised: current month -> previous -> next + for bY in [current_monthyear, prev_monthyear, next_monthyear]: + if check_file_exists(url := url_pattern.format(bY=bY)): + break + else: + # If None of the three URLs are working + url = False + + assert url, f"No WDPA files found at {url_pattern} for bY='{current_monthyear}, {prev_monthyear}, or {next_monthyear}'" # Downloading protected area database from WDPA # extract the main zip and then merge the contained 3 zipped shapefiles @@ -269,7 +278,7 @@ if config["enable"]["retrieve"]: rule download_wdpa: input: HTTP.remote( - f"d1gam3xoknrgr2.cloudfront.net/current/WDPA_{bYYYY}_Public_shp.zip", + url, static=True, keep_local=True, ), From 3a474af71fcd5f4793e7ab64f90bce4c295fc173 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Dec 2023 17:40:46 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/retrieve.smk | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index b3969f41..de3a6709 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -246,13 +246,12 @@ if config["enable"]["retrieve"]: if config["enable"]["retrieve"]: - # Some logic to find the correct file URL # Sometimes files are released delayed or ahead of schedule, check which file is currently available def check_file_exists(url): - response = requests.head(url) - return response.status_code == 200 + response = requests.head(url) + return response.status_code == 200 # Basic pattern where WDPA files can be found url_pattern = "https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{bY}_Public.zip" @@ -270,7 +269,9 @@ if config["enable"]["retrieve"]: # If None of the three URLs are working url = False - assert url, f"No WDPA files found at {url_pattern} for bY='{current_monthyear}, {prev_monthyear}, or {next_monthyear}'" + assert ( + url + ), f"No WDPA files found at {url_pattern} for bY='{current_monthyear}, {prev_monthyear}, or {next_monthyear}'" # Downloading protected area database from WDPA # extract the main zip and then merge the contained 3 zipped shapefiles From d817212896a7fd754a5c764ca5d01217db843a2f Mon Sep 17 00:00:00 2001 From: euronion <42553970+euronion@users.noreply.github.com> Date: Fri, 22 Dec 2023 14:40:30 +0100 Subject: [PATCH 3/4] Fix missing bYYYY --- rules/retrieve.smk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index de3a6709..d08f94db 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -254,7 +254,7 @@ if config["enable"]["retrieve"]: return response.status_code == 200 # Basic pattern where WDPA files can be found - url_pattern = "https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{bY}_Public.zip" + url_pattern = "https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{bYYYY}_Public.zip" # 3-letter month + 4 digit year for current/previous/next month to test current_monthyear = datetime.now().strftime("%b%Y") @@ -262,8 +262,8 @@ if config["enable"]["retrieve"]: next_monthyear = (datetime.now() + timedelta(30)).strftime("%b%Y") # Test prioritised: current month -> previous -> next - for bY in [current_monthyear, prev_monthyear, next_monthyear]: - if check_file_exists(url := url_pattern.format(bY=bY)): + for bYYYY in [current_monthyear, prev_monthyear, next_monthyear]: + if check_file_exists(url := url_pattern.format(bYYYY=bYYYY)): break else: # If None of the three URLs are working From fec641ce0a9d1cdc66f6a035b376cec44abed8c6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 22 Dec 2023 13:41:23 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/retrieve.smk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index d08f94db..2485a842 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -254,7 +254,9 @@ if config["enable"]["retrieve"]: return response.status_code == 200 # Basic pattern where WDPA files can be found - url_pattern = "https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{bYYYY}_Public.zip" + url_pattern = ( + "https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{bYYYY}_Public.zip" + ) # 3-letter month + 4 digit year for current/previous/next month to test current_monthyear = datetime.now().strftime("%b%Y")