diff --git a/.gitignore b/.gitignore index 0adf0ae6..459ef860 100644 --- a/.gitignore +++ b/.gitignore @@ -28,18 +28,18 @@ dconf /data/links_p_nom.csv /data/*totals.csv /data/biomass* -/data/emobility/ -/data/eea* -/data/jrc* +/data/bundle-sector/emobility/ +/data/bundle-sector/eea* +/data/bundle-sector/jrc* /data/heating/ -/data/eurostat* +/data/bundle-sector/eurostat* /data/odyssee/ /data/transport_data.csv -/data/switzerland* +/data/bundle-sector/switzerland* /data/.nfs* -/data/Industrial_Database.csv +/data/bundle-sector/Industrial_Database.csv /data/retro/tabula-calculator-calcsetbuilding.csv -/data/nuts* +/data/bundle-sector/nuts* data/gas_network/scigrid-gas/ data/costs_*.csv diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 243e78ca..3a787104 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -16,6 +16,10 @@ Upcoming Release * The minimum capacity for renewable generators when using the myopic option has been fixed. +* Files downloaded from zenodo are now write-protected to prevent accidental re-download. + +* Files extracted from sector-coupled data bundle have been moved from ``data/`` to ``data/sector-bundle``. + PyPSA-Eur 0.8.1 (27th July 2023) ================================ diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 383951bd..f9fdc3ac 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -350,7 +350,9 @@ rule add_electricity: hydro_capacities=ancient("data/bundle/hydro_capacities.csv"), geth_hydro_capacities="data/geth2015_hydro_capacities.csv", unit_commitment="data/unit_commitment.csv", - fuel_price=RESOURCES + "monthly_fuel_price.csv", + fuel_price=RESOURCES + "monthly_fuel_price.csv" + if config["conventional"]["dynamic_fuel_price"] + else [], load=RESOURCES + "load.csv", nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", output: @@ -478,7 +480,7 @@ rule prepare_network: input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", tech_costs=COSTS, - co2_price=RESOURCES + "co2_price.csv", + co2_price=lambda w: RESOURCES + "co2_price.csv" if "Ept" in w.opts else [], output: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", log: diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 356abdc5..10a5f821 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -242,9 +242,9 @@ rule build_energy_totals: energy=config["energy"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", - co2="data/eea/UNFCCC_v23.csv", - swiss="data/switzerland-sfoe/switzerland-new_format.csv", - idees="data/jrc-idees-2015", + co2="data/bundle-sector/eea/UNFCCC_v23.csv", + swiss="data/bundle-sector/switzerland-sfoe/switzerland-new_format.csv", + idees="data/bundle-sector/jrc-idees-2015", district_heat_share="data/district_heat_share.csv", eurostat=input_eurostat, output: @@ -272,7 +272,7 @@ rule build_biomass_potentials: "https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", keep_local=True, ), - nuts2="data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 + nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"), swiss_cantons=ancient("data/bundle/ch_cantons.csv"), @@ -366,7 +366,7 @@ if not config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_salt_cavern_potentials: input: - salt_caverns="data/h2_salt_caverns_GWh_per_sqkm.geojson", + salt_caverns="data/bundle-sector/h2_salt_caverns_GWh_per_sqkm.geojson", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}_{clusters}.geojson", output: @@ -388,7 +388,7 @@ rule build_ammonia_production: params: countries=config["countries"], input: - usgs="data/myb1-2017-nitro.xls", + usgs="data/bundle-sector/myb1-2017-nitro.xls", output: ammonia_production=RESOURCES + "ammonia_production.csv", threads: 1 @@ -410,7 +410,7 @@ rule build_industry_sector_ratios: ammonia=config["sector"].get("ammonia", False), input: ammonia_production=RESOURCES + "ammonia_production.csv", - idees="data/jrc-idees-2015", + idees="data/bundle-sector/jrc-idees-2015", output: industry_sector_ratios=RESOURCES + "industry_sector_ratios.csv", threads: 1 @@ -432,8 +432,8 @@ rule build_industrial_production_per_country: countries=config["countries"], input: ammonia_production=RESOURCES + "ammonia_production.csv", - jrc="data/jrc-idees-2015", - eurostat="data/eurostat-energy_balances-may_2018_edition", + jrc="data/bundle-sector/jrc-idees-2015", + eurostat="data/bundle-sector/eurostat-energy_balances-may_2018_edition", output: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -483,7 +483,7 @@ rule build_industrial_distribution_key: input: regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", - hotmaps_industrial_database="data/Industrial_Database.csv", + hotmaps_industrial_database="data/bundle-sector/Industrial_Database.csv", output: industrial_distribution_key=RESOURCES + "industrial_distribution_key_elec_s{simpl}_{clusters}.csv", @@ -558,7 +558,7 @@ rule build_industrial_energy_demand_per_country_today: countries=config["countries"], industry=config["industry"], input: - jrc="data/jrc-idees-2015", + jrc="data/bundle-sector/jrc-idees-2015", ammonia_production=RESOURCES + "ammonia_production.csv", industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -684,8 +684,8 @@ rule build_transport_demand: pop_weighted_energy_totals=RESOURCES + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv", transport_data=RESOURCES + "transport_data.csv", - traffic_data_KFZ="data/emobility/KFZ__count", - traffic_data_Pkw="data/emobility/Pkw__count", + traffic_data_KFZ="data/bundle-sector/emobility/KFZ__count", + traffic_data_Pkw="data/bundle-sector/emobility/Pkw__count", temp_air_total=RESOURCES + "temp_air_total_elec_s{simpl}_{clusters}.nc", output: transport_demand=RESOURCES + "transport_demand_s{simpl}_{clusters}.csv", @@ -734,7 +734,7 @@ rule prepare_sector_network: avail_profile=RESOURCES + "avail_profile_s{simpl}_{clusters}.csv", dsm_profile=RESOURCES + "dsm_profile_s{simpl}_{clusters}.csv", co2_totals_name=RESOURCES + "co2_totals.csv", - co2="data/eea/UNFCCC_v23.csv", + co2="data/bundle-sector/eea/UNFCCC_v23.csv", biomass_potentials=RESOURCES + "biomass_potentials_s{simpl}_{clusters}.csv", heat_profile="data/heat_load_profile_BDEW.csv", costs="data/costs_{}.csv".format(config["costs"]["year"]) diff --git a/rules/common.smk b/rules/common.smk index ec5be355..d3416050 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -42,7 +42,7 @@ def has_internet_access(url="www.zenodo.org") -> bool: def input_eurostat(w): # 2016 includes BA, 2017 does not report_year = config["energy"]["eurostat_report_year"] - return f"data/eurostat-energy_balances-june_{report_year}_edition" + return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition" def solved_previous_horizon(wildcards): diff --git a/rules/postprocess.smk b/rules/postprocess.smk index d0909c4e..cf0038a3 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -115,7 +115,7 @@ rule plot_summary: energy=RESULTS + "csvs/energy.csv", balances=RESULTS + "csvs/supply_energy.csv", eurostat=input_eurostat, - co2="data/eea/UNFCCC_v23.csv", + co2="data/bundle-sector/eea/UNFCCC_v23.csv", output: costs=RESULTS + "graphs/costs.pdf", energy=RESULTS + "graphs/energy.pdf", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 0b60ee2e..66ce76df 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -27,7 +27,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", rule retrieve_databundle: output: - expand("data/bundle/{file}", file=datafiles), + protected(expand("data/bundle/{file}", file=datafiles)), log: LOGS + "retrieve_databundle.log", resources: @@ -92,7 +92,7 @@ if config["enable"]["retrieve"] and config["enable"].get( static=True, ), output: - RESOURCES + "natura.tiff", + protected(RESOURCES + "natura.tiff"), log: LOGS + "retrieve_natura_raster.log", resources: @@ -106,22 +106,30 @@ if config["enable"]["retrieve"] and config["enable"].get( "retrieve_sector_databundle", True ): datafiles = [ - "data/eea/UNFCCC_v23.csv", - "data/switzerland-sfoe/switzerland-new_format.csv", - "data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", - "data/myb1-2017-nitro.xls", - "data/Industrial_Database.csv", - "data/emobility/KFZ__count", - "data/emobility/Pkw__count", - "data/h2_salt_caverns_GWh_per_sqkm.geojson", - directory("data/eurostat-energy_balances-june_2016_edition"), - directory("data/eurostat-energy_balances-may_2018_edition"), - directory("data/jrc-idees-2015"), + "eea/UNFCCC_v23.csv", + "switzerland-sfoe/switzerland-new_format.csv", + "nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", + "myb1-2017-nitro.xls", + "Industrial_Database.csv", + "emobility/KFZ__count", + "emobility/Pkw__count", + "h2_salt_caverns_GWh_per_sqkm.geojson", + ] + + datafolders = [ + protected( + directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition") + ), + protected( + directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition") + ), + protected(directory("data/bundle-sector/jrc-idees-2015")), ] rule retrieve_sector_databundle: output: - *datafiles, + protected(expand("data/bundle-sector/{files}", files=datafiles)), + *datafolders, log: LOGS + "retrieve_sector_databundle.log", retries: 2 @@ -143,7 +151,9 @@ if config["enable"]["retrieve"] and ( rule retrieve_gas_infrastructure_data: output: - expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles), + protected( + expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles) + ), log: LOGS + "retrieve_gas_infrastructure_data.log", retries: 2 @@ -187,7 +197,7 @@ if config["enable"]["retrieve"]: static=True, ), output: - "data/shipdensity_global.zip", + protected("data/shipdensity_global.zip"), log: LOGS + "retrieve_ship_raster.log", resources: diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 269705a7..08810470 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -447,8 +447,11 @@ def add_heating_capacities_installed_before_baseyear( ) # if rural heating demand for one of the nodes doesn't exist, # then columns were dropped before and heating demand share should be 0.0 - if all(f"{node} {service} rural heat" in p_set_sum.index for service in ["residential", "services"]) - else 0. + if all( + f"{node} {service} rural heat" in p_set_sum.index + for service in ["residential", "services"] + ) + else 0.0 for node in nodal_df.index ], index=nodal_df.index, diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 63299570..bc4a26bc 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -13,13 +13,14 @@ logger = logging.getLogger(__name__) import uuid from itertools import product +import country_converter as coco import geopandas as gpd import pandas as pd from packaging.version import Version, parse -import country_converter as coco cc = coco.CountryConverter() + def locate_missing_industrial_sites(df): """ Locate industrial sites without valid locations based on city and diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 072c7128..a501dcf7 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -354,7 +354,7 @@ def historical_emissions(countries): """ # https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16 # downloaded 201228 (modified by EEA last on 201221) - fn = "data/eea/UNFCCC_v23.csv" + fn = "data/bundle-sector/eea/UNFCCC_v23.csv" df = pd.read_csv(fn, encoding="latin-1") df.loc[df["Year"] == "1985-1987", "Year"] = 1986 df["Year"] = df["Year"].astype(int)