update and reduce databundle size
This commit is contained in:
Fabian Neumann
parent
a4d58b70f5
commit
991eef6d97
2
.github/workflows/ci.yaml
vendored
2
.github/workflows/ci.yaml
vendored
@ -19,7 +19,7 @@ on:
|
||||
- cron: "0 5 * * TUE"
|
||||
|
||||
env:
|
||||
DATA_CACHE_NUMBER: 1
|
||||
DATA_CACHE_NUMBER: 2
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
14
.gitignore
vendored
14
.gitignore
vendored
@ -37,18 +37,16 @@ dconf
|
||||
/data/links_p_nom.csv
|
||||
/data/*totals.csv
|
||||
/data/biomass*
|
||||
/data/bundle-sector/emobility/
|
||||
/data/bundle-sector/eea*
|
||||
/data/bundle-sector/jrc*
|
||||
/data/bundle/emobility/
|
||||
/data/bundle/eea*
|
||||
/data/bundle/jrc*
|
||||
/data/heating/
|
||||
/data/bundle-sector/eurostat*
|
||||
/data/bundle/eurostat*
|
||||
/data/odyssee/
|
||||
/data/transport_data.csv
|
||||
/data/bundle-sector/switzerland*
|
||||
/data/.nfs*
|
||||
/data/bundle-sector/Industrial_Database.csv
|
||||
/data/retro/tabula-calculator-calcsetbuilding.csv
|
||||
/data/bundle-sector/nuts*
|
||||
/data/retro/*
|
||||
/data/bundle/nuts*
|
||||
data/gas_network/scigrid-gas/
|
||||
data/costs_*.csv
|
||||
|
||||
|
||||
@ -69,13 +69,10 @@ enable:
|
||||
retrieve: auto
|
||||
prepare_links_p_nom: false
|
||||
retrieve_databundle: true
|
||||
retrieve_sector_databundle: true
|
||||
retrieve_cost_data: true
|
||||
build_cutout: false
|
||||
retrieve_irena: false
|
||||
retrieve_cutout: true
|
||||
build_natura_raster: false
|
||||
retrieve_natura_raster: true
|
||||
custom_busmap: false
|
||||
drop_leap_day: true
|
||||
|
||||
|
||||
@ -38,6 +38,5 @@ lines:
|
||||
enable:
|
||||
retrieve: true
|
||||
retrieve_databundle: true
|
||||
retrieve_sector_databundle: false
|
||||
retrieve_cost_data: true
|
||||
retrieve_cutout: true
|
||||
|
||||
27
data/ch_cantons.csv
Normal file
27
data/ch_cantons.csv
Normal file
@ -0,0 +1,27 @@
|
||||
Canton,HASC,NUTS
|
||||
Aargau,CH.AG,CH033
|
||||
Appenzell Inner Rhodes,CH.AI,CH054
|
||||
Appenzell Outer Rhodes,CH.AR,CH053
|
||||
Basel-Landschaft,CH.BL,CH032
|
||||
Basel-Stadt,CH.BS,CH031
|
||||
Bern,CH.BE,CH021
|
||||
Fribourg,CH.FR,CH022
|
||||
Geneva,CH.GE,CH013
|
||||
Glarus,CH.GL,CH051
|
||||
Graubünden,CH.GR,CH056
|
||||
Jura,CH.JU,CH025
|
||||
Lucerne,CH.LU,CH061
|
||||
Neuchâtel,CH.NE,CH024
|
||||
Nidwalden,CH.NW,CH065
|
||||
Obwalden,CH.OW,CH064
|
||||
Sankt Gallen,CH.SG,CH055
|
||||
Schaffhausen,CH.SH,CH052
|
||||
Schwyz,CH.SZ,CH063
|
||||
Solothurn,CH.SO,CH023
|
||||
Thurgau,CH.TG,CH057
|
||||
Ticino,CH.TI,CH07
|
||||
Uri,CH.UR,CH062
|
||||
Valais,CH.VS,CH012
|
||||
Vaud,CH.VD,CH011
|
||||
Zug,CH.ZG,CH066
|
||||
Zurich,CH.ZH,CH04
|
||||
|
31
data/hydro_capacities.csv
Normal file
31
data/hydro_capacities.csv
Normal file
@ -0,0 +1,31 @@
|
||||
Country,p_nom_discharge[GW],p_nom_store[GW],E_store[TWh],InflowHourlyAvg[GWh]
|
||||
AT,13.08,3.8,3.2,4.02
|
||||
BE,1.42,1.31,0,0.04
|
||||
BA,2.05,0.62,2.5,0.71
|
||||
BG,3.13,0.86,4,0.53
|
||||
HR,2,0.61,2.8,0.57
|
||||
CZ,2.21,0.68,1.5,0.24
|
||||
DK,0.01,0,0,0
|
||||
EE,0.01,0,0,0
|
||||
FI,3.2,0,5.5,1.59
|
||||
FR,25.37,6.99,9.8,7.82
|
||||
DE,11.26,6.8,0.3,1.93
|
||||
GB,4.43,2.74,0,0.46
|
||||
GR,3.24,0.7,2.3,0.26
|
||||
HU,0.06,0,0.1,0.02
|
||||
IE,0.53,0.29,0,0.08
|
||||
IT,21.88,7.55,7.9,5.19
|
||||
LV,1.58,0,1.8,0.3
|
||||
LT,0.88,0.76,0.2,0.05
|
||||
LU,1.13,1.29,0,0
|
||||
NL,0.04,0,0,0.01
|
||||
NO,30.51,1.35,84.4,14
|
||||
PL,2.35,1.4,1.6,0.23
|
||||
PT,5.72,1.03,2.6,1.37
|
||||
RO,6.55,0.09,12.1,1.95
|
||||
RS,2.14,0.61,0,1.18
|
||||
SK,2.52,0.92,2.2,0.49
|
||||
SI,1.25,0.18,2.2,0.36
|
||||
ES,18.55,2.75,18.4,2.61
|
||||
SE,16.41,0.1,33.8,7.8
|
||||
CH,13.3,4.03,8.4,4.29
|
||||
|
@ -2,12 +2,9 @@
|
||||
enable,str or bool,"{auto, true, false}","Switch to include (true) or exclude (false) the retrieve_* rules of snakemake into the workflow; 'auto' sets true|false based on availability of an internet connection to prevent issues with snakemake failing due to lack of internet connection."
|
||||
prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia <https://en.wikipedia.org/wiki/List_of_HVDC_projects>`_"
|
||||
retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder."
|
||||
retrieve_sector_databundle,bool,"{true, false}","Switch to retrieve sector databundle from zenodo via the rule :mod:`retrieve_sector_databundle` or whether to keep a custom databundle located in the corresponding folder."
|
||||
retrieve_cost_data,bool,"{true, false}","Switch to retrieve technology cost data from `technology-data repository <https://github.com/PyPSA/technology-data>`_."
|
||||
build_cutout,bool,"{true, false}","Switch to enable the building of cutouts via the rule :mod:`build_cutout`."
|
||||
retrieve_irena,bool,"{true, false}",Switch to enable the retrieval of ``existing_capacities`` from IRENASTAT with :mod:`retrieve_irena`.
|
||||
retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts from zenodo with :mod:`retrieve_cutout`."
|
||||
build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`."
|
||||
retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`."
|
||||
custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``."
|
||||
drop_leap_day,bool,"{true, false}","Switch to drop February 29 from all time-dependent data in leap years"
|
||||
|
||||
|
@ -5,10 +5,8 @@
|
||||
"naturalearth/*",,,,,http://www.naturalearthdata.com/about/terms-of-use/
|
||||
"NUTS_2013 _60M_SH/*","x","x",,"x",https://ec.europa.eu/eurostat/web/gisco/geodata/reference-data/administrative-units-statistical-units
|
||||
"cantons.csv","x",,"x",,https://en.wikipedia.org/wiki/Data_codes_for_Switzerland
|
||||
"eia_hydro_annual_generation.csv","x",,,,https://www.eia.gov/about/copyrights_reuse.php
|
||||
"GEBCO_2014_2D.nc","x",,,,https://www.gebco.net/data_and_products/gridded_bathymetry_data/documents/gebco_2014_historic.pdf
|
||||
"gebco/GEBCO_2014_2D.nc","x",,,,https://www.gebco.net/data_and_products/gridded_bathymetry_data/documents/gebco_2014_historic.pdf
|
||||
"hydro_capacities.csv","x",,,,
|
||||
"je-e-21.03.02.xls","x","x",,,https://www.bfs.admin.ch/bfs/en/home/fso/swiss-federal-statistical-office/terms-of-use.html
|
||||
"nama_10r_3 gdp.tsv.gz","x",,,"x",https://ec.europa.eu/eurostat/about/policies/copyright
|
||||
"nama_10r_3 popgdp.tsv.gz","x",,,"x",https://ec.europa.eu/eurostat/about/policies/copyright
|
||||
"time_series_60min _singleindex_filtered.csv","x",,,,https://data.open-power-system-data.org/time_series/2019-06-05/README.md
|
||||
|
||||
|
@ -28,16 +28,13 @@ Electricity Systems Databundle
|
||||
More details are included in `the description of the
|
||||
data bundles on zenodo <https://zenodo.org/record/3517935#.XbGeXvzRZGo>`__.
|
||||
|
||||
.. csv-table::
|
||||
:header-rows: 1
|
||||
:file: configtables/licenses.csv
|
||||
|
||||
* BY: Attribute Source
|
||||
* NC: Non-Commercial Use Only
|
||||
* SA: Share Alike
|
||||
|
||||
Sector-Coupled Systems Databundle
|
||||
=================================
|
||||
.. csv-table::
|
||||
:header-rows: 1
|
||||
:file: configtables/licenses.csv
|
||||
|
||||
.. csv-table::
|
||||
:header-rows: 1
|
||||
|
||||
@ -27,7 +27,6 @@ With these and the externally extracted ENTSO-E online map topology
|
||||
Then the process continues by calculating conventional power plant capacities, potentials, and per-unit availability time series for variable renewable energy carriers and hydro power plants with the following rules:
|
||||
|
||||
- :mod:`build_powerplants` for today's thermal power plant capacities using `powerplantmatching <https://github.com/FRESNA/powerplantmatching>`__ allocating these to the closest substation for each powerplant,
|
||||
- :mod:`build_natura_raster` for rasterising NATURA2000 natural protection areas,
|
||||
- :mod:`build_ship_raster` for building shipping traffic density,
|
||||
- :mod:`build_renewable_profiles` for the hourly capacity factors and installation potentials constrained by land-use in each substation's Voronoi cell for PV, onshore and offshore wind, and
|
||||
- :mod:`build_hydro_profile` for the hourly per-unit hydro power availability time series.
|
||||
@ -55,14 +54,6 @@ Rule ``prepare_links_p_nom``
|
||||
|
||||
.. automodule:: prepare_links_p_nom
|
||||
|
||||
.. _natura:
|
||||
|
||||
Rule ``build_natura_raster``
|
||||
===============================
|
||||
|
||||
.. automodule:: build_natura_raster
|
||||
|
||||
|
||||
.. _base:
|
||||
|
||||
Rule ``base_network``
|
||||
|
||||
@ -53,32 +53,6 @@ The :ref:`tutorial` uses a smaller cutout than required for the full model (30 M
|
||||
For details see :mod:`build_cutout` and read the `atlite documentation <https://atlite.readthedocs.io>`__.
|
||||
|
||||
|
||||
Rule ``retrieve_natura_raster``
|
||||
================================
|
||||
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.4706686.svg
|
||||
:target: https://doi.org/10.5281/zenodo.4706686
|
||||
|
||||
This rule, as a substitute for :mod:`build_natura_raster`, downloads an already rasterized version (`natura.tiff <https://zenodo.org/record/4706686/files/natura.tiff>`__) of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`__ natural protection areas to reduce computation times. The file is placed into the ``resources`` sub-directory.
|
||||
|
||||
**Relevant Settings**
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
enable:
|
||||
build_natura_raster:
|
||||
|
||||
.. seealso::
|
||||
Documentation of the configuration file ``config/config.yaml`` at
|
||||
:ref:`toplevel_cf`
|
||||
|
||||
**Outputs**
|
||||
|
||||
- ``resources/natura.tiff``: Rasterized version of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`__ natural protection areas to reduce computation times.
|
||||
|
||||
.. seealso::
|
||||
For details see :mod:`build_natura_raster`.
|
||||
|
||||
|
||||
Rule ``retrieve_electricity_demand``
|
||||
====================================
|
||||
@ -135,14 +109,3 @@ None.
|
||||
**Outputs**
|
||||
|
||||
- ``data/shipdensity_global.zip``
|
||||
|
||||
|
||||
Rule ``retrieve_sector_databundle``
|
||||
====================================
|
||||
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.5546516.svg
|
||||
:target: https://doi.org/10.5281/zenodo.5546516
|
||||
|
||||
In addition to the databundle required for electricity-only studies,
|
||||
another databundle is required for modelling sector-coupled systems.
|
||||
The size of this data bundle is around 640 MB.
|
||||
|
||||
@ -34,8 +34,7 @@ configuration, execute
|
||||
|
||||
snakemake -call results/test-elec/networks/elec_s_6_ec_lcopt_Co2L-24H.nc --configfile config/test/config.electricity.yaml
|
||||
|
||||
This configuration is set to download a reduced data set via the rules :mod:`retrieve_databundle`,
|
||||
:mod:`retrieve_natura_raster`, :mod:`retrieve_cutout`.
|
||||
This configuration is set to download a reduced cutout via the rule :mod:`retrieve_cutout`.
|
||||
For more information on the data dependencies of PyPSA-Eur, continue reading :ref:`data`.
|
||||
|
||||
How to configure runs?
|
||||
@ -143,7 +142,6 @@ This triggers a workflow of multiple preceding jobs that depend on each rule's i
|
||||
7[label = "base_network", color = "0.27 0.6 0.85", style="rounded"];
|
||||
8[label = "build_shapes", color = "0.26 0.6 0.85", style="rounded"];
|
||||
9[label = "retrieve_databundle", color = "0.59 0.6 0.85", style="rounded"];
|
||||
10[label = "retrieve_natura_raster", color = "0.47 0.6 0.85", style="rounded"];
|
||||
11[label = "build_bus_regions", color = "0.13 0.6 0.85", style="rounded"];
|
||||
12[label = "retrieve_cutout\ncutout: be-03-2013-era5", color = "0.36 0.6 0.85", style="rounded,dashed"];
|
||||
13[label = "build_renewable_profiles\ntechnology: onwind", color = "0.21 0.6 0.85", style="rounded"];
|
||||
@ -244,7 +242,6 @@ In the terminal, this will show up as a list of jobs to be run:
|
||||
retrieve_cost_data 1
|
||||
retrieve_databundle 1
|
||||
retrieve_electricity_demand 1
|
||||
retrieve_natura_raster 1
|
||||
simplify_network 1
|
||||
solve_network 1
|
||||
total 22
|
||||
|
||||
@ -119,8 +119,6 @@ which were already included in the electricity-only tutorial:
|
||||
retrieve_databundle 1
|
||||
retrieve_electricity_demand 1
|
||||
retrieve_gas_infrastructure_data 1
|
||||
retrieve_natura_raster 1
|
||||
retrieve_sector_databundle 1
|
||||
simplify_network 1
|
||||
solve_sector_network 1
|
||||
total 60
|
||||
@ -153,7 +151,6 @@ successfully.
|
||||
8[label = "base_network", color = "0.31 0.6 0.85", style="rounded"];
|
||||
9[label = "build_shapes", color = "0.36 0.6 0.85", style="rounded"];
|
||||
10[label = "retrieve_databundle", color = "0.29 0.6 0.85", style="rounded"];
|
||||
11[label = "retrieve_natura_raster", color = "0.01 0.6 0.85", style="rounded"];
|
||||
12[label = "build_bus_regions", color = "0.10 0.6 0.85", style="rounded"];
|
||||
13[label = "retrieve_cutout\ncutout: be-03-2013-era5", color = "0.37 0.6 0.85", style="rounded,dashed"];
|
||||
14[label = "build_renewable_profiles\ntechnology: onwind", color = "0.20 0.6 0.85", style="rounded"];
|
||||
@ -174,7 +171,6 @@ successfully.
|
||||
29[label = "prepare_network\nll: v1.5\nopts: ", color = "0.49 0.6 0.85", style="rounded"];
|
||||
30[label = "add_extra_components", color = "0.14 0.6 0.85", style="rounded"];
|
||||
31[label = "build_energy_totals", color = "0.39 0.6 0.85", style="rounded"];
|
||||
32[label = "retrieve_sector_databundle", color = "0.58 0.6 0.85", style="rounded"];
|
||||
33[label = "build_population_weighted_energy_totals", color = "0.56 0.6 0.85", style="rounded"];
|
||||
34[label = "build_clustered_population_layouts", color = "0.49 0.6 0.85", style="rounded"];
|
||||
35[label = "build_population_layouts", color = "0.06 0.6 0.85", style="rounded"];
|
||||
@ -440,7 +436,6 @@ workflow:
|
||||
8[label = "base_network", color = "0.12 0.6 0.85", style="rounded"];
|
||||
9[label = "build_shapes", color = "0.23 0.6 0.85", style="rounded"];
|
||||
10[label = "retrieve_databundle", color = "0.61 0.6 0.85", style="rounded"];
|
||||
11[label = "retrieve_natura_raster", color = "0.50 0.6 0.85", style="rounded"];
|
||||
12[label = "build_bus_regions", color = "0.51 0.6 0.85", style="rounded"];
|
||||
13[label = "retrieve_cutout\ncutout: be-03-2013-era5", color = "0.37 0.6 0.85", style="rounded,dashed"];
|
||||
14[label = "build_renewable_profiles\ntechnology: onwind", color = "0.63 0.6 0.85", style="rounded"];
|
||||
@ -462,7 +457,6 @@ workflow:
|
||||
30[label = "prepare_network\nll: v1.5\nopts: ", color = "0.56 0.6 0.85", style="rounded"];
|
||||
31[label = "add_extra_components", color = "0.11 0.6 0.85", style="rounded"];
|
||||
32[label = "build_energy_totals", color = "0.18 0.6 0.85", style="rounded"];
|
||||
33[label = "retrieve_sector_databundle", color = "0.06 0.6 0.85", style="rounded"];
|
||||
34[label = "build_population_weighted_energy_totals", color = "0.03 0.6 0.85", style="rounded"];
|
||||
35[label = "build_clustered_population_layouts", color = "0.25 0.6 0.85", style="rounded"];
|
||||
36[label = "build_population_layouts", color = "0.57 0.6 0.85", style="rounded"];
|
||||
|
||||
@ -109,7 +109,7 @@ rule build_shapes:
|
||||
nuts3=ancient("data/bundle/NUTS_2013_60M_SH/data/NUTS_RG_60M_2013.shp"),
|
||||
nuts3pop=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"),
|
||||
nuts3gdp=ancient("data/bundle/nama_10r_3gdp.tsv.gz"),
|
||||
ch_cantons=ancient("data/bundle/ch_cantons.csv"),
|
||||
ch_cantons=ancient("data/ch_cantons.csv"),
|
||||
ch_popgdp=ancient("data/bundle/je-e-21.03.02.xls"),
|
||||
output:
|
||||
country_shapes=resources("country_shapes.geojson"),
|
||||
@ -172,27 +172,6 @@ if config["enable"].get("build_cutout", False):
|
||||
"../scripts/build_cutout.py"
|
||||
|
||||
|
||||
if config["enable"].get("build_natura_raster", False):
|
||||
|
||||
rule build_natura_raster:
|
||||
input:
|
||||
natura=ancient("data/bundle/natura/Natura2000_end2015.shp"),
|
||||
cutout=lambda w: "cutouts/"
|
||||
+ CDIR
|
||||
+ config_provider("atlite", "default_cutout")(w)
|
||||
+ ".nc",
|
||||
output:
|
||||
resources("natura.tiff"),
|
||||
resources:
|
||||
mem_mb=5000,
|
||||
log:
|
||||
logs("build_natura_raster.log"),
|
||||
conda:
|
||||
"../envs/environment.yaml"
|
||||
script:
|
||||
"../scripts/build_natura_raster.py"
|
||||
|
||||
|
||||
rule build_ship_raster:
|
||||
input:
|
||||
ship_density="data/shipdensity_global.zip",
|
||||
@ -220,7 +199,7 @@ rule determine_availability_matrix_MD_UA:
|
||||
wdpa="data/WDPA.gpkg",
|
||||
wdpa_marine="data/WDPA_WDOECM_marine.gpkg",
|
||||
gebco=lambda w: (
|
||||
"data/bundle/GEBCO_2014_2D.nc"
|
||||
"data/bundle/gebco/GEBCO_2014_2D.nc"
|
||||
if config_provider("renewable", w.technology)(w).get("max_depth")
|
||||
else []
|
||||
),
|
||||
@ -276,7 +255,7 @@ rule build_renewable_profiles:
|
||||
base_network=resources("networks/base.nc"),
|
||||
corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"),
|
||||
natura=lambda w: (
|
||||
resources("natura.tiff")
|
||||
"data/bundle/natura/natura.tiff"
|
||||
if config_provider("renewable", w.technology, "natura")(w)
|
||||
else []
|
||||
),
|
||||
@ -287,7 +266,7 @@ rule build_renewable_profiles:
|
||||
),
|
||||
gebco=ancient(
|
||||
lambda w: (
|
||||
"data/bundle/GEBCO_2014_2D.nc"
|
||||
"data/bundle/gebco/GEBCO_2014_2D.nc"
|
||||
if config_provider("renewable", w.technology)(w).get("max_depth")
|
||||
else []
|
||||
)
|
||||
@ -437,7 +416,7 @@ rule add_electricity:
|
||||
),
|
||||
regions=resources("regions_onshore.geojson"),
|
||||
powerplants=resources("powerplants.csv"),
|
||||
hydro_capacities=ancient("data/bundle/hydro_capacities.csv"),
|
||||
hydro_capacities=ancient("data/hydro_capacities.csv"),
|
||||
geth_hydro_capacities="data/geth2015_hydro_capacities.csv",
|
||||
unit_commitment="data/unit_commitment.csv",
|
||||
fuel_price=lambda w: (
|
||||
|
||||
@ -287,10 +287,10 @@ rule build_energy_totals:
|
||||
energy=config_provider("energy"),
|
||||
input:
|
||||
nuts3_shapes=resources("nuts3_shapes.geojson"),
|
||||
co2="data/bundle-sector/eea/UNFCCC_v23.csv",
|
||||
co2="data/bundle/eea/UNFCCC_v23.csv",
|
||||
swiss="data/switzerland-new_format-all_years.csv",
|
||||
swiss_transport="data/gr-e-11.03.02.01.01-cc.csv",
|
||||
idees="data/bundle-sector/jrc-idees-2015",
|
||||
idees="data/bundle/jrc-idees-2015",
|
||||
district_heat_share="data/district_heat_share.csv",
|
||||
eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
|
||||
output:
|
||||
@ -338,10 +338,10 @@ rule build_biomass_potentials:
|
||||
"https://zenodo.org/records/10356004/files/ENSPRESO_BIOMASS.xlsx",
|
||||
keep_local=True,
|
||||
),
|
||||
nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21
|
||||
nuts2="data/bundle/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21
|
||||
regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
|
||||
nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"),
|
||||
swiss_cantons=ancient("data/bundle/ch_cantons.csv"),
|
||||
swiss_cantons=ancient("data/ch_cantons.csv"),
|
||||
swiss_population=ancient("data/bundle/je-e-21.03.02.xls"),
|
||||
country_shapes=resources("country_shapes.geojson"),
|
||||
output:
|
||||
@ -416,7 +416,7 @@ rule build_sequestration_potentials:
|
||||
|
||||
rule build_salt_cavern_potentials:
|
||||
input:
|
||||
salt_caverns="data/bundle-sector/h2_salt_caverns_GWh_per_sqkm.geojson",
|
||||
salt_caverns="data/bundle/h2_salt_caverns_GWh_per_sqkm.geojson",
|
||||
regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
|
||||
regions_offshore=resources("regions_offshore_elec_s{simpl}_{clusters}.geojson"),
|
||||
output:
|
||||
@ -436,7 +436,7 @@ rule build_salt_cavern_potentials:
|
||||
|
||||
rule build_ammonia_production:
|
||||
input:
|
||||
usgs="data/bundle-sector/myb1-2017-nitro.xls",
|
||||
usgs="data/bundle/myb1-2017-nitro.xls",
|
||||
output:
|
||||
ammonia_production=resources("ammonia_production.csv"),
|
||||
threads: 1
|
||||
@ -458,7 +458,7 @@ rule build_industry_sector_ratios:
|
||||
ammonia=config_provider("sector", "ammonia", default=False),
|
||||
input:
|
||||
ammonia_production=resources("ammonia_production.csv"),
|
||||
idees="data/bundle-sector/jrc-idees-2015",
|
||||
idees="data/bundle/jrc-idees-2015",
|
||||
output:
|
||||
industry_sector_ratios=resources("industry_sector_ratios.csv"),
|
||||
threads: 1
|
||||
@ -508,7 +508,7 @@ rule build_industrial_production_per_country:
|
||||
countries=config_provider("countries"),
|
||||
input:
|
||||
ammonia_production=resources("ammonia_production.csv"),
|
||||
jrc="data/bundle-sector/jrc-idees-2015",
|
||||
jrc="data/bundle/jrc-idees-2015",
|
||||
eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
|
||||
output:
|
||||
industrial_production_per_country=resources(
|
||||
@ -564,7 +564,10 @@ rule build_industrial_distribution_key:
|
||||
input:
|
||||
regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
|
||||
clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
|
||||
hotmaps_industrial_database="data/bundle-sector/Industrial_Database.csv",
|
||||
hotmaps_industrial_database=storage(
|
||||
"https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database/-/blob/master/data/Industrial_Database.csv",
|
||||
keep_local=True,
|
||||
),
|
||||
output:
|
||||
industrial_distribution_key=resources(
|
||||
"industrial_distribution_key_elec_s{simpl}_{clusters}.csv"
|
||||
@ -652,7 +655,7 @@ rule build_industrial_energy_demand_per_country_today:
|
||||
countries=config_provider("countries"),
|
||||
industry=config_provider("industry"),
|
||||
input:
|
||||
jrc="data/bundle-sector/jrc-idees-2015",
|
||||
jrc="data/bundle/jrc-idees-2015",
|
||||
industrial_production_per_country=resources(
|
||||
"industrial_production_per_country.csv"
|
||||
),
|
||||
@ -704,7 +707,7 @@ rule build_retro_cost:
|
||||
countries=config_provider("countries"),
|
||||
input:
|
||||
building_stock="data/retro/data_building_stock.csv",
|
||||
data_tabula="data/bundle-sector/retro/tabula-calculator-calcsetbuilding.csv",
|
||||
data_tabula="data/bundle/retro/tabula-calculator-calcsetbuilding.csv",
|
||||
air_temperature=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"),
|
||||
u_values_PL="data/retro/u_values_poland.csv",
|
||||
tax_w="data/retro/electricity_taxes_eu.csv",
|
||||
@ -780,8 +783,8 @@ rule build_transport_demand:
|
||||
"pop_weighted_energy_totals_s{simpl}_{clusters}.csv"
|
||||
),
|
||||
transport_data=resources("transport_data.csv"),
|
||||
traffic_data_KFZ="data/bundle-sector/emobility/KFZ__count",
|
||||
traffic_data_Pkw="data/bundle-sector/emobility/Pkw__count",
|
||||
traffic_data_KFZ="data/bundle/emobility/KFZ__count",
|
||||
traffic_data_Pkw="data/bundle/emobility/Pkw__count",
|
||||
temp_air_total=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"),
|
||||
output:
|
||||
transport_demand=resources("transport_demand_s{simpl}_{clusters}.csv"),
|
||||
@ -925,7 +928,7 @@ rule prepare_sector_network:
|
||||
avail_profile=resources("avail_profile_s{simpl}_{clusters}.csv"),
|
||||
dsm_profile=resources("dsm_profile_s{simpl}_{clusters}.csv"),
|
||||
co2_totals_name=resources("co2_totals.csv"),
|
||||
co2="data/bundle-sector/eea/UNFCCC_v23.csv",
|
||||
co2="data/bundle/eea/UNFCCC_v23.csv",
|
||||
biomass_potentials=lambda w: (
|
||||
resources(
|
||||
"biomass_potentials_s{simpl}_{clusters}_"
|
||||
|
||||
@ -230,7 +230,7 @@ rule plot_summary:
|
||||
energy=RESULTS + "csvs/energy.csv",
|
||||
balances=RESULTS + "csvs/supply_energy.csv",
|
||||
eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
|
||||
co2="data/bundle-sector/eea/UNFCCC_v23.csv",
|
||||
co2="data/bundle/eea/UNFCCC_v23.csv",
|
||||
output:
|
||||
costs=RESULTS + "graphs/costs.pdf",
|
||||
energy=RESULTS + "graphs/energy.pdf",
|
||||
|
||||
@ -14,23 +14,27 @@ if config["enable"]["retrieve"] is False:
|
||||
|
||||
if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", True):
|
||||
datafiles = [
|
||||
"ch_cantons.csv",
|
||||
"je-e-21.03.02.xls",
|
||||
"eez/World_EEZ_v8_2014.shp",
|
||||
"hydro_capacities.csv",
|
||||
"naturalearth/ne_10m_admin_0_countries.shp",
|
||||
"NUTS_2013_60M_SH/data/NUTS_RG_60M_2013.shp",
|
||||
"nama_10r_3popgdp.tsv.gz",
|
||||
"nama_10r_3gdp.tsv.gz",
|
||||
"corine/g250_clc06_V18_5.tif",
|
||||
"eea/UNFCCC_v23.csv",
|
||||
"nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
|
||||
"myb1-2017-nitro.xls",
|
||||
"emobility/KFZ__count",
|
||||
"emobility/Pkw__count",
|
||||
"h2_salt_caverns_GWh_per_sqkm.geojson",
|
||||
"natura/natura.tiff",
|
||||
"gebco/GEBCO_2014_2D.nc",
|
||||
]
|
||||
|
||||
if not config.get("tutorial", False):
|
||||
datafiles.extend(["natura/Natura2000_end2015.shp", "GEBCO_2014_2D.nc"])
|
||||
|
||||
rule retrieve_databundle:
|
||||
output:
|
||||
protected(expand("data/bundle/{file}", file=datafiles)),
|
||||
protected(directory("data/bundle/jrc-idees-2015")),
|
||||
log:
|
||||
"logs/retrieve_databundle.log",
|
||||
resources:
|
||||
@ -41,6 +45,15 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle",
|
||||
script:
|
||||
"../scripts/retrieve_databundle.py"
|
||||
|
||||
rule retrieve_eurostat_data:
|
||||
output:
|
||||
directory("data/eurostat/eurostat-energy_balances-april_2023_edition"),
|
||||
log:
|
||||
"logs/retrieve_eurostat_data.log",
|
||||
retries: 2
|
||||
script:
|
||||
"../scripts/retrieve_eurostat_data.py"
|
||||
|
||||
|
||||
if config["enable"].get("retrieve_irena"):
|
||||
|
||||
@ -97,64 +110,6 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", T
|
||||
"../scripts/retrieve_cost_data.py"
|
||||
|
||||
|
||||
if config["enable"]["retrieve"] and config["enable"].get(
|
||||
"retrieve_natura_raster", True
|
||||
):
|
||||
|
||||
rule retrieve_natura_raster:
|
||||
input:
|
||||
storage(
|
||||
"https://zenodo.org/record/4706686/files/natura.tiff",
|
||||
keep_local=True,
|
||||
),
|
||||
output:
|
||||
resources("natura.tiff"),
|
||||
log:
|
||||
logs("retrieve_natura_raster.log"),
|
||||
resources:
|
||||
mem_mb=5000,
|
||||
retries: 2
|
||||
run:
|
||||
copyfile(input[0], output[0])
|
||||
validate_checksum(output[0], input[0])
|
||||
|
||||
|
||||
if config["enable"]["retrieve"] and config["enable"].get(
|
||||
"retrieve_sector_databundle", True
|
||||
):
|
||||
datafiles = [
|
||||
"eea/UNFCCC_v23.csv",
|
||||
"switzerland-sfoe/switzerland-new_format.csv",
|
||||
"nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
|
||||
"myb1-2017-nitro.xls",
|
||||
"Industrial_Database.csv",
|
||||
"emobility/KFZ__count",
|
||||
"emobility/Pkw__count",
|
||||
"h2_salt_caverns_GWh_per_sqkm.geojson",
|
||||
]
|
||||
|
||||
rule retrieve_sector_databundle:
|
||||
output:
|
||||
protected(expand("data/bundle-sector/{files}", files=datafiles)),
|
||||
protected(directory("data/bundle-sector/jrc-idees-2015")),
|
||||
log:
|
||||
"logs/retrieve_sector_databundle.log",
|
||||
retries: 2
|
||||
conda:
|
||||
"../envs/retrieve.yaml"
|
||||
script:
|
||||
"../scripts/retrieve_sector_databundle.py"
|
||||
|
||||
rule retrieve_eurostat_data:
|
||||
output:
|
||||
directory("data/eurostat/eurostat-energy_balances-april_2023_edition"),
|
||||
log:
|
||||
"logs/retrieve_eurostat_data.log",
|
||||
retries: 2
|
||||
script:
|
||||
"../scripts/retrieve_eurostat_data.py"
|
||||
|
||||
|
||||
if config["enable"]["retrieve"]:
|
||||
datafiles = [
|
||||
"IGGIELGN_LNGs.geojson",
|
||||
|
||||
@ -46,7 +46,7 @@ Inputs
|
||||
------
|
||||
|
||||
- ``resources/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
|
||||
- ``data/bundle/hydro_capacities.csv``: Hydropower plant store/discharge power capacities, energy storage capacity, and average hourly inflow by country.
|
||||
- ``data/hydro_capacities.csv``: Hydropower plant store/discharge power capacities, energy storage capacity, and average hourly inflow by country.
|
||||
|
||||
.. image:: img/hydrocapacities.png
|
||||
:scale: 34 %
|
||||
|
||||
@ -1,118 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""
|
||||
Rasters the vector data of the `Natura 2000.
|
||||
|
||||
<https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas onto all
|
||||
cutout regions.
|
||||
|
||||
Relevant Settings
|
||||
-----------------
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
renewable:
|
||||
{technology}:
|
||||
cutout:
|
||||
|
||||
.. seealso::
|
||||
Documentation of the configuration file ``config/config.yaml`` at
|
||||
:ref:`renewable_cf`
|
||||
|
||||
Inputs
|
||||
------
|
||||
|
||||
- ``data/bundle/natura/Natura2000_end2015.shp``: `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas.
|
||||
|
||||
.. image:: img/natura.png
|
||||
:scale: 33 %
|
||||
|
||||
Outputs
|
||||
-------
|
||||
|
||||
- ``resources/natura.tiff``: Rasterized version of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas to reduce computation times.
|
||||
|
||||
.. image:: img/natura.png
|
||||
:scale: 33 %
|
||||
|
||||
Description
|
||||
-----------
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import atlite
|
||||
import geopandas as gpd
|
||||
import rasterio as rio
|
||||
from _helpers import configure_logging, set_scenario_config
|
||||
from rasterio.features import geometry_mask
|
||||
from rasterio.warp import transform_bounds
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def determine_cutout_xXyY(cutout_name):
|
||||
"""
|
||||
Determine the full extent of a cutout.
|
||||
|
||||
Since the coordinates of the cutout data are given as the
|
||||
center of the grid cells, the extent of the cutout is
|
||||
calculated by adding/subtracting half of the grid cell size.
|
||||
|
||||
|
||||
Parameters
|
||||
----------
|
||||
cutout_name : str
|
||||
Path to the cutout.
|
||||
|
||||
Returns
|
||||
-------
|
||||
A list of extent coordinates in the order [x, X, y, Y].
|
||||
"""
|
||||
cutout = atlite.Cutout(cutout_name)
|
||||
assert cutout.crs.to_epsg() == 4326
|
||||
x, X, y, Y = cutout.extent
|
||||
dx, dy = cutout.dx, cutout.dy
|
||||
return [x - dx / 2.0, X + dx / 2.0, y - dy / 2.0, Y + dy / 2.0]
|
||||
|
||||
|
||||
def get_transform_and_shape(bounds, res):
|
||||
left, bottom = [(b // res) * res for b in bounds[:2]]
|
||||
right, top = [(b // res + 1) * res for b in bounds[2:]]
|
||||
shape = int((top - bottom) // res), int((right - left) / res)
|
||||
transform = rio.Affine(res, 0, left, 0, -res, top)
|
||||
return transform, shape
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "snakemake" not in globals():
|
||||
from _helpers import mock_snakemake
|
||||
|
||||
snakemake = mock_snakemake("build_natura_raster")
|
||||
configure_logging(snakemake)
|
||||
set_scenario_config(snakemake)
|
||||
|
||||
x, X, y, Y = determine_cutout_xXyY(snakemake.input.cutout)
|
||||
bounds = transform_bounds(4326, 3035, x, y, X, Y)
|
||||
transform, out_shape = get_transform_and_shape(bounds, res=100)
|
||||
|
||||
# adjusted boundaries
|
||||
shapes = gpd.read_file(snakemake.input.natura).to_crs(3035)
|
||||
raster = ~geometry_mask(shapes.geometry, out_shape, transform)
|
||||
raster = raster.astype(rio.uint8)
|
||||
|
||||
with rio.open(
|
||||
snakemake.output[0],
|
||||
"w",
|
||||
driver="GTiff",
|
||||
dtype=rio.uint8,
|
||||
count=1,
|
||||
transform=transform,
|
||||
crs=3035,
|
||||
compress="lzw",
|
||||
width=raster.shape[1],
|
||||
height=raster.shape[0],
|
||||
) as dst:
|
||||
dst.write(raster, indexes=1)
|
||||
@ -52,7 +52,7 @@ Inputs
|
||||
CORINE land cover, see `Annex 1 of the technical documentation
|
||||
<https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_.
|
||||
|
||||
- ``data/bundle/GEBCO_2014_2D.nc``: A `bathymetric
|
||||
- ``data/bundle/gebco/GEBCO_2014_2D.nc``: A `bathymetric
|
||||
<https://en.wikipedia.org/wiki/Bathymetry>`_ data set with a global terrain
|
||||
model for ocean and land at 15 arc-second intervals by the `General
|
||||
Bathymetric Chart of the Oceans (GEBCO)
|
||||
|
||||
@ -38,7 +38,7 @@ Inputs
|
||||
|
||||
- ``data/bundle/nama_10r_3popgdp.tsv.gz``: Average annual population by NUTS3 region (`eurostat <http://appsso.eurostat.ec.europa.eu/nui/show.do?dataset=nama_10r_3popgdp&lang=en>`__)
|
||||
- ``data/bundle/nama_10r_3gdp.tsv.gz``: Gross domestic product (GDP) by NUTS 3 regions (`eurostat <http://appsso.eurostat.ec.europa.eu/nui/show.do?dataset=nama_10r_3gdp&lang=en>`__)
|
||||
- ``data/bundle/ch_cantons.csv``: Mapping between Swiss Cantons and NUTS3 regions
|
||||
- ``data/ch_cantons.csv``: Mapping between Swiss Cantons and NUTS3 regions
|
||||
- ``data/bundle/je-e-21.03.02.xls``: Population and GDP data per Canton (`BFS - Swiss Federal Statistical Office <https://www.bfs.admin.ch/bfs/en/home/news/whats-new.assetdetail.7786557.html>`_ )
|
||||
|
||||
Outputs
|
||||
|
||||
@ -47,10 +47,35 @@ from pathlib import Path
|
||||
|
||||
import rioxarray
|
||||
from _helpers import configure_logging, set_scenario_config
|
||||
from build_natura_raster import determine_cutout_xXyY
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def determine_cutout_xXyY(cutout_name):
|
||||
"""
|
||||
Determine the full extent of a cutout.
|
||||
|
||||
Since the coordinates of the cutout data are given as the
|
||||
center of the grid cells, the extent of the cutout is
|
||||
calculated by adding/subtracting half of the grid cell size.
|
||||
|
||||
|
||||
Parameters
|
||||
----------
|
||||
cutout_name : str
|
||||
Path to the cutout.
|
||||
|
||||
Returns
|
||||
-------
|
||||
A list of extent coordinates in the order [x, X, y, Y].
|
||||
"""
|
||||
cutout = atlite.Cutout(cutout_name)
|
||||
assert cutout.crs.to_epsg() == 4326
|
||||
x, X, y, Y = cutout.extent
|
||||
dx, dy = cutout.dx, cutout.dy
|
||||
return [x - dx / 2.0, X + dx / 2.0, y - dy / 2.0, Y + dy / 2.0]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "snakemake" not in globals():
|
||||
from _helpers import mock_snakemake
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2019-2022 Fabian Hofmann (TUB, FIAS)
|
||||
# Copyright 2019-2024 Fabian Hofmann (TUB, FIAS), Fabian Neumann (TUB)
|
||||
# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
@ -7,24 +7,15 @@
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517935.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3517935
|
||||
|
||||
The data bundle (1.4 GB) contains common GIS datasets like NUTS3 shapes, EEZ shapes, CORINE Landcover, Natura 2000 and also electricity specific summary statistics like historic per country yearly totals of hydro generation, GDP and POP on NUTS3 levels and per-country load time-series.
|
||||
The data bundle contains common GIS datasets like NUTS3 shapes, EEZ shapes,
|
||||
CORINE Landcover, Natura 2000 and also electricity specific summary statistics
|
||||
like historic per country yearly totals of hydro generation, GDP and population
|
||||
data on NUTS3 levels and energy balances.
|
||||
|
||||
This rule downloads the data bundle from `zenodo <https://doi.org/10.5281/zenodo.3517935>`_ and extracts it in the ``data`` sub-directory, such that all files of the bundle are stored in the ``data/bundle`` subdirectory.
|
||||
|
||||
The :ref:`tutorial` uses a smaller `data bundle <https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz>`_ than required for the full model (188 MB)
|
||||
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517921.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3517921
|
||||
|
||||
**Relevant Settings**
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
tutorial:
|
||||
|
||||
.. seealso::
|
||||
Documentation of the configuration file ``config/config.yaml`` at
|
||||
:ref:`toplevel_cf`
|
||||
This rule downloads the data bundle from `zenodo
|
||||
<https://doi.org/10.5281/zenodo.3517935>`_ and extracts it in the ``data``
|
||||
sub-directory, such that all files of the bundle are stored in the
|
||||
``data/bundle`` subdirectory.
|
||||
|
||||
**Outputs**
|
||||
|
||||
@ -57,10 +48,7 @@ if __name__ == "__main__":
|
||||
configure_logging(snakemake)
|
||||
set_scenario_config(snakemake)
|
||||
|
||||
if snakemake.config["tutorial"]:
|
||||
url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz"
|
||||
else:
|
||||
url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz"
|
||||
url = "https://zenodo.org/records/10973944/files/bundle.tar.xz"
|
||||
|
||||
tarball_fn = Path(f"{rootpath}/bundle.tar.xz")
|
||||
to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent
|
||||
@ -74,6 +62,7 @@ if __name__ == "__main__":
|
||||
logger.info("Extracting databundle.")
|
||||
tarfile.open(tarball_fn).extractall(to_fn)
|
||||
|
||||
logger.info("Unlinking tarball.")
|
||||
tarball_fn.unlink()
|
||||
|
||||
logger.info(f"Databundle available in '{to_fn}'.")
|
||||
|
||||
@ -1,49 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""
|
||||
Retrieve and extract data bundle for sector-coupled studies.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
from _helpers import (
|
||||
configure_logging,
|
||||
progress_retrieve,
|
||||
set_scenario_config,
|
||||
validate_checksum,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "snakemake" not in globals():
|
||||
from _helpers import mock_snakemake
|
||||
|
||||
snakemake = mock_snakemake("retrieve_databundle")
|
||||
rootpath = ".."
|
||||
else:
|
||||
rootpath = "."
|
||||
configure_logging(snakemake)
|
||||
set_scenario_config(snakemake)
|
||||
|
||||
url = "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz"
|
||||
|
||||
tarball_fn = Path(f"{rootpath}/sector-bundle.tar.gz")
|
||||
to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent
|
||||
|
||||
logger.info(f"Downloading databundle from '{url}'.")
|
||||
disable_progress = snakemake.config["run"].get("disable_progressbar", False)
|
||||
progress_retrieve(url, tarball_fn, disable=disable_progress)
|
||||
|
||||
validate_checksum(tarball_fn, url)
|
||||
|
||||
logger.info("Extracting databundle.")
|
||||
tarfile.open(tarball_fn).extractall(to_fn)
|
||||
|
||||
tarball_fn.unlink()
|
||||
|
||||
logger.info(f"Databundle available in '{to_fn}'.")
|
||||
Loading…
Reference in New Issue
Block a user