automatically retrieve technology-data, no git clone

.github/workflows/ci.yaml

@@ -52,10 +52,9 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Clone pypsa-eur and technology-data repositories
+      - name: Clone pypsa-eur subworkflow
         run: |
           git clone https://github.com/pypsa/pypsa-eur ../pypsa-eur
-          git clone https://github.com/pypsa/technology-data ../technology-data
           cp ../pypsa-eur/test/config.test1.yaml ../pypsa-eur/config.yaml
       
       - name: Setup secrets

Snakefile

@@ -1,6 +1,6 @@
 
 from os.path import exists
-from shutil import copyfile
+from shutil import copyfile, move
 
 from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider
 HTTP = HTTPRemoteProvider()
@@ -21,7 +21,6 @@ wildcard_constraints:
 
 SDIR = config['summary_dir'] + '/' + config['run']
 RDIR = config['results_dir'] + config['run']
-CDIR = config['costs_dir']
 
 
 subworkflow pypsaeur:
@@ -72,6 +71,15 @@ if config.get('retrieve_sector_databundle', True):
         script: 'scripts/retrieve_sector_databundle.py'
 
 
+if config.get("retrieve_cost_data", True):
+    rule retrieve_cost_data:
+        input: HTTP.remote("raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format(config['costs']['version']) + "costs_{year}.csv", keep_local=True)
+        output: "data/costs_{year}.csv"
+        log: "logs/" + RDIR + "retrieve_cost_data_{year}.log",
+        resources: mem_mb=1000,
+        run: move(input[0], output[0])
+
+
 rule build_population_layouts:
     input:
         nuts3_shapes=pypsaeur('resources/nuts3_shapes.geojson'),
@@ -483,7 +491,7 @@ rule prepare_sector_network:
         co2="data/eea/UNFCCC_v23.csv",
         biomass_potentials='resources/biomass_potentials_s{simpl}_{clusters}.csv',
         heat_profile="data/heat_load_profile_BDEW.csv",
-        costs=CDIR + "costs_{}.csv".format(config['costs']['year']) if config["foresight"] == "overnight" else CDIR + "costs_{planning_horizons}.csv",
+        costs="data/costs_{}.csv".format(config['costs']['year']) if config["foresight"] == "overnight" else "data/costs_{planning_horizons}.csv",
         profile_offwind_ac=pypsaeur("resources/profile_offwind-ac.nc"),
         profile_offwind_dc=pypsaeur("resources/profile_offwind-dc.nc"),
         h2_cavern="resources/salt_cavern_potentials_s{simpl}_{clusters}.csv",
@@ -557,7 +565,7 @@ rule make_summary:
             RDIR + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
             **config['scenario']
         ),
-        costs=CDIR + "costs_{}.csv".format(config['costs']['year']) if config["foresight"] == "overnight" else CDIR + "costs_{}.csv".format(config['scenario']['planning_horizons'][0]),
+        costs="data/costs_{}.csv".format(config['costs']['year']) if config["foresight"] == "overnight" else "data/costs_{}.csv".format(config['scenario']['planning_horizons'][0]),
         plots=expand(
             RDIR + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
             **config['scenario']
@@ -607,7 +615,7 @@ if config["foresight"] == "overnight":
         input:
             overrides="data/override_component_attrs",
             network=RDIR + "/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
-            costs=CDIR + "costs_{}.csv".format(config['costs']['year']),
+            costs="data/costs_{}.csv".format(config['costs']['year']),
             config=SDIR + '/configs/config.yaml',
             #env=SDIR + '/configs/environment.yaml',
         output: RDIR + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc"
@@ -632,7 +640,7 @@ if config["foresight"] == "myopic":
             busmap_s=pypsaeur("resources/busmap_elec_s{simpl}.csv"),
             busmap=pypsaeur("resources/busmap_elec_s{simpl}_{clusters}.csv"),
             clustered_pop_layout="resources/pop_layout_elec_s{simpl}_{clusters}.csv",
-            costs=CDIR + "costs_{}.csv".format(config['scenario']['planning_horizons'][0]),
+            costs="data/costs_{}.csv".format(config['scenario']['planning_horizons'][0]),
             cop_soil_total="resources/cop_soil_total_elec_s{simpl}_{clusters}.nc",
             cop_air_total="resources/cop_air_total_elec_s{simpl}_{clusters}.nc",
             existing_heating='data/existing_infrastructure/existing_heating_raw.csv',
@@ -661,7 +669,7 @@ if config["foresight"] == "myopic":
             overrides="data/override_component_attrs",
             network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc',
             network_p=solved_previous_horizon, #solved network at previous time step
-            costs=CDIR + "costs_{planning_horizons}.csv",
+            costs="data/costs_{planning_horizons}.csv",
             cop_soil_total="resources/cop_soil_total_elec_s{simpl}_{clusters}.nc",
             cop_air_total="resources/cop_air_total_elec_s{simpl}_{clusters}.nc"
         output: RDIR + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc"
@@ -678,7 +686,7 @@ if config["foresight"] == "myopic":
         input:
             overrides="data/override_component_attrs",
             network=RDIR + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
-            costs=CDIR + "costs_{planning_horizons}.csv",
+            costs="data/costs_{planning_horizons}.csv",
             config=SDIR + '/configs/config.yaml'
         output: RDIR + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc"
         shadow: "shallow"

config.default.yaml

@@ -3,10 +3,10 @@ version: 0.6.0
 logging_level: INFO
 
 retrieve_sector_databundle: true
+retrieve_cost_data: true
 
 results_dir: results/
 summary_dir: results
-costs_dir: ../technology-data/outputs/
 run: your-run-name  # use this to keep track of runs with different settings
 foresight: overnight # options are overnight, myopic, perfect (perfect is not yet implemented)
 # if you use myopic or perfect foresight, set the investment years in "planning_horizons" below
@@ -338,6 +338,7 @@ industry:
 
 costs:
   year: 2030
+  version: v0.4.0
   lifetime: 25 #default lifetime
   # From a Lion Hirth paper, also reflects average of Noothout et al 2016
   discountrate: 0.07

doc/installation.rst

@@ -25,15 +25,6 @@ then download and unpack all the PyPSA-Eur data files by running the following s
     projects/pypsa-eur % snakemake -j 1 retrieve_databundle
 
 
-Clone technology-data repository
-================================
-
-Next install the technology assumptions database `technology-data <https://github.com/PyPSA/technology-data>`_ by creating a parallel directory:
-
-.. code:: bash
-
-    projects % git clone https://github.com/PyPSA/technology-data.git
-
 
 Clone PyPSA-Eur-Sec repository
 ==============================

test/config.myopic.yaml

@@ -3,10 +3,10 @@ version: 0.6.0
 logging_level: INFO
 
 retrieve_sector_databundle: true
+retrieve_cost_data: true
 
 results_dir: results/
 summary_dir: results
-costs_dir: ../technology-data/outputs/
 run: test-myopic  # use this to keep track of runs with different settings
 foresight: myopic # options are overnight, myopic, perfect (perfect is not yet implemented)
 # if you use myopic or perfect foresight, set the investment years in "planning_horizons" below
@@ -320,6 +320,7 @@ industry:
 
 costs:
   year: 2030
+  version: v0.4.0
   lifetime: 25 #default lifetime
   # From a Lion Hirth paper, also reflects average of Noothout et al 2016
   discountrate: 0.07

test/config.overnight.yaml

@@ -3,10 +3,10 @@ version: 0.6.0
 logging_level: INFO
 
 retrieve_sector_databundle: true
+retrieve_cost_data: true
 
 results_dir: results/
 summary_dir: results
-costs_dir: ../technology-data/outputs/
 run: test-overnight  # use this to keep track of runs with different settings
 foresight: overnight # options are overnight, myopic, perfect (perfect is not yet implemented)
 # if you use myopic or perfect foresight, set the investment years in "planning_horizons" below
@@ -318,6 +318,7 @@ industry:
 
 costs:
   year: 2030
+  version: v0.4.0
   lifetime: 25 #default lifetime
   # From a Lion Hirth paper, also reflects average of Noothout et al 2016
   discountrate: 0.07