From 6711d721b9f83719e8871ddf66b2627c25b45ef4 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Mon, 9 Aug 2021 16:30:38 +0200
Subject: [PATCH] biomass_transport: fix cost calculation and get from remote

---
 Snakefile                                |  4 +-
 scripts/build_biomass_transport_costs.py | 93 +++++++++++++++---------
 scripts/prepare_sector_network.py        |  2 +-
 3 files changed, 61 insertions(+), 38 deletions(-)

diff --git a/Snakefile b/Snakefile
index e768bc90..1ef55c86 100644
--- a/Snakefile
+++ b/Snakefile
@@ -186,11 +186,11 @@ rule build_biomass_potentials:
 if config["sector"]["biomass_transport"]:
     rule build_biomass_transport_costs:
         input:
-            transport_cost_data=HTTP.remote("https://publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass%20potentials%20in%20europe_web%20rev.pdf", keep_local=True)
+            transport_cost_data=HTTP.remote("publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", keep_local=True)
         output:
             supply_chain1="resources/biomass_transport_costs_supply_chain1.csv",
             supply_chain2="resources/biomass_transport_costs_supply_chain2.csv",
-            transport_costs="resources/biomass_transport_costs.csv",
+            biomass_transport_costs="resources/biomass_transport_costs.csv",
         threads: 1
         resources: mem_mb=1000
         benchmark: "benchmarks/build_biomass_transport_costs"
diff --git a/scripts/build_biomass_transport_costs.py b/scripts/build_biomass_transport_costs.py
index b17c97fd..dc1137ca 100644
--- a/scripts/build_biomass_transport_costs.py
+++ b/scripts/build_biomass_transport_costs.py
@@ -16,52 +16,75 @@ assuming as an approximation energy content of wood pellets
 import pandas as pd
 import tabula as tbl
 
-ENERGY_CONTENT = 4.8  # unit MWh/tonne (assuming wood pellets)
+ENERGY_CONTENT = 4.8  # unit MWh/t (wood pellets)
+
+def get_countries():
+
+    pandas_options = dict(
+        skiprows=list(range(6)),
+        header=None,
+        index_col=0
+    )
+
+    return tbl.read_pdf(
+        str(snakemake.input.transport_cost_data),
+        pages="145",
+        multiple_tables=False,
+        pandas_options=pandas_options
+    )[0].index
+
+
+def get_cost_per_tkm(page, countries):
+    
+    pandas_options = dict(
+        skiprows=range(6),
+        header=0,
+        sep=' |,',
+        engine='python',
+        index_col=False,
+    )
+
+    sc = tbl.read_pdf(
+        str(snakemake.input.transport_cost_data),
+        pages=page,
+        multiple_tables=False,
+        pandas_options=pandas_options
+    )[0]
+    sc.index = countries
+    sc.columns = sc.columns.str.replace("€", "EUR")
+    
+    return sc
 
 
 def build_biomass_transport_costs():
 
-    df_list = tbl.read_pdf(
-        snakemake.input.transport_cost_data,
-        pages="145-147",
-        multiple_tables=True,
-    )
-    
-    countries = df_list[0][0].iloc[6:].rename(index=lambda x: x + 1)
+    countries = get_countries()
 
-    # supply chain 1
-    df = df_list[1].copy().rename(index=countries.to_dict())
-    df.rename(
-        columns=df.iloc[:6].apply(lambda col: col.str.cat(sep=" "), axis=0).to_dict(),
-        inplace=True,
-    )
-    df = df.iloc[6:]
-    df.loc[6] = df.loc[6].str.replace("€", "EUR")
+    sc1 = get_cost_per_tkm(146, countries)
+    sc2 = get_cost_per_tkm(147, countries)
 
-    # supply chain 2
-    df2 = df_list[2].copy().rename(index=countries.to_dict())
-    df2.rename(
-        columns=df2.iloc[:6].apply(lambda col: col.str.cat(sep=" "), axis=0).to_dict(),
-        inplace=True,
-    )
-    df2 = df2.iloc[6:]
-    df2.loc[6] = df2.loc[6].str.replace("€", "EUR")
+    sc1.to_csv(snakemake.output.supply_chain1)
+    sc2.to_csv(snakemake.output.supply_chain2)
 
-    df.to_csv(snakemake.output.supply_chain1)
-    df2.to_csv(snakemake.output.supply_chain1)
+    # take mean of both supply chains
+    to_concat = [sc1["EUR/km/ton"], sc2["EUR/km/ton"]]
+    transport_costs = pd.concat(to_concat, axis=1).mean(axis=1)
 
-    transport_costs = pd.concat([df["per km/ton"], df2["per km/ton"]], axis=1).drop(6)
-    transport_costs = transport_costs.astype(float, errors="ignore").mean(axis=1)
-
-    # convert unit to EUR/MWh
+    # convert tonnes to MWh
     transport_costs /= ENERGY_CONTENT
-    transport_costs = pd.DataFrame(transport_costs, columns=["cost [EUR/(km MWh)]"])
+    transport_costs.name = "EUR/km/MWh"
 
-    # rename
-    transport_costs.rename({"UK": "GB", "XK": "KO", "EL": "GR"}, inplace=True)
+    # rename country names
+    to_rename = {
+        "UK": "GB",
+        "XK": "KO",
+        "EL": "GR"
+    }
+    transport_costs.rename(to_rename, inplace=True)
+
+    # add missing Norway with data from Sweden
+    transport_costs["NO"] = transport_costs["SE"]
 
-    # add missing Norway
-    transport_costs.loc["NO"] = transport_costs.loc["SE"]
     transport_costs.to_csv(snakemake.output.transport_costs)
 
 
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 0749a8c3..1dd92b28 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -1613,7 +1613,7 @@ def add_biomass(n, costs):
     biomass_potentials = pd.read_csv(snakemake.input.biomass_potentials, index_col=0)
 
     transport_costs = pd.read_csv(
-        snakemake.input.biomass_transport,
+        snakemake.input.biomass_transport_costs,
         index_col=0,
         squeeze=True
     )