Build biomass transport costs from pre-extracted .csv instead of pdf (#1272)
* Removed tabula dependency for building biomass transport costs. Instead use pre-extracted .csvs. Updated rule and script accordingly. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updated script to optionally allow running the script with tabula-py dependency to recreate csv, if needed. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Bobby Xiong
GitHub
parent
ca9556597e
commit
ac606c1f67
41
data/biomass_transport_costs_supplychain1.csv
Normal file
41
data/biomass_transport_costs_supplychain1.csv
Normal file
@ -0,0 +1,41 @@
|
||||
country,Loading,Loading,Loading,Transport to plant,Transport to plant,Unloading,TOTAL,TOTAL,TOTAL
|
||||
country,Time,Machinery costs,Waiting time truck driver,60 average speed km/h,Trailer,Waiting time truck driver,Fix,per km,per km/ton
|
||||
country,h,EUR,EUR.1,EUR/km,EUR.2,EUR.3,EUR.4,EUR/km.1,EUR/km/ton
|
||||
BE,0.08,3.56,1.31,1.45,6.42,1.31,12.61,0.02,0.47
|
||||
BG,0.08,2.06,0.56,1.06,2.61,0.56,5.79,0.02,0.22
|
||||
CZ,0.08,2.34,0.62,1.1,4.08,0.62,7.65,0.02,0.29
|
||||
DK,0.08,5.14,2.54,1.85,7.91,2.54,18.13,0.03,0.68
|
||||
DE,0.08,3.41,1.31,1.37,5.99,1.31,12.01,0.02,0.45
|
||||
EE,0.08,2.26,0.5,1.1,4.14,0.5,7.38,0.02,0.28
|
||||
IE,0.08,3.29,1.03,1.4,6.29,1.03,11.64,0.02,0.44
|
||||
EL,0.08,2.83,0.85,1.25,5.17,0.85,9.7,0.02,0.37
|
||||
ES,0.08,3.19,1.22,1.32,5.28,1.22,10.91,0.02,0.41
|
||||
FR,0.08,3.52,1.35,1.38,6.49,1.35,12.71,0.02,0.48
|
||||
IT,0.08,3.59,1.26,1.56,5.82,1.26,11.94,0.03,0.45
|
||||
CY,0.08,2.82,0.85,1.25,5.09,0.85,9.6,0.02,0.36
|
||||
LV,0.08,2.11,0.43,1.05,3.87,0.43,6.84,0.02,0.26
|
||||
LT,0.08,1.99,0.39,1.02,3.49,0.39,6.25,0.02,0.24
|
||||
LU,0.08,3.38,1.21,1.31,6.97,1.21,12.78,0.02,0.48
|
||||
HU,0.08,2.12,0.42,1.12,3.33,0.42,6.28,0.02,0.24
|
||||
MT,0.08,2.44,0.62,1.16,4.32,0.62,7.99,0.02,0.3
|
||||
NL,0.08,3.83,1.57,1.51,6.37,1.57,13.33,0.03,0.5
|
||||
AT,0.08,3.33,1.18,1.35,6.37,1.18,12.06,0.02,0.45
|
||||
PL,0.08,2.27,0.68,1.08,3.35,0.68,6.98,0.02,0.27
|
||||
PT,0.08,2.3,0.37,1.16,4.67,0.37,7.71,0.02,0.29
|
||||
RO,0.08,1.88,0.4,0.99,2.8,0.4,5.49,0.02,0.21
|
||||
SI,0.08,2.47,0.58,1.18,4.65,0.58,8.29,0.02,0.32
|
||||
SK,0.08,2.24,0.48,1.12,3.93,0.48,7.13,0.02,0.27
|
||||
FI,0.08,3.59,1.2,1.48,7.0,1.2,12.98,0.02,0.49
|
||||
SE,0.08,3.84,1.26,1.57,7.71,1.26,14.07,0.03,0.53
|
||||
UK,0.08,3.65,1.21,1.56,6.56,1.21,12.64,0.03,0.48
|
||||
HR,0.08,2.12,0.46,1.05,3.79,0.46,6.84,0.02,0.26
|
||||
AL,0.08,1.67,0.19,0.96,2.55,0.19,4.61,0.02,0.18
|
||||
BA,0.08,1.66,0.23,0.91,2.81,0.23,4.92,0.02,0.19
|
||||
MK,0.08,1.5,0.22,0.84,2.32,0.22,4.27,0.01,0.17
|
||||
ME,0.08,1.71,0.24,0.94,2.83,0.24,5.02,0.02,0.2
|
||||
RS,0.08,1.69,0.22,0.96,2.64,0.22,4.77,0.02,0.19
|
||||
XK,0.08,1.62,0.21,0.89,2.81,0.21,4.85,0.01,0.19
|
||||
UA,0.08,1.25,0.13,0.65,2.8,0.13,4.32,0.01,0.16
|
||||
TR,0.08,2.35,0.52,1.24,3.44,0.52,6.82,0.02,0.26
|
||||
MD,0.08,1.33,0.09,0.74,2.8,0.09,4.31,0.01,0.17
|
||||
CH,0.08,3.79,1.18,1.47,8.9,1.18,15.06,0.02,0.56
|
||||
|
41
data/biomass_transport_costs_supplychain2.csv
Normal file
41
data/biomass_transport_costs_supplychain2.csv
Normal file
@ -0,0 +1,41 @@
|
||||
country,Loading,Loading,Loading,Transport to plant,Transport to plant,Unloading,TOTAL,TOTAL,TOTAL
|
||||
country,Time,Machinery costs,Waiting time truck driver,60 average speed km/h,Trailer,Waiting time truck driver,Fix,per km,per km/ton
|
||||
country,h,EUR,EUR.1,EUR/km,EUR.2,EUR.3,EUR.4,EUR/km.1,EUR/km/ton
|
||||
BE,0.05,3.06,0.88,1.49,16.57,0.88,21.39,0.02,0.88
|
||||
BG,0.05,1.77,0.37,1.07,6.74,0.37,9.26,0.02,0.39
|
||||
CZ,0.05,2.05,0.41,1.13,10.52,0.41,13.4,0.02,0.55
|
||||
DK,0.05,4.24,1.7,1.89,20.4,1.7,28.04,0.03,1.15
|
||||
DE,0.05,2.91,0.87,1.41,15.45,0.87,20.11,0.02,0.83
|
||||
EE,0.05,2.01,0.33,1.13,10.67,0.33,13.34,0.02,0.55
|
||||
IE,0.05,2.87,0.69,1.44,16.23,0.69,20.48,0.02,0.84
|
||||
EL,0.05,2.47,0.57,1.28,13.34,0.57,16.95,0.02,0.7
|
||||
ES,0.05,2.71,0.82,1.35,13.63,0.82,17.98,0.02,0.74
|
||||
FR,0.05,3.02,0.91,1.42,16.74,0.91,21.56,0.02,0.89
|
||||
IT,0.05,3.07,0.85,1.59,15.0,0.85,19.77,0.03,0.82
|
||||
CY,0.05,2.46,0.57,1.28,13.12,0.57,16.71,0.02,0.69
|
||||
LV,0.05,1.89,0.29,1.08,9.98,0.29,12.44,0.02,0.52
|
||||
LT,0.05,1.78,0.26,1.04,9.01,0.26,11.3,0.02,0.47
|
||||
LU,0.05,2.94,0.81,1.35,17.98,0.81,22.54,0.02,0.92
|
||||
HU,0.05,1.88,0.28,1.14,8.59,0.28,11.03,0.02,0.46
|
||||
MT,0.05,2.15,0.41,1.19,11.15,0.41,14.12,0.02,0.58
|
||||
NL,0.05,3.23,1.05,1.55,16.42,1.05,21.76,0.03,0.9
|
||||
AT,0.05,2.88,0.79,1.38,16.42,0.79,20.88,0.02,0.86
|
||||
PL,0.05,1.96,0.46,1.1,8.64,0.46,11.51,0.02,0.48
|
||||
PT,0.05,2.09,0.25,1.18,12.04,0.25,14.63,0.02,0.6
|
||||
RO,0.05,1.66,0.27,1.01,7.23,0.27,9.43,0.02,0.39
|
||||
SI,0.05,2.2,0.39,1.2,12.0,0.39,14.98,0.02,0.62
|
||||
SK,0.05,1.99,0.32,1.15,10.13,0.32,12.77,0.02,0.53
|
||||
FI,0.05,3.12,0.8,1.52,18.05,0.8,22.78,0.03,0.94
|
||||
SE,0.05,3.35,0.85,1.62,19.89,0.85,24.93,0.03,1.02
|
||||
UK,0.05,3.16,0.81,1.6,16.93,0.81,21.71,0.03,0.9
|
||||
HR,0.05,1.89,0.31,1.07,9.77,0.31,12.28,0.02,0.51
|
||||
AL,0.05,1.5,0.13,0.98,6.59,0.13,8.35,0.02,0.35
|
||||
BA,0.05,1.5,0.15,0.93,7.25,0.15,9.05,0.02,0.38
|
||||
MK,0.05,1.35,0.15,0.85,5.98,0.15,7.62,0.01,0.32
|
||||
ME,0.05,1.54,0.16,0.96,7.31,0.16,9.17,0.02,0.38
|
||||
RS,0.05,1.52,0.15,0.97,6.8,0.15,8.62,0.02,0.36
|
||||
XK,0.05,1.46,0.14,0.91,7.25,0.14,8.99,0.02,0.37
|
||||
UA,0.05,1.16,0.09,0.66,7.23,0.09,8.57,0.01,0.35
|
||||
TR,0.05,2.07,0.35,1.26,8.88,0.35,11.63,0.02,0.49
|
||||
MD,0.05,1.24,0.06,0.75,7.23,0.06,8.59,0.01,0.36
|
||||
CH,0.05,3.37,0.79,1.52,22.97,0.79,27.92,0.03,1.14
|
||||
|
@ -64,4 +64,3 @@ dependencies:
|
||||
- snakemake-executor-plugin-slurm
|
||||
- snakemake-executor-plugin-cluster-generic
|
||||
- highspy
|
||||
- tabula-py
|
||||
|
||||
@ -435,10 +435,8 @@ rule build_biomass_potentials:
|
||||
|
||||
rule build_biomass_transport_costs:
|
||||
input:
|
||||
transport_cost_data=storage(
|
||||
"https://publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf",
|
||||
keep_local=True,
|
||||
),
|
||||
sc1="data/biomass_transport_costs_supplychain1.csv",
|
||||
sc2="data/biomass_transport_costs_supplychain2.csv",
|
||||
output:
|
||||
biomass_transport_costs=resources("biomass_transport_costs.csv"),
|
||||
threads: 1
|
||||
|
||||
@ -16,33 +16,51 @@ assuming as an approximation energy content of wood pellets
|
||||
|
||||
@author: bw0928
|
||||
"""
|
||||
|
||||
import platform
|
||||
|
||||
import pandas as pd
|
||||
import tabula as tbl
|
||||
|
||||
ENERGY_CONTENT = 4.8 # unit MWh/t (wood pellets)
|
||||
system = platform.system()
|
||||
encoding = "cp1252" if system == "Windows" else "utf-8"
|
||||
|
||||
|
||||
def get_countries():
|
||||
pandas_options = dict(
|
||||
def get_cost_per_tkm(pdf, datapage, countrypage):
|
||||
"""
|
||||
Extracts the cost tables from the JRC report PDF.
|
||||
|
||||
https://publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass%20potentials%20in%20europe_web%20rev.pdf
|
||||
- pdf (str): The filepath of the PDF file containing the data.
|
||||
- datapage (int): The page number of the data table in the PDF.
|
||||
- countrypage (int): The page number of the table containing the country indices in the PDF.
|
||||
|
||||
Returns:
|
||||
- pandas.DataFrame: The data table with the cost per tkm for biomass transport, indexed by country.
|
||||
|
||||
Raises:
|
||||
- ImportError: If tabula-py and platform are not installed.
|
||||
"""
|
||||
try:
|
||||
import platform
|
||||
|
||||
import tabula as tbl
|
||||
except:
|
||||
ImportError("Please install tabula-py and platform")
|
||||
|
||||
system = platform.system()
|
||||
encoding = "cp1252" if system == "Windows" else "utf-8"
|
||||
|
||||
# Obtain countries:
|
||||
pandas_options_country = dict(
|
||||
skiprows=range(6), header=None, index_col=0, encoding=encoding
|
||||
)
|
||||
|
||||
return tbl.read_pdf(
|
||||
str(snakemake.input.transport_cost_data),
|
||||
pages="145",
|
||||
countries = tbl.read_pdf(
|
||||
pdf,
|
||||
pages=countrypage,
|
||||
multiple_tables=False,
|
||||
pandas_options=pandas_options,
|
||||
pandas_options=pandas_options_country,
|
||||
encoding=encoding,
|
||||
)[0].index
|
||||
|
||||
|
||||
def get_cost_per_tkm(page, countries):
|
||||
pandas_options = dict(
|
||||
# Obtain data tables
|
||||
pandas_options_data = dict(
|
||||
skiprows=range(6),
|
||||
header=0,
|
||||
sep=" |,",
|
||||
@ -52,10 +70,10 @@ def get_cost_per_tkm(page, countries):
|
||||
)
|
||||
|
||||
sc = tbl.read_pdf(
|
||||
str(snakemake.input.transport_cost_data),
|
||||
pages=page,
|
||||
pdf,
|
||||
pages=datapage,
|
||||
multiple_tables=False,
|
||||
pandas_options=pandas_options,
|
||||
pandas_options=pandas_options_data,
|
||||
encoding=encoding,
|
||||
)[0]
|
||||
sc.index = countries
|
||||
@ -65,10 +83,16 @@ def get_cost_per_tkm(page, countries):
|
||||
|
||||
|
||||
def build_biomass_transport_costs():
|
||||
countries = get_countries()
|
||||
# Optional build from JRC report pdf, requires tabula and java dependencies.
|
||||
# Update `pdf` path to the JRC report if needed.
|
||||
# sc1 = get_cost_per_tkm(pdf = "report.pdf", datapage=146, countrypage=145)
|
||||
# sc2 = get_cost_per_tkm(pdf = "report.pdf", datapage=147, countrypage=145)
|
||||
|
||||
sc1 = get_cost_per_tkm(146, countries)
|
||||
sc2 = get_cost_per_tkm(147, countries)
|
||||
# Use extracted csv from JRC report
|
||||
# https://publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass%20potentials%20in%20europe_web%20rev.pdf
|
||||
# Pages 146 (144) for supply chain 1 and 147 (145) for supply chain 2
|
||||
sc1 = pd.read_csv(snakemake.input.sc1, index_col=0, skiprows=2)
|
||||
sc2 = pd.read_csv(snakemake.input.sc2, index_col=0, skiprows=2)
|
||||
|
||||
# take mean of both supply chains
|
||||
to_concat = [sc1["EUR/km/ton"], sc2["EUR/km/ton"]]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user