CSV and TOML loading global functions (#379)

Local CSV/TOML/JSON loading Tera function

Cargo.lock

@@ -449,6 +449,23 @@ name = "crossbeam-utils"
 version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "csv"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "ctrlc"
 version = "3.1.1"
@@ -2167,12 +2184,16 @@ version = "0.1.0"
 dependencies = [
  "base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "config 0.1.0",
+ "csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "errors 0.1.0",
  "imageproc 0.1.0",
  "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "library 0.1.0",
  "pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)",
  "tera 0.11.17 (registry+https://github.com/rust-lang/crates.io-index)",
+ "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "utils 0.1.0",
 ]
 
@@ -2875,6 +2896,8 @@ dependencies = [
 "checksum crossbeam-epoch 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9c90f1474584f38e270b5b613e898c8c328aa4f3dea85e0a27ac2e642f009416"
 "checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9"
 "checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015"
+"checksum csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d54f6b0fd69128a2894b1a3e57af5849a0963c1cc77b165d30b896e40296452"
+"checksum csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4dd8e6d86f7ba48b4276ef1317edc8cc36167546d8972feb4a2b5fec0b374105"
 "checksum ctrlc 3.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "630391922b1b893692c6334369ff528dcc3a9d8061ccf4c803aa8f83cb13db5e"
 "checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
 "checksum deflate 0.7.19 (registry+https://github.com/rust-lang/crates.io-index)" = "8a6abb26e16e8d419b5c78662aa9f82857c2386a073da266840e474d5055ec86"

README.md

@@ -29,7 +29,7 @@ in the `docs/content` folder of the repository.
 | Pagination                      |     ✔     |    ✕   |   ✔  |    ✔    |
 | Custom taxonomies               |     ✔     |    ✕   |   ✔  |    ✕    |
 | Search                          |     ✔     |    ✕   |   ✕  |    ✔    |
-| Data files                      |     ✕     |    ✔   |   ✔  |    ✕    |
+| Data files                      |     ✔     |    ✔   |   ✔  |    ✕    |
 | LiveReload                      |     ✔     |    ✕   |   ✔  |    ✔    |
 | Netlify support                 |     ✔     |    ✕   |   ✔  |    ✕    |
 

components/site/src/lib.rs

@@ -307,6 +307,7 @@ impl Site {
             "get_taxonomy_url",
             global_fns::make_get_taxonomy_url(&self.taxonomies),
         );
+        self.tera.register_function("load_data", global_fns::make_load_data(self.content_path.clone()));
     }
 
     /// Add a page to the site

components/templates/Cargo.toml

@@ -8,6 +8,10 @@ tera = "0.11"
 base64 = "0.9"
 lazy_static = "1"
 pulldown-cmark = "0"
+toml = "0.4"
+csv = "1"
+serde_json = "1.0"
+error-chain = "0.12"
 
 errors = { path = "../errors" }
 utils = { path = "../utils" }

components/templates/src/global_fns.rs

@@ -1,14 +1,21 @@
+extern crate toml;
+extern crate serde_json;
+extern crate error_chain;
+
 use std::collections::HashMap;
 use std::sync::{Arc, Mutex};
+use std::path::PathBuf;
 
-use tera::{GlobalFn, Value, from_value, to_value, Result};
+use csv::Reader;
+
+use tera::{GlobalFn, Value, from_value, to_value, Result, Map};
 
 use library::{Taxonomy, Library};
 use config::Config;
 use utils::site::resolve_internal_link;
+use utils::fs::read_file;
 use imageproc;
 
-
 macro_rules! required_arg {
     ($ty: ty, $e: expr, $err: expr) => {
         match $e {
@@ -264,12 +271,132 @@ pub fn make_resize_image(imageproc: Arc<Mutex<imageproc::Processor>>) -> GlobalF
     })
 }
 
+/// A global function to load data from a data file.
+/// Currently the supported formats are json, toml and csv
+pub fn make_load_data(content_path: PathBuf) -> GlobalFn {
+    Box::new(move |args| -> Result<Value> {
+        let path_arg: String = required_arg!(
+            String, 
+            args.get("path"), 
+            "`load_data`: requires a `path` argument with a string value, being a path to a file"
+        );
+        let kind_arg = optional_arg!(
+            String,
+            args.get("kind"),
+            "`load_data`: `kind` needs to be an argument with a string value, being one of the supported `load_data` file types (csv, json, toml)"
+        );
+
+        let full_path = content_path.join(&path_arg);
+
+        let extension = match full_path.extension() {
+            Some(value) => value.to_str().unwrap().to_lowercase(),
+            None => return Err(format!("`load_data`: Cannot parse file extension of specified file: {}", path_arg).into())
+        };
+
+        let file_kind = kind_arg.unwrap_or(extension);
+
+        let result_value: Result<Value> = match file_kind.as_str() {
+            "toml" => load_toml(&full_path),
+            "csv" => load_csv(&full_path),
+            "json" => load_json(&full_path),
+            _ => return Err(format!("'load_data': {} - is an unsupported file kind", file_kind).into())
+        };
+
+        result_value
+    })
+}
+
+/// load/parse a json file from the given path and place it into a 
+/// tera value
+fn load_json(json_path: &PathBuf) -> Result<Value> {
+    
+    let content_string: String = read_file(json_path)
+        .map_err(|e| format!("`load_data`: error {} loading json file {}", json_path.to_str().unwrap(), e))?;
+
+    let json_content = serde_json::from_str(content_string.as_str()).unwrap();
+    let tera_value: Value = json_content;
+
+    return Ok(tera_value);
+}
+
+/// load/parse a toml file from the given path, and place it into a
+/// tera Value
+fn load_toml(toml_path: &PathBuf) -> Result<Value> {
+    let content_string: String = read_file(toml_path)
+        .map_err(|e| format!("`load_data`: error {} loading toml file {}", toml_path.to_str().unwrap(), e))?;
+
+    let toml_content: toml::Value = toml::from_str(&content_string)
+        .map_err(|e| format!("'load_data': {} - {}", toml_path.to_str().unwrap(), e))?;
+
+    to_value(toml_content).map_err(|err| err.into())
+}
+
+/// Load/parse a csv file from the given path, and place it into a 
+/// tera Value.
+/// 
+/// An example csv file `example.csv` could be:
+/// ```csv
+/// Number, Title
+/// 1,Gutenberg
+/// 2,Printing
+/// ```
+/// The json value output would be:
+/// ```json
+/// {
+///     "headers": ["Number", "Title"],
+///     "records": [
+///                     ["1", "Gutenberg"], 
+///                     ["2", "Printing"]
+///                ],
+/// }
+/// ```
+fn load_csv(csv_path: &PathBuf) -> Result<Value> {
+    let mut reader = Reader::from_path(csv_path.clone())
+        .map_err(|e| format!("'load_data': {} - {}", csv_path.to_str().unwrap(), e))?;
+
+    let mut csv_map = Map::new();
+
+    {
+        let hdrs = reader.headers()
+            .map_err(|e| format!("'load_data': {} - {} - unable to read CSV header line (line 1) for CSV file", csv_path.to_str().unwrap(), e))?;
+
+        let headers_array = hdrs.iter()
+            .map(|v| Value::String(v.to_string()))
+            .collect();
+
+        csv_map.insert(String::from("headers"), Value::Array(headers_array));
+    }
+
+    {
+        let records = reader.records();
+
+        let mut records_array: Vec<Value> = Vec::new();
+
+        for result in records {
+            let record = result.unwrap();
+
+            let mut elements_array: Vec<Value> = Vec::new();
+
+            for e in record.into_iter() {
+                elements_array.push(Value::String(String::from(e)));
+            }
+
+            records_array.push(Value::Array(elements_array));
+        }
+
+        csv_map.insert(String::from("records"), Value::Array(records_array));
+    }
+
+    let csv_value: Value = Value::Object(csv_map);
+    to_value(csv_value).map_err(|err| err.into())
+}
 
 #[cfg(test)]
 mod tests {
-    use super::{make_get_url, make_get_taxonomy, make_get_taxonomy_url, make_trans};
+    use super::{make_get_url, make_get_taxonomy, make_get_taxonomy_url, make_trans, make_load_data};
 
     use std::collections::HashMap;
+    use std::path::PathBuf;
 
     use tera::{to_value, Value};
 
@@ -422,4 +549,58 @@ title = "A title"
         args.insert("lang".to_string(), to_value("fr").unwrap());
         assert_eq!(static_fn(args.clone()).unwrap(), "Un titre");
     }
+
+    #[test]
+    fn can_load_toml()
+    {
+        let static_fn = make_load_data(PathBuf::from("../utils/test-files"));
+        let mut args = HashMap::new();
+        args.insert("path".to_string(), to_value("test.toml").unwrap());
+        let result = static_fn(args.clone()).unwrap();
+
+        //TOML does not load in order, and also dates are not returned as strings, but
+        //rather as another object with a key and value
+        assert_eq!(result, json!({
+            "category": {
+                "date": {
+                    "$__toml_private_datetime": "1979-05-27T07:32:00Z"
+                },
+                "key": "value"
+            },
+        }));
+    }
+
+    #[test]
+    fn can_load_csv()
+    {
+        let static_fn = make_load_data(PathBuf::from("../utils/test-files"));
+        let mut args = HashMap::new();
+        args.insert("path".to_string(), to_value("test.csv").unwrap());
+        let result = static_fn(args.clone()).unwrap();
+
+        assert_eq!(result, json!({
+            "headers": ["Number", "Title"],
+            "records": [
+                            ["1", "Gutenberg"], 
+                            ["2", "Printing"]
+                        ],
+        }))
+    }
+
+    #[test]
+    fn can_load_json()
+    {
+        let static_fn = make_load_data(PathBuf::from("../utils/test-files"));
+        let mut args = HashMap::new();
+        args.insert("path".to_string(), to_value("test.json").unwrap());
+        let result = static_fn(args.clone()).unwrap();
+
+        assert_eq!(result, json!({
+            "key": "value",
+            "array": [1, 2, 3],
+            "subpackage": {
+                "subkey": 5
+            }
+        }))
+    }
 }

components/templates/src/lib.rs

@@ -4,6 +4,13 @@ extern crate lazy_static;
 extern crate tera;
 extern crate base64;
 extern crate pulldown_cmark;
+extern crate csv;
+
+#[cfg(test)]
+#[macro_use]
+extern crate serde_json;
+#[cfg(not(test))]
+extern crate serde_json;
 
 extern crate errors;
 extern crate utils;

components/utils/test-files/test.csv

@@ -0,0 +1,3 @@
+Number,Title
+1,Gutenberg
+2,Printing

components/utils/test-files/test.json

@@ -0,0 +1,7 @@
+{
+    "key": "value",
+    "array": [1, 2, 3],
+    "subpackage": {
+        "subkey": 5
+    }
+}

components/utils/test-files/test.toml

@@ -0,0 +1,3 @@
+[category]
+key = "value"
+date = 1979-05-27T07:32:00Z

docs/content/documentation/templates/overview.md

@@ -142,6 +142,52 @@ Gets the whole taxonomy of a specific kind.
 {% set categories = get_taxonomy_url(kind="categories") %}
 ```
 
+### `load_data`
+Loads data from a file. Supported file types include *toml*, *json* and *csv*.
+
+The `path` argument specifies the path to the data file relative to your content directory.
+
+```jinja2
+{% set data = load_data(path="blog/story/data.toml") %}
+```
+
+The optional `kind` argument allows you to specify and override which data type is contained 
+within the file specified in the `path` argument. Valid entries are *"toml"*, *"json"*
+or *"csv"*. 
+
+```jinja2
+{% set data = load_data(path="blog/story/data.txt", kind="json") %}
+```
+
+For *toml* and *json* the data is loaded into a structure matching the original data file,
+however for *csv* there is no native notion of such a structure. Instead the data is seperated
+into a data structure containing *headers* and *records*. See the example below to see 
+how this works.
+
+In the template:
+```jinja2
+{% set data = load_data(path="blog/story/data.csv") %}
+```
+
+In the *blog/story/data.csv* file:
+ ```csv
+Number, Title
+1,Gutenberg
+2,Printing
+```
+
+The equivalent json value of the parsed data would be stored in the `data` variable in the
+template:
+```json
+{
+    "headers": ["Number", "Title"],
+    "records": [
+        ["1", "Gutenberg"], 
+        ["2", "Printing"]
+    ],
+}
+ ```
+
 ### `trans`
 Gets the translation of the given `key`, for the `default_language` or the `language given