Fix load_toml date handling and fix bug in date conversion

2018-11-01 10:36:19 +01:00 · 2018-11-01 10:36:19 +01:00 · cb3c42078a
parent 3ba6707b90
commit cb3c42078a
9 changed files with 86 additions and 75 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -11,6 +11,7 @@ Tera function
 - Table of content now strips HTML from the titles to avoid various issues

 ### Others
+- Many many times faster (x5-x40) for most sites
 - Update dependencies, fixing a few bugs with templates
 - Load only .html files in themes from the templates folder
 - Background colour is set fewer times when highlighting syntaxes, resulting in smaller HTML filesize
@ -18,7 +19,6 @@ Tera function
 - Load table and footnote markdown extensions in `markdown` filter
 - `get_url` now defaults to not adding a trailing slash
 - Fix `--base-url` not overriding processed images URLs
- Many many times faster (x5-x40) for most sites
 - Add more Emacs temp file to the ignored patterns in `gutenberg serve`
 - Files starting with `.` are not considered pages anymore even if they end with `.md`
 - `_processed_images` folder for image processing has been renamed `processed_images` to avoid issues with GitHub Pages
--- a/Cargo.lock
+++ b/Cargo.lock
@ -48,8 +48,8 @@ dependencies = [
 "tokio-tcp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
 "tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
 "tower-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "trust-dns-proto 0.5.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "trust-dns-resolver 0.10.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trust-dns-resolver 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

 [[package]]
@ -638,7 +638,7 @@ name = "errors"
 version = "0.1.0"
 dependencies = [
 "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "image 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)",
 "syntect 3.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
 "tera 0.11.19 (registry+https://github.com/rust-lang/crates.io-index)",
 "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
@ -719,6 +719,7 @@ dependencies = [
 "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
 "tera 0.11.19 (registry+https://github.com/rust-lang/crates.io-index)",
 "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "utils 0.1.0",
 ]

 [[package]]
@ -945,7 +946,7 @@ dependencies = [

 [[package]]
 name = "image"
-version = "0.20.0"
+version = "0.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
 "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
@ -965,7 +966,7 @@ name = "imageproc"
 version = "0.1.0"
 dependencies = [
 "errors 0.1.0",
- "image 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)",
 "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 "rayon 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
 "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2518,7 +2519,7 @@ dependencies = [

 [[package]]
 name = "trust-dns-proto"
-version = "0.5.0-alpha.5"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
 "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2559,7 +2560,7 @@ dependencies = [

 [[package]]
 name = "trust-dns-resolver"
-version = "0.10.0-alpha.3"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
 "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2572,7 +2573,7 @@ dependencies = [
 "resolv-conf 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
 "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
 "tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "trust-dns-proto 0.5.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

 [[package]]
@ -2717,8 +2718,10 @@ name = "utils"
 version = "0.1.0"
 dependencies = [
 "errors 0.1.0",
+ "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
 "tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
 "tera 0.11.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
 "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
 "walkdir 2.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@ -3000,7 +3003,7 @@ dependencies = [
 "checksum hyper 0.12.13 (registry+https://github.com/rust-lang/crates.io-index)" = "95ffee0d1d30de4313fdaaa485891ce924991d45bbc18adfc8ac5b1639e62fbb"
 "checksum hyper-tls 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "32cd73f14ad370d3b4d4b7dce08f69b81536c82e39fcc89731930fe5788cd661"
 "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e"
-"checksum image 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60710fd3cb40c2434451d8d5147bcf39bbb68aae0741041133e09439cb2401e3"
+"checksum image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44665b4395d1844c96e7dc8ed5754782a1cdfd9ef458a80bbe45702681450504"
 "checksum indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d"
 "checksum inflate 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6f53b811ee8e2057ccf9643ca6b4277de90efaf5e61e55fd5254576926bb4245"
 "checksum inotify 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "40b54539f3910d6f84fbf9a643efd6e3aa6e4f001426c0329576128255994718"
@ -3154,8 +3157,8 @@ dependencies = [
 "checksum toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "4a2ecc31b0351ea18b3fe11274b8db6e4d82bce861bbb22e6dbed40417902c65"
 "checksum tower-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b32f72af77f1bfe3d3d4da8516a238ebe7039b51dd8637a09841ac7f16d2c987"
 "checksum trust-dns-proto 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f1525ca4e26f5a09d81b79584f19225e7dba5606ae3a416311c2751c5cea60bb"
-"checksum trust-dns-proto 0.5.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3daf083dfce3c3554e82b768078331864df1ab220e6c3c1dbb4c847173d33ff6"
-"checksum trust-dns-resolver 0.10.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "476d293db1c1027727a1683b681550a24207277e96bb1fb763053e1164621971"
+"checksum trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0838272e89f1c693b4df38dc353412e389cf548ceed6f9fd1af5a8d6e0e7cf74"
+"checksum trust-dns-resolver 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4e913a5df94658858e548cc95a3212797ee524e487ede091c32f27ca26e11620"
 "checksum trust-dns-resolver 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4a821ad51a29816420b8cac4b026756b81c023630b97eaa4c8090637ee3508bd"
 "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
 "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169"
--- a/components/front_matter/Cargo.toml
+++ b/components/front_matter/Cargo.toml
@ -13,3 +13,4 @@ regex = "1"
 lazy_static = "1"

 errors = { path = "../errors" }
+utils = { path = "../utils" }
--- a/components/front_matter/src/lib.rs
+++ b/components/front_matter/src/lib.rs
@ -10,6 +10,7 @@ extern crate toml;

 #[macro_use]
 extern crate errors;
+extern crate utils;

 use errors::{Result, ResultExt};
 use regex::Regex;
--- a/components/front_matter/src/page.rs
+++ b/components/front_matter/src/page.rs
@ -1,66 +1,11 @@
 use std::collections::HashMap;
-use std::result::Result as StdResult;

 use chrono::prelude::*;
-use serde::{Deserialize, Deserializer};
 use tera::{Map, Value};
 use toml;

 use errors::Result;
-
-fn from_toml_datetime<'de, D>(deserializer: D) -> StdResult<Option<String>, D::Error>
-where
-    D: Deserializer<'de>,
-{
-    toml::value::Datetime::deserialize(deserializer).map(|s| Some(s.to_string()))
-}
-
-/// Returns key/value for a converted date from TOML.
-/// If the table itself is the TOML struct, only return its value without the key
-fn convert_toml_date(table: Map<String, Value>) -> Value {
-    let mut new = Map::new();
-
-    for (k, v) in table {
-        if k == "$__toml_private_datetime" {
-            return v;
-        }
-
-        match v {
-            Value::Object(mut o) => {
-                // that was a toml datetime object, just return the date
-                if let Some(toml_date) = o.remove("$__toml_private_datetime") {
-                    new.insert(k, toml_date);
-                    return Value::Object(new);
-                }
-                new.insert(k, convert_toml_date(o));
-            }
-            _ => {
-                new.insert(k, v);
-            }
-        }
-    }
-
-    Value::Object(new)
-}
-
-/// TOML datetimes will be serialized as a struct but we want the
-/// stringified version for json, otherwise they are going to be weird
-fn fix_toml_dates(table: Map<String, Value>) -> Value {
-    let mut new = Map::new();
-
-    for (key, value) in table {
-        match value {
-            Value::Object(mut o) => {
-                new.insert(key, convert_toml_date(o));
-            }
-            _ => {
-                new.insert(key, value);
-            }
-        }
-    }
-
-    Value::Object(new)
-}
+use utils::de::fix_toml_dates;

 /// The front matter of every page
 #[derive(Debug, Clone, PartialEq, Deserialize)]
@ -71,7 +16,7 @@ pub struct PageFrontMatter {
    /// Description in <meta> that appears when linked, e.g. on twitter
    pub description: Option<String>,
    /// Date if we want to order pages (ie blog post)
-    #[serde(default, deserialize_with = "from_toml_datetime")]
+    #[serde(default, deserialize_with = "utils::de::from_toml_datetime")]
    pub date: Option<String>,
    /// Chrono converted datetime
    #[serde(default, skip_deserializing)]
--- a/components/templates/src/global_fns/load_data.rs
+++ b/components/templates/src/global_fns/load_data.rs
@ -1,6 +1,7 @@
 extern crate serde_json;
 extern crate toml;

+use utils::de::fix_toml_dates;
 use utils::fs::{get_file_time, is_path_in_directory, read_file};

 use reqwest::{header, Client};
@ -236,7 +237,12 @@ fn load_json(json_data: String) -> Result<Value> {
 /// Parse a TOML string and convert it to a Tera Value
 fn load_toml(toml_data: String) -> Result<Value> {
    let toml_content: toml::Value = toml::from_str(&toml_data).map_err(|e| format!("{:?}", e))?;
-    to_value(toml_content).map_err(|e| e.into())
+    let toml_value = to_value(toml_content).expect("Got invalid JSON that was valid TOML somehow");
+
+    match toml_value {
+        Value::Object(m) => Ok(fix_toml_dates(m)),
+        _ => unreachable!("Loaded something other than a TOML object"),
+    }
 }

 /// Parse a CSV string and convert it to a Tera Value
@ -409,15 +415,12 @@ mod tests {
        args.insert("path".to_string(), to_value("test.toml").unwrap());
        let result = static_fn(args.clone()).unwrap();

-        //TOML does not load in order, and also dates are not returned as strings, but
-        //rather as another object with a key and value
+        //TOML does not load in order
        assert_eq!(
            result,
            json!({
            "category": {
-                "date": {
-                    "$__toml_private_datetime": "1979-05-27T07:32:00Z"
-                },
+                "date": "1979-05-27T07:32:00Z",
                "key": "value"
            },
        })
--- a/components/utils/Cargo.toml
+++ b/components/utils/Cargo.toml
@ -8,6 +8,8 @@ errors = { path = "../errors" }
 tera = "0.11"
 unicode-segmentation = "1.2"
 walkdir = "2"
+toml = "0.4"
+serde = "1"

 [dev-dependencies]
 tempfile = "3"
--- a/components/utils/src/de.rs
+++ b/components/utils/src/de.rs
@ -0,0 +1,53 @@
+use serde::{Deserialize, Deserializer};
+use tera::{Map, Value};
+use toml;
+
+/// Used as an attribute when we want to convert from TOML to a string date
+pub fn from_toml_datetime<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    toml::value::Datetime::deserialize(deserializer).map(|s| Some(s.to_string()))
+}
+
+/// Returns key/value for a converted date from TOML.
+/// If the table itself is the TOML struct, only return its value without the key
+fn convert_toml_date(table: Map<String, Value>) -> Value {
+    let mut new = Map::new();
+
+    for (k, v) in table {
+        if k == "$__toml_private_datetime" {
+            return v;
+        }
+
+        match v {
+            Value::Object(o) => {
+                new.insert(k, convert_toml_date(o));
+            }
+            _ => {
+                new.insert(k, v);
+            }
+        }
+    }
+
+    Value::Object(new)
+}
+
+/// TOML datetimes will be serialized as a struct but we want the
+/// stringified version for json, otherwise they are going to be weird
+pub fn fix_toml_dates(table: Map<String, Value>) -> Value {
+    let mut new = Map::new();
+
+    for (key, value) in table {
+        match value {
+            Value::Object(mut o) => {
+                new.insert(key, convert_toml_date(o));
+            }
+            _ => {
+                new.insert(key, value);
+            }
+        }
+    }
+
+    Value::Object(new)
+}
--- a/components/utils/src/lib.rs
+++ b/components/utils/src/lib.rs
@ -1,12 +1,15 @@
 #[macro_use]
 extern crate errors;

+extern crate serde;
 #[cfg(test)]
 extern crate tempfile;
 extern crate tera;
+extern crate toml;
 extern crate unicode_segmentation;
 extern crate walkdir;

+pub mod de;
 pub mod fs;
 pub mod net;
 pub mod site;