From cb3c42078a658f4fd905186f836e1be86ae10ee0 Mon Sep 17 00:00:00 2001 From: Vincent Prouillet Date: Thu, 1 Nov 2018 10:36:19 +0100 Subject: [PATCH] Fix load_toml date handling and fix bug in date conversion --- CHANGELOG.md | 2 +- Cargo.lock | 25 ++++---- components/front_matter/Cargo.toml | 1 + components/front_matter/src/lib.rs | 1 + components/front_matter/src/page.rs | 59 +------------------ .../templates/src/global_fns/load_data.rs | 15 +++-- components/utils/Cargo.toml | 2 + components/utils/src/de.rs | 53 +++++++++++++++++ components/utils/src/lib.rs | 3 + 9 files changed, 86 insertions(+), 75 deletions(-) create mode 100644 components/utils/src/de.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ec47fb8..7f6ac6ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Tera function - Table of content now strips HTML from the titles to avoid various issues ### Others +- Many many times faster (x5-x40) for most sites - Update dependencies, fixing a few bugs with templates - Load only .html files in themes from the templates folder - Background colour is set fewer times when highlighting syntaxes, resulting in smaller HTML filesize @@ -18,7 +19,6 @@ Tera function - Load table and footnote markdown extensions in `markdown` filter - `get_url` now defaults to not adding a trailing slash - Fix `--base-url` not overriding processed images URLs -- Many many times faster (x5-x40) for most sites - Add more Emacs temp file to the ignored patterns in `gutenberg serve` - Files starting with `.` are not considered pages anymore even if they end with `.md` - `_processed_images` folder for image processing has been renamed `processed_images` to avoid issues with GitHub Pages diff --git a/Cargo.lock b/Cargo.lock index 0aaa1864..f3f5c1b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,8 +48,8 @@ dependencies = [ "tokio-tcp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", "tower-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trust-dns-proto 0.5.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)", - "trust-dns-resolver 0.10.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)", + "trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trust-dns-resolver 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -638,7 +638,7 @@ name = "errors" version = "0.1.0" dependencies = [ "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", - "image 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", + "image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)", "syntect 3.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "tera 0.11.19 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -719,6 +719,7 @@ dependencies = [ "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", "tera 0.11.19 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "utils 0.1.0", ] [[package]] @@ -945,7 +946,7 @@ dependencies = [ [[package]] name = "image" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -965,7 +966,7 @@ name = "imageproc" version = "0.1.0" dependencies = [ "errors 0.1.0", - "image 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", + "image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2518,7 +2519,7 @@ dependencies = [ [[package]] name = "trust-dns-proto" -version = "0.5.0-alpha.5" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2559,7 +2560,7 @@ dependencies = [ [[package]] name = "trust-dns-resolver" -version = "0.10.0-alpha.3" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2572,7 +2573,7 @@ dependencies = [ "resolv-conf 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", - "trust-dns-proto 0.5.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)", + "trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2717,8 +2718,10 @@ name = "utils" version = "0.1.0" dependencies = [ "errors 0.1.0", + "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "tera 0.11.19 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "walkdir 2.2.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3000,7 +3003,7 @@ dependencies = [ "checksum hyper 0.12.13 (registry+https://github.com/rust-lang/crates.io-index)" = "95ffee0d1d30de4313fdaaa485891ce924991d45bbc18adfc8ac5b1639e62fbb" "checksum hyper-tls 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "32cd73f14ad370d3b4d4b7dce08f69b81536c82e39fcc89731930fe5788cd661" "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" -"checksum image 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60710fd3cb40c2434451d8d5147bcf39bbb68aae0741041133e09439cb2401e3" +"checksum image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44665b4395d1844c96e7dc8ed5754782a1cdfd9ef458a80bbe45702681450504" "checksum indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d" "checksum inflate 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6f53b811ee8e2057ccf9643ca6b4277de90efaf5e61e55fd5254576926bb4245" "checksum inotify 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "40b54539f3910d6f84fbf9a643efd6e3aa6e4f001426c0329576128255994718" @@ -3154,8 +3157,8 @@ dependencies = [ "checksum toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "4a2ecc31b0351ea18b3fe11274b8db6e4d82bce861bbb22e6dbed40417902c65" "checksum tower-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b32f72af77f1bfe3d3d4da8516a238ebe7039b51dd8637a09841ac7f16d2c987" "checksum trust-dns-proto 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f1525ca4e26f5a09d81b79584f19225e7dba5606ae3a416311c2751c5cea60bb" -"checksum trust-dns-proto 0.5.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3daf083dfce3c3554e82b768078331864df1ab220e6c3c1dbb4c847173d33ff6" -"checksum trust-dns-resolver 0.10.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "476d293db1c1027727a1683b681550a24207277e96bb1fb763053e1164621971" +"checksum trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0838272e89f1c693b4df38dc353412e389cf548ceed6f9fd1af5a8d6e0e7cf74" +"checksum trust-dns-resolver 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4e913a5df94658858e548cc95a3212797ee524e487ede091c32f27ca26e11620" "checksum trust-dns-resolver 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4a821ad51a29816420b8cac4b026756b81c023630b97eaa4c8090637ee3508bd" "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" diff --git a/components/front_matter/Cargo.toml b/components/front_matter/Cargo.toml index 76c6a9fa..1de7545a 100644 --- a/components/front_matter/Cargo.toml +++ b/components/front_matter/Cargo.toml @@ -13,3 +13,4 @@ regex = "1" lazy_static = "1" errors = { path = "../errors" } +utils = { path = "../utils" } diff --git a/components/front_matter/src/lib.rs b/components/front_matter/src/lib.rs index ec6d7ae7..986399c4 100644 --- a/components/front_matter/src/lib.rs +++ b/components/front_matter/src/lib.rs @@ -10,6 +10,7 @@ extern crate toml; #[macro_use] extern crate errors; +extern crate utils; use errors::{Result, ResultExt}; use regex::Regex; diff --git a/components/front_matter/src/page.rs b/components/front_matter/src/page.rs index 2f3f99f3..b77259bd 100644 --- a/components/front_matter/src/page.rs +++ b/components/front_matter/src/page.rs @@ -1,66 +1,11 @@ use std::collections::HashMap; -use std::result::Result as StdResult; use chrono::prelude::*; -use serde::{Deserialize, Deserializer}; use tera::{Map, Value}; use toml; use errors::Result; - -fn from_toml_datetime<'de, D>(deserializer: D) -> StdResult, D::Error> -where - D: Deserializer<'de>, -{ - toml::value::Datetime::deserialize(deserializer).map(|s| Some(s.to_string())) -} - -/// Returns key/value for a converted date from TOML. -/// If the table itself is the TOML struct, only return its value without the key -fn convert_toml_date(table: Map) -> Value { - let mut new = Map::new(); - - for (k, v) in table { - if k == "$__toml_private_datetime" { - return v; - } - - match v { - Value::Object(mut o) => { - // that was a toml datetime object, just return the date - if let Some(toml_date) = o.remove("$__toml_private_datetime") { - new.insert(k, toml_date); - return Value::Object(new); - } - new.insert(k, convert_toml_date(o)); - } - _ => { - new.insert(k, v); - } - } - } - - Value::Object(new) -} - -/// TOML datetimes will be serialized as a struct but we want the -/// stringified version for json, otherwise they are going to be weird -fn fix_toml_dates(table: Map) -> Value { - let mut new = Map::new(); - - for (key, value) in table { - match value { - Value::Object(mut o) => { - new.insert(key, convert_toml_date(o)); - } - _ => { - new.insert(key, value); - } - } - } - - Value::Object(new) -} +use utils::de::fix_toml_dates; /// The front matter of every page #[derive(Debug, Clone, PartialEq, Deserialize)] @@ -71,7 +16,7 @@ pub struct PageFrontMatter { /// Description in that appears when linked, e.g. on twitter pub description: Option, /// Date if we want to order pages (ie blog post) - #[serde(default, deserialize_with = "from_toml_datetime")] + #[serde(default, deserialize_with = "utils::de::from_toml_datetime")] pub date: Option, /// Chrono converted datetime #[serde(default, skip_deserializing)] diff --git a/components/templates/src/global_fns/load_data.rs b/components/templates/src/global_fns/load_data.rs index 77614a97..c747cde7 100644 --- a/components/templates/src/global_fns/load_data.rs +++ b/components/templates/src/global_fns/load_data.rs @@ -1,6 +1,7 @@ extern crate serde_json; extern crate toml; +use utils::de::fix_toml_dates; use utils::fs::{get_file_time, is_path_in_directory, read_file}; use reqwest::{header, Client}; @@ -236,7 +237,12 @@ fn load_json(json_data: String) -> Result { /// Parse a TOML string and convert it to a Tera Value fn load_toml(toml_data: String) -> Result { let toml_content: toml::Value = toml::from_str(&toml_data).map_err(|e| format!("{:?}", e))?; - to_value(toml_content).map_err(|e| e.into()) + let toml_value = to_value(toml_content).expect("Got invalid JSON that was valid TOML somehow"); + + match toml_value { + Value::Object(m) => Ok(fix_toml_dates(m)), + _ => unreachable!("Loaded something other than a TOML object"), + } } /// Parse a CSV string and convert it to a Tera Value @@ -409,15 +415,12 @@ mod tests { args.insert("path".to_string(), to_value("test.toml").unwrap()); let result = static_fn(args.clone()).unwrap(); - //TOML does not load in order, and also dates are not returned as strings, but - //rather as another object with a key and value + //TOML does not load in order assert_eq!( result, json!({ "category": { - "date": { - "$__toml_private_datetime": "1979-05-27T07:32:00Z" - }, + "date": "1979-05-27T07:32:00Z", "key": "value" }, }) diff --git a/components/utils/Cargo.toml b/components/utils/Cargo.toml index eaa8419d..e6c558da 100644 --- a/components/utils/Cargo.toml +++ b/components/utils/Cargo.toml @@ -8,6 +8,8 @@ errors = { path = "../errors" } tera = "0.11" unicode-segmentation = "1.2" walkdir = "2" +toml = "0.4" +serde = "1" [dev-dependencies] tempfile = "3" diff --git a/components/utils/src/de.rs b/components/utils/src/de.rs new file mode 100644 index 00000000..27259cce --- /dev/null +++ b/components/utils/src/de.rs @@ -0,0 +1,53 @@ +use serde::{Deserialize, Deserializer}; +use tera::{Map, Value}; +use toml; + +/// Used as an attribute when we want to convert from TOML to a string date +pub fn from_toml_datetime<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + toml::value::Datetime::deserialize(deserializer).map(|s| Some(s.to_string())) +} + +/// Returns key/value for a converted date from TOML. +/// If the table itself is the TOML struct, only return its value without the key +fn convert_toml_date(table: Map) -> Value { + let mut new = Map::new(); + + for (k, v) in table { + if k == "$__toml_private_datetime" { + return v; + } + + match v { + Value::Object(o) => { + new.insert(k, convert_toml_date(o)); + } + _ => { + new.insert(k, v); + } + } + } + + Value::Object(new) +} + +/// TOML datetimes will be serialized as a struct but we want the +/// stringified version for json, otherwise they are going to be weird +pub fn fix_toml_dates(table: Map) -> Value { + let mut new = Map::new(); + + for (key, value) in table { + match value { + Value::Object(mut o) => { + new.insert(key, convert_toml_date(o)); + } + _ => { + new.insert(key, value); + } + } + } + + Value::Object(new) +} diff --git a/components/utils/src/lib.rs b/components/utils/src/lib.rs index edca04bc..25581e80 100644 --- a/components/utils/src/lib.rs +++ b/components/utils/src/lib.rs @@ -1,12 +1,15 @@ #[macro_use] extern crate errors; +extern crate serde; #[cfg(test)] extern crate tempfile; extern crate tera; +extern crate toml; extern crate unicode_segmentation; extern crate walkdir; +pub mod de; pub mod fs; pub mod net; pub mod site;