Fix load_toml date handling and fix bug in date conversion

This commit is contained in:
Vincent Prouillet 2018-11-01 10:36:19 +01:00
parent 3ba6707b90
commit cb3c42078a
9 changed files with 86 additions and 75 deletions

View file

@ -11,6 +11,7 @@ Tera function
- Table of content now strips HTML from the titles to avoid various issues
### Others
- Many many times faster (x5-x40) for most sites
- Update dependencies, fixing a few bugs with templates
- Load only .html files in themes from the templates folder
- Background colour is set fewer times when highlighting syntaxes, resulting in smaller HTML filesize
@ -18,7 +19,6 @@ Tera function
- Load table and footnote markdown extensions in `markdown` filter
- `get_url` now defaults to not adding a trailing slash
- Fix `--base-url` not overriding processed images URLs
- Many many times faster (x5-x40) for most sites
- Add more Emacs temp file to the ignored patterns in `gutenberg serve`
- Files starting with `.` are not considered pages anymore even if they end with `.md`
- `_processed_images` folder for image processing has been renamed `processed_images` to avoid issues with GitHub Pages

25
Cargo.lock generated
View file

@ -48,8 +48,8 @@ dependencies = [
"tokio-tcp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
"tower-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"trust-dns-proto 0.5.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)",
"trust-dns-resolver 0.10.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)",
"trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"trust-dns-resolver 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -638,7 +638,7 @@ name = "errors"
version = "0.1.0"
dependencies = [
"error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"image 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
"image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)",
"syntect 3.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tera 0.11.19 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
@ -719,6 +719,7 @@ dependencies = [
"serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"tera 0.11.19 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"utils 0.1.0",
]
[[package]]
@ -945,7 +946,7 @@ dependencies = [
[[package]]
name = "image"
version = "0.20.0"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
@ -965,7 +966,7 @@ name = "imageproc"
version = "0.1.0"
dependencies = [
"errors 0.1.0",
"image 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
"image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rayon 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2518,7 +2519,7 @@ dependencies = [
[[package]]
name = "trust-dns-proto"
version = "0.5.0-alpha.5"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2559,7 +2560,7 @@ dependencies = [
[[package]]
name = "trust-dns-resolver"
version = "0.10.0-alpha.3"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2572,7 +2573,7 @@ dependencies = [
"resolv-conf 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
"trust-dns-proto 0.5.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)",
"trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -2717,8 +2718,10 @@ name = "utils"
version = "0.1.0"
dependencies = [
"errors 0.1.0",
"serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
"tera 0.11.19 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"walkdir 2.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -3000,7 +3003,7 @@ dependencies = [
"checksum hyper 0.12.13 (registry+https://github.com/rust-lang/crates.io-index)" = "95ffee0d1d30de4313fdaaa485891ce924991d45bbc18adfc8ac5b1639e62fbb"
"checksum hyper-tls 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "32cd73f14ad370d3b4d4b7dce08f69b81536c82e39fcc89731930fe5788cd661"
"checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e"
"checksum image 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60710fd3cb40c2434451d8d5147bcf39bbb68aae0741041133e09439cb2401e3"
"checksum image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44665b4395d1844c96e7dc8ed5754782a1cdfd9ef458a80bbe45702681450504"
"checksum indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d"
"checksum inflate 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6f53b811ee8e2057ccf9643ca6b4277de90efaf5e61e55fd5254576926bb4245"
"checksum inotify 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "40b54539f3910d6f84fbf9a643efd6e3aa6e4f001426c0329576128255994718"
@ -3154,8 +3157,8 @@ dependencies = [
"checksum toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "4a2ecc31b0351ea18b3fe11274b8db6e4d82bce861bbb22e6dbed40417902c65"
"checksum tower-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b32f72af77f1bfe3d3d4da8516a238ebe7039b51dd8637a09841ac7f16d2c987"
"checksum trust-dns-proto 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f1525ca4e26f5a09d81b79584f19225e7dba5606ae3a416311c2751c5cea60bb"
"checksum trust-dns-proto 0.5.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3daf083dfce3c3554e82b768078331864df1ab220e6c3c1dbb4c847173d33ff6"
"checksum trust-dns-resolver 0.10.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "476d293db1c1027727a1683b681550a24207277e96bb1fb763053e1164621971"
"checksum trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0838272e89f1c693b4df38dc353412e389cf548ceed6f9fd1af5a8d6e0e7cf74"
"checksum trust-dns-resolver 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4e913a5df94658858e548cc95a3212797ee524e487ede091c32f27ca26e11620"
"checksum trust-dns-resolver 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4a821ad51a29816420b8cac4b026756b81c023630b97eaa4c8090637ee3508bd"
"checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169"

View file

@ -13,3 +13,4 @@ regex = "1"
lazy_static = "1"
errors = { path = "../errors" }
utils = { path = "../utils" }

View file

@ -10,6 +10,7 @@ extern crate toml;
#[macro_use]
extern crate errors;
extern crate utils;
use errors::{Result, ResultExt};
use regex::Regex;

View file

@ -1,66 +1,11 @@
use std::collections::HashMap;
use std::result::Result as StdResult;
use chrono::prelude::*;
use serde::{Deserialize, Deserializer};
use tera::{Map, Value};
use toml;
use errors::Result;
fn from_toml_datetime<'de, D>(deserializer: D) -> StdResult<Option<String>, D::Error>
where
D: Deserializer<'de>,
{
toml::value::Datetime::deserialize(deserializer).map(|s| Some(s.to_string()))
}
/// Returns key/value for a converted date from TOML.
/// If the table itself is the TOML struct, only return its value without the key
fn convert_toml_date(table: Map<String, Value>) -> Value {
let mut new = Map::new();
for (k, v) in table {
if k == "$__toml_private_datetime" {
return v;
}
match v {
Value::Object(mut o) => {
// that was a toml datetime object, just return the date
if let Some(toml_date) = o.remove("$__toml_private_datetime") {
new.insert(k, toml_date);
return Value::Object(new);
}
new.insert(k, convert_toml_date(o));
}
_ => {
new.insert(k, v);
}
}
}
Value::Object(new)
}
/// TOML datetimes will be serialized as a struct but we want the
/// stringified version for json, otherwise they are going to be weird
fn fix_toml_dates(table: Map<String, Value>) -> Value {
let mut new = Map::new();
for (key, value) in table {
match value {
Value::Object(mut o) => {
new.insert(key, convert_toml_date(o));
}
_ => {
new.insert(key, value);
}
}
}
Value::Object(new)
}
use utils::de::fix_toml_dates;
/// The front matter of every page
#[derive(Debug, Clone, PartialEq, Deserialize)]
@ -71,7 +16,7 @@ pub struct PageFrontMatter {
/// Description in <meta> that appears when linked, e.g. on twitter
pub description: Option<String>,
/// Date if we want to order pages (ie blog post)
#[serde(default, deserialize_with = "from_toml_datetime")]
#[serde(default, deserialize_with = "utils::de::from_toml_datetime")]
pub date: Option<String>,
/// Chrono converted datetime
#[serde(default, skip_deserializing)]

View file

@ -1,6 +1,7 @@
extern crate serde_json;
extern crate toml;
use utils::de::fix_toml_dates;
use utils::fs::{get_file_time, is_path_in_directory, read_file};
use reqwest::{header, Client};
@ -236,7 +237,12 @@ fn load_json(json_data: String) -> Result<Value> {
/// Parse a TOML string and convert it to a Tera Value
fn load_toml(toml_data: String) -> Result<Value> {
let toml_content: toml::Value = toml::from_str(&toml_data).map_err(|e| format!("{:?}", e))?;
to_value(toml_content).map_err(|e| e.into())
let toml_value = to_value(toml_content).expect("Got invalid JSON that was valid TOML somehow");
match toml_value {
Value::Object(m) => Ok(fix_toml_dates(m)),
_ => unreachable!("Loaded something other than a TOML object"),
}
}
/// Parse a CSV string and convert it to a Tera Value
@ -409,15 +415,12 @@ mod tests {
args.insert("path".to_string(), to_value("test.toml").unwrap());
let result = static_fn(args.clone()).unwrap();
//TOML does not load in order, and also dates are not returned as strings, but
//rather as another object with a key and value
//TOML does not load in order
assert_eq!(
result,
json!({
"category": {
"date": {
"$__toml_private_datetime": "1979-05-27T07:32:00Z"
},
"date": "1979-05-27T07:32:00Z",
"key": "value"
},
})

View file

@ -8,6 +8,8 @@ errors = { path = "../errors" }
tera = "0.11"
unicode-segmentation = "1.2"
walkdir = "2"
toml = "0.4"
serde = "1"
[dev-dependencies]
tempfile = "3"

View file

@ -0,0 +1,53 @@
use serde::{Deserialize, Deserializer};
use tera::{Map, Value};
use toml;
/// Used as an attribute when we want to convert from TOML to a string date
pub fn from_toml_datetime<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
where
D: Deserializer<'de>,
{
toml::value::Datetime::deserialize(deserializer).map(|s| Some(s.to_string()))
}
/// Returns key/value for a converted date from TOML.
/// If the table itself is the TOML struct, only return its value without the key
fn convert_toml_date(table: Map<String, Value>) -> Value {
let mut new = Map::new();
for (k, v) in table {
if k == "$__toml_private_datetime" {
return v;
}
match v {
Value::Object(o) => {
new.insert(k, convert_toml_date(o));
}
_ => {
new.insert(k, v);
}
}
}
Value::Object(new)
}
/// TOML datetimes will be serialized as a struct but we want the
/// stringified version for json, otherwise they are going to be weird
pub fn fix_toml_dates(table: Map<String, Value>) -> Value {
let mut new = Map::new();
for (key, value) in table {
match value {
Value::Object(mut o) => {
new.insert(key, convert_toml_date(o));
}
_ => {
new.insert(key, value);
}
}
}
Value::Object(new)
}

View file

@ -1,12 +1,15 @@
#[macro_use]
extern crate errors;
extern crate serde;
#[cfg(test)]
extern crate tempfile;
extern crate tera;
extern crate toml;
extern crate unicode_segmentation;
extern crate walkdir;
pub mod de;
pub mod fs;
pub mod net;
pub mod site;