From 36ec33f0429a9f85247a6f76cf4bf176270285d7 Mon Sep 17 00:00:00 2001 From: Hannu Hartikainen Date: Sat, 23 May 2020 12:46:50 +0300 Subject: [PATCH] Change get_url(cachebust=true) to use a hash (#1032) Cache-busting was previously done with a compile-time timestamp. Change to the SHA-256 hash of the file to avoid refreshing unchanged files. The implementation could be used to add a new global fn (say, get_file_hash) for subresource integrity use, but that's for another commit. Fixes #519. Co-authored-by: Vincent Prouillet --- Cargo.lock | 13 ++++ components/site/src/lib.rs | 2 +- components/templates/Cargo.toml | 1 + components/templates/src/global_fns/mod.rs | 66 +++++++++++++++---- .../documentation/templates/overview.md | 2 +- 5 files changed, 70 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 781b2a13..1fd31fa6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2251,6 +2251,17 @@ dependencies = [ "opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "sha2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "shlex" version = "0.1.1" @@ -2430,6 +2441,7 @@ dependencies = [ "pulldown-cmark 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "tera 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3210,6 +3222,7 @@ dependencies = [ "checksum serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)" = "993948e75b189211a9b31a7528f950c6adc21f9720b6438ff80a7fa2f864cea2" "checksum serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" "checksum sha-1 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +"checksum sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0" "checksum shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" "checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" "checksum siphasher 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index c52f685b..ee0cc89d 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -533,7 +533,7 @@ impl Site { pub fn register_early_global_fns(&mut self) { self.tera.register_function( "get_url", - global_fns::GetUrl::new(self.config.clone(), self.permalinks.clone()), + global_fns::GetUrl::new(self.config.clone(), self.permalinks.clone(), self.content_path.clone()), ); self.tera.register_function( "resize_image", diff --git a/components/templates/Cargo.toml b/components/templates/Cargo.toml index e7f2fa2e..24189765 100644 --- a/components/templates/Cargo.toml +++ b/components/templates/Cargo.toml @@ -13,6 +13,7 @@ toml = "0.5" csv = "1" image = "0.23" serde_json = "1.0" +sha2 = "0.8" url = "2" errors = { path = "../errors" } diff --git a/components/templates/src/global_fns/mod.rs b/components/templates/src/global_fns/mod.rs index 89fd9054..4535d8b7 100644 --- a/components/templates/src/global_fns/mod.rs +++ b/components/templates/src/global_fns/mod.rs @@ -1,7 +1,9 @@ use std::collections::HashMap; use std::path::PathBuf; use std::sync::{Arc, Mutex, RwLock}; +use std::{fs, io, result}; +use sha2::{Digest, Sha256}; use tera::{from_value, to_value, Error, Function as TeraFn, Result, Value}; use config::Config; @@ -47,10 +49,11 @@ impl TeraFn for Trans { pub struct GetUrl { config: Config, permalinks: HashMap, + content_path: PathBuf, } impl GetUrl { - pub fn new(config: Config, permalinks: HashMap) -> Self { - Self { config, permalinks } + pub fn new(config: Config, permalinks: HashMap, content_path: PathBuf) -> Self { + Self { config, permalinks, content_path } } } @@ -71,6 +74,13 @@ fn make_path_with_lang(path: String, lang: &str, config: &Config) -> Result result::Result { + let mut file = fs::File::open(path)?; + let mut hasher = Sha256::new(); + io::copy(&mut file, &mut hasher)?; + Ok(format!("{:x}", hasher.result())) +} + impl TeraFn for GetUrl { fn call(&self, args: &HashMap) -> Result { let cachebust = @@ -110,7 +120,11 @@ impl TeraFn for GetUrl { } if cachebust { - permalink = format!("{}?t={}", permalink, self.config.build_timestamp.unwrap()); + let full_path = self.content_path.join(&path); + permalink = match compute_file_sha256(&full_path) { + Ok(digest) => format!("{}?h={}", permalink, digest), + Err(_) => return Err(format!("Could not read file `{}`. Expected location: {}", path, full_path.to_str().unwrap()).into()), + }; } Ok(to_value(permalink).unwrap()) } @@ -368,28 +382,56 @@ mod tests { use super::{GetTaxonomy, GetTaxonomyUrl, GetUrl, Trans}; use std::collections::HashMap; + use std::env::temp_dir; + use std::fs::remove_dir_all; + use std::path::PathBuf; use std::sync::{Arc, RwLock}; + use lazy_static::lazy_static; + use tera::{to_value, Function, Value}; use config::{Config, Taxonomy as TaxonomyConfig}; use library::{Library, Taxonomy, TaxonomyItem}; + use utils::fs::{create_directory, create_file}; use utils::slugs::SlugifyStrategy; + struct TestContext { + content_path: PathBuf, + } + impl TestContext { + fn setup() -> Self { + let dir = temp_dir().join("test_global_fns"); + create_directory(&dir).expect("Could not create test directory"); + create_file(&dir.join("app.css"), "// Hello world!") + .expect("Could not create test content (app.css)"); + Self { content_path: dir } + } + } + impl Drop for TestContext { + fn drop(&mut self) { + remove_dir_all(&self.content_path).expect("Could not free test directory"); + } + } + + lazy_static! { + static ref TEST_CONTEXT: TestContext = TestContext::setup(); + } + #[test] fn can_add_cachebust_to_url() { let config = Config::default(); - let static_fn = GetUrl::new(config, HashMap::new()); + let static_fn = GetUrl::new(config, HashMap::new(), TEST_CONTEXT.content_path.clone()); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("app.css").unwrap()); args.insert("cachebust".to_string(), to_value(true).unwrap()); - assert_eq!(static_fn.call(&args).unwrap(), "http://a-website.com/app.css?t=1"); + assert_eq!(static_fn.call(&args).unwrap(), "http://a-website.com/app.css?h=572e691dc68c3fcd653ae463261bdb38f35dc6f01715d9ce68799319dd158840"); } #[test] fn can_add_trailing_slashes() { let config = Config::default(); - let static_fn = GetUrl::new(config, HashMap::new()); + let static_fn = GetUrl::new(config, HashMap::new(), TEST_CONTEXT.content_path.clone()); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("app.css").unwrap()); args.insert("trailing_slash".to_string(), to_value(true).unwrap()); @@ -399,18 +441,18 @@ mod tests { #[test] fn can_add_slashes_and_cachebust() { let config = Config::default(); - let static_fn = GetUrl::new(config, HashMap::new()); + let static_fn = GetUrl::new(config, HashMap::new(), TEST_CONTEXT.content_path.clone()); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("app.css").unwrap()); args.insert("trailing_slash".to_string(), to_value(true).unwrap()); args.insert("cachebust".to_string(), to_value(true).unwrap()); - assert_eq!(static_fn.call(&args).unwrap(), "http://a-website.com/app.css/?t=1"); + assert_eq!(static_fn.call(&args).unwrap(), "http://a-website.com/app.css/?h=572e691dc68c3fcd653ae463261bdb38f35dc6f01715d9ce68799319dd158840"); } #[test] fn can_link_to_some_static_file() { let config = Config::default(); - let static_fn = GetUrl::new(config, HashMap::new()); + let static_fn = GetUrl::new(config, HashMap::new(), TEST_CONTEXT.content_path.clone()); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("app.css").unwrap()); assert_eq!(static_fn.call(&args).unwrap(), "http://a-website.com/app.css"); @@ -597,7 +639,7 @@ title = "A title" #[test] fn error_when_language_not_available() { let config = Config::parse(TRANS_CONFIG).unwrap(); - let static_fn = GetUrl::new(config, HashMap::new()); + let static_fn = GetUrl::new(config, HashMap::new(), TEST_CONTEXT.content_path.clone()); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("@/a_section/a_page.md").unwrap()); args.insert("lang".to_string(), to_value("it").unwrap()); @@ -620,7 +662,7 @@ title = "A title" "a_section/a_page.en.md".to_string(), "https://remplace-par-ton-url.fr/en/a_section/a_page/".to_string(), ); - let static_fn = GetUrl::new(config, permalinks); + let static_fn = GetUrl::new(config, permalinks, TEST_CONTEXT.content_path.clone()); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("@/a_section/a_page.md").unwrap()); args.insert("lang".to_string(), to_value("fr").unwrap()); @@ -642,7 +684,7 @@ title = "A title" "a_section/a_page.en.md".to_string(), "https://remplace-par-ton-url.fr/en/a_section/a_page/".to_string(), ); - let static_fn = GetUrl::new(config, permalinks); + let static_fn = GetUrl::new(config, permalinks, TEST_CONTEXT.content_path.clone()); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("@/a_section/a_page.md").unwrap()); args.insert("lang".to_string(), to_value("en").unwrap()); diff --git a/docs/content/documentation/templates/overview.md b/docs/content/documentation/templates/overview.md index 4f64e63c..e9c967a7 100644 --- a/docs/content/documentation/templates/overview.md +++ b/docs/content/documentation/templates/overview.md @@ -142,7 +142,7 @@ An example is: {{/* get_url(path="css/app.css", trailing_slash=true) */}} ``` -In the case of non-internal links, you can also add a cachebust of the format `?t=1290192` at the end of a URL +In the case of non-internal links, you can also add a cachebust of the format `?h=` at the end of a URL by passing `cachebust=true` to the `get_url` function.