From 1bf5cd7bf83e6aaaa5ed7253a4c793471c73d41f Mon Sep 17 00:00:00 2001 From: Vincent Prouillet Date: Fri, 11 Jun 2021 21:14:45 +0200 Subject: [PATCH] Update docs + sandbox files --- CHANGELOG.md | 9 +- components/templates/src/global_fns/files.rs | 29 +-- .../templates/src/global_fns/helpers.rs | 15 +- components/templates/src/global_fns/images.rs | 11 +- .../templates/src/global_fns/load_data.rs | 60 +++--- .../documentation/templates/overview.md | 175 ++++++++++-------- 6 files changed, 169 insertions(+), 130 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80ec6cce..c71244cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,14 +6,19 @@ - Newlines are now required after the closing `+++` of front-matter - `resize_image` now returns an object: `{url, static_path}` instead of just the URL so you can follow up with other functions on the new file if needed -- `get_file_hash` now has the `base64` option set to `true` by default (from `false`) since it's mainly used for integrity hashes which are base64 -- `get_url` does not automatically strip leading `/` from paths anymore +- `get_file_hash` now has the `base64` option set to `true` by default (from `false`) since it's mainly used for integrity hashes which are base64 - i18n rework: languages now have their sections in `config.toml` to set up all their options 1. taxonomies don't have a `lang` anymore in the config, you need to declare them in their respective language section 2. the `config` variable in templates has been changed and is now a stripped down language aware version of the previous `config` object 3. Search settings are now language specific 4. Translations are now nested in the languages table +- Paths unification: + 1. `get_url` does not load automatically from the `static` folder anymore + 2. New path resolving logic for all on-disk files: replace `@/` by `content/`, trim leading `/` and + search in $BASE_DIR + $path, $BASE_DIR + static + $path and $BASE_DIR + content + $path + 3. `get_file_hash` now returns base64 encoded hash by default + 4. all functions working on files can now only load files in the Zola directory ### Other diff --git a/components/templates/src/global_fns/files.rs b/components/templates/src/global_fns/files.rs index a379b044..0fefe229 100644 --- a/components/templates/src/global_fns/files.rs +++ b/components/templates/src/global_fns/files.rs @@ -19,7 +19,6 @@ where { let mut hasher = D::new(); io::copy(&mut file, &mut hasher)?; - println!("base64: {}", as_base64); if as_base64 { Ok(encode_b64(hasher.finalize())) } else { @@ -79,9 +78,6 @@ impl TeraFn for GetUrl { let lang = optional_arg!(String, args.get("lang"), "`get_url`: `lang` must be a string.") .unwrap_or_else(|| self.config.default_language.clone()); - // TODO: handle rss files with langs - // https://zola.discourse.group/t/rss-and-languages-do-not-work/878 - // TODO: clean up everything // if it starts with @/, resolve it as an internal link if path.starts_with("@/") { let path_with_lang = match make_path_with_lang(path, &lang, &self.config) { @@ -91,10 +87,11 @@ impl TeraFn for GetUrl { match resolve_internal_link(&path_with_lang, &self.permalinks) { Ok(resolved) => Ok(to_value(resolved.permalink).unwrap()), - Err(_) => { - Err(format!("Could not resolve URL for link `{}` not found.", path_with_lang) - .into()) - } + Err(_) => Err(format!( + "`get_url`: could not resolve URL for link `{}` not found.", + path_with_lang + ) + .into()), } } else { // anything else @@ -115,6 +112,7 @@ impl TeraFn for GetUrl { if cachebust { match search_for_file(&self.base_path, &path_with_lang) + .map_err(|e| format!("`get_url`: {}", e))? .and_then(|(p, _)| fs::File::open(&p).ok()) .and_then(|f| compute_file_hash::(f, false).ok()) { @@ -122,7 +120,11 @@ impl TeraFn for GetUrl { permalink = format!("{}?h={}", permalink, hash); } None => { - return Err(format!("Could not find or open file {}", path_with_lang).into()) + return Err(format!( + "`get_url`: Could not find or open file {}", + path_with_lang + ) + .into()) } }; } @@ -166,7 +168,9 @@ impl TeraFn for GetFileHash { ) .unwrap_or(true); - let file_path = match search_for_file(&self.base_path, &path) { + let file_path = match search_for_file(&self.base_path, &path) + .map_err(|e| format!("`get_file_hash`: {}", e))? + { Some((f, _)) => f, None => { return Err(format!("`get_file_hash`: Cannot find file: {}", path).into()); @@ -262,6 +266,10 @@ title = "A title" let mut args = HashMap::new(); args.insert("path".to_string(), to_value("app.css").unwrap()); assert_eq!(static_fn.call(&args).unwrap(), "http://a-website.com/app.css"); + + let mut args = HashMap::new(); + args.insert("path".to_string(), to_value("/app.css").unwrap()); + assert_eq!(static_fn.call(&args).unwrap(), "http://a-website.com/app.css"); } #[test] @@ -431,7 +439,6 @@ title = "A title" let mut args = HashMap::new(); args.insert("path".to_string(), to_value("doesnt-exist").unwrap()); let err = format!("{}", static_fn.call(&args).unwrap_err()); - println!("{:?}", err); assert!(err.contains("Cannot find file")); } diff --git a/components/templates/src/global_fns/helpers.rs b/components/templates/src/global_fns/helpers.rs index 1a1f2a85..f835ebef 100644 --- a/components/templates/src/global_fns/helpers.rs +++ b/components/templates/src/global_fns/helpers.rs @@ -1,6 +1,9 @@ use std::borrow::Cow; use std::path::{Path, PathBuf}; +use errors::{bail, Result}; +use utils::fs::is_path_in_directory; + /// This is used by a few Tera functions to search for files on the filesystem. /// This does try to find the file in 3 different spots: /// 1. base_path + path @@ -9,16 +12,22 @@ use std::path::{Path, PathBuf}; /// A path starting with @/ will replace it with `content/` and a path starting with `/` will have /// it removed. /// It also returns the unified path so it can be used as unique hash for a given file. -pub fn search_for_file(base_path: &Path, path: &str) -> Option<(PathBuf, String)> { +/// It will error if the file is not contained in the Zola directory. +pub fn search_for_file(base_path: &Path, path: &str) -> Result> { let search_paths = [base_path.join("static"), base_path.join("content")]; let actual_path = if path.starts_with("@/") { Cow::Owned(path.replace("@/", "content/")) } else { Cow::Borrowed(path.trim_start_matches('/')) }; + let mut file_path = base_path.join(&*actual_path); let mut file_exists = file_path.exists(); + if file_exists && !is_path_in_directory(base_path, &file_path)? { + bail!("{:?} is not inside the base site directory {:?}", path, base_path); + } + if !file_exists { // we need to search in both search folders now for dir in &search_paths { @@ -32,8 +41,8 @@ pub fn search_for_file(base_path: &Path, path: &str) -> Option<(PathBuf, String) } if file_exists { - Some((file_path, actual_path.into_owned())) + Ok(Some((file_path, actual_path.into_owned()))) } else { - None + Ok(None) } } diff --git a/components/templates/src/global_fns/images.rs b/components/templates/src/global_fns/images.rs index 0cdd87e3..9695ea92 100644 --- a/components/templates/src/global_fns/images.rs +++ b/components/templates/src/global_fns/images.rs @@ -55,7 +55,9 @@ impl TeraFn for ResizeImage { } let mut imageproc = self.imageproc.lock().unwrap(); - let (file_path, unified_path) = match search_for_file(&self.base_path, &path) { + let (file_path, unified_path) = match search_for_file(&self.base_path, &path) + .map_err(|e| format!("`resize_image`: {}", e))? + { Some(f) => f, None => { return Err(format!("`resize_image`: Cannot find file: {}", path).into()); @@ -95,14 +97,15 @@ impl TeraFn for GetImageMetadata { "`get_image_metadata`: `allow_missing` must be a boolean (true or false)" ) .unwrap_or(false); - let (src_path, _) = match search_for_file(&self.base_path, &path) { + let (src_path, _) = match search_for_file(&self.base_path, &path) + .map_err(|e| format!("`get_image_metadata`: {}", e))? + { Some((f, p)) => (f, p), None => { if allow_missing { - println!("Image at path {} could not be found or loaded", path); return Ok(Value::Null); } - return Err(format!("`resize_image`: Cannot find path: {}", path).into()); + return Err(format!("`get_image_metadata`: Cannot find path: {}", path).into()); } }; diff --git a/components/templates/src/global_fns/load_data.rs b/components/templates/src/global_fns/load_data.rs index 5478fd7a..bb6f238c 100644 --- a/components/templates/src/global_fns/load_data.rs +++ b/components/templates/src/global_fns/load_data.rs @@ -11,7 +11,7 @@ use reqwest::{blocking::Client, header}; use tera::{from_value, to_value, Error, Function as TeraFn, Map, Result, Value}; use url::Url; use utils::de::fix_toml_dates; -use utils::fs::{get_file_time, is_path_in_directory, read_file}; +use utils::fs::{get_file_time, read_file}; use crate::global_fns::helpers::search_for_file; @@ -94,7 +94,9 @@ impl DataSource { } if let Some(path) = path_arg { - return match search_for_file(&base_path, &path) { + return match search_for_file(&base_path, &path) + .map_err(|e| format!("`load_data`: {}", e))? + { Some((f, _)) => Ok(Some(DataSource::Path(f))), None => Ok(None), }; @@ -104,7 +106,7 @@ impl DataSource { return Url::parse(&url) .map(DataSource::Url) .map(Some) - .map_err(|e| format!("Failed to parse {} as url: {}", url, e).into()); + .map_err(|e| format!("`load_data`: Failed to parse {} as url: {}", url, e).into()); } Err(GET_DATA_ARGUMENT_ERROR_MESSAGE.into()) @@ -139,21 +141,6 @@ impl Hash for DataSource { } } -fn read_local_data_file(base_path: &Path, full_path: PathBuf) -> Result { - if !is_path_in_directory(&base_path, &full_path) - .map_err(|e| format!("Failed to read data file {}: {}", full_path.display(), e))? - { - return Err(format!( - "{:?} is not inside the base site directory {:?}", - full_path, base_path - ) - .into()); - } - - read_file(&full_path) - .map_err(|e| format!("`load_data`: error reading file {:?}: {}", full_path, e).into()) -} - fn get_output_format_from_args( format_arg: Option, data_source: &DataSource, @@ -235,21 +222,24 @@ impl TeraFn for LoadData { // If the file doesn't exist, source is None let data_source = - match (DataSource::from_args(path_arg.clone(), url_arg, &self.base_path)?, required) { + match (DataSource::from_args(path_arg.clone(), url_arg, &self.base_path), required) { // If the file was not required, return a Null value to the template - (None, false) => { + (Ok(None), false) | (Err(_), false) => { return Ok(Value::Null); } + (Err(e), true) => { + return Err(e); + } // If the file was required, error - (None, true) => { + (Ok(None), true) => { // source is None only with path_arg (not URL), so path_arg is safely unwrap return Err(format!( - "{} doesn't exist", + "`load_data`: {} doesn't exist", &self.base_path.join(path_arg.unwrap()).display() ) .into()); } - (Some(data_source), _) => data_source, + (Ok(Some(data_source)), _) => data_source, }; let file_format = get_output_format_from_args(format_arg, &data_source)?; @@ -262,7 +252,8 @@ impl TeraFn for LoadData { } let data = match data_source { - DataSource::Path(path) => read_local_data_file(&self.base_path, path), + DataSource::Path(path) => read_file(&path) + .map_err(|e| format!("`load_data`: error reading file {:?}: {}", path, e)), DataSource::Url(url) => { let response_client = self.client.lock().expect("response client lock"); let req = match method { @@ -280,7 +271,7 @@ impl TeraFn for LoadData { } Err(_) => { return Err(format!( - "{} is an illegal content type", + "`load_data`: {} is an illegal content type", &content_type ) .into()); @@ -296,7 +287,7 @@ impl TeraFn for LoadData { match req.send().and_then(|res| res.error_for_status()) { Ok(r) => r.text().map_err(|e| { - format!("Failed to parse response from {}: {:?}", url, e).into() + format!("`load_data`: Failed to parse response from {}: {:?}", url, e) }), Err(e) => { if !required { @@ -305,10 +296,14 @@ impl TeraFn for LoadData { return Ok(Value::Null); } Err(match e.status() { - Some(status) => format!("Failed to request {}: {}", url, status), - None => format!("Could not get response status for url: {}", url), - } - .into()) + Some(status) => { + format!("`load_data`: Failed to request {}: {}", url, status) + } + None => format!( + "`load_data`: Could not get response status for url: {}", + url + ), + }) } } } @@ -634,13 +629,14 @@ mod tests { } #[test] - fn cannot_load_outside_content_dir() { + fn cannot_load_outside_base_dir() { let static_fn = LoadData::new(PathBuf::from(PathBuf::from("../utils"))); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("../../README.md").unwrap()); args.insert("format".to_string(), to_value("plain").unwrap()); let result = static_fn.call(&args); assert!(result.is_err()); + println!("{:?} {:?}", std::env::current_dir(), result); assert!(result.unwrap_err().to_string().contains("is not inside the base site directory")); } @@ -736,7 +732,7 @@ mod tests { assert!(result.is_err()); assert_eq!( result.unwrap_err().to_string(), - format!("Failed to request {}: 404 Not Found", url) + format!("`load_data`: Failed to request {}: 404 Not Found", url) ); } diff --git a/docs/content/documentation/templates/overview.md b/docs/content/documentation/templates/overview.md index db17018d..ef3e3a1d 100644 --- a/docs/content/documentation/templates/overview.md +++ b/docs/content/documentation/templates/overview.md @@ -82,21 +82,36 @@ Encode the variable to base64. Decode the variable from base64. -## Built-in global functions +## Built-in functions -Zola adds a few global functions to [those in Tera](https://tera.netlify.com/docs#built-in-functions) +Zola adds a few Tera functions to [those built-in in Tera](https://tera.netlify.com/docs#built-in-functions) to make it easier to develop complex sites. +### File searching logic +For functions that are searching for a file on disk other than through `get_page` and `get_section`, the following +logic applies. + +1. The base directory is the Zola root directory, where the `config.toml` is +2. For the given path: if it starts with `@/`, replace that with `content/` instead and trim any leading `/` +3. Search in the following 3 locations in this order, returning the first where the file exists: + a. $base_directory + $path + b. $base_directory + "static/" + $path + c. $base_directory + "content/" + $path + +In practice this means that `@/some/image.jpg`, `/content/some/image.jpg` and `content/some/image.jpg` will point to the +same thing. + +It will error if the path is outside the Zola directory. ### `get_page` -Takes a path to an `.md` file and returns the associated page. +Takes a path to an `.md` file and returns the associated page. The base path is the `content` directory. ```jinja2 {% set page = get_page(path="blog/page2.md") %} ``` ### `get_section` -Takes a path to an `_index.md` file and returns the associated section. +Takes a path to an `_index.md` file and returns the associated section. The base path is the `content` directory. ```jinja2 {% set section = get_section(path="blog/_index.md") %} @@ -108,78 +123,6 @@ If you only need the metadata of the section, you can pass `metadata_only=true` {% set section = get_section(path="blog/_index.md", metadata_only=true) %} ``` -### `get_url` -Gets the permalink for the given path. -If the path starts with `@/`, it will be treated as an internal -link like the ones used in Markdown, starting from the root `content` directory. - -```jinja2 -{% set url = get_url(path="@/blog/_index.md") %} -``` - -It accepts an optional parameter `lang` in order to compute a *language-aware URL* in multilingual websites. Assuming `config.base_url` is `"http://example.com"`, the following snippet will: - -- return `"http://example.com/blog/"` if `config.default_language` is `"en"` -- return `"http://example.com/en/blog/"` if `config.default_language` is **not** `"en"` and `"en"` appears in `config.languages` -- fail otherwise, with the error message `"'en' is not an authorized language (check config.languages)."` - -```jinja2 -{% set url = get_url(path="@/blog/_index.md", lang="en") %} -``` - -This can also be used to get the permalinks for static assets, for example if -we want to link to the file that is located at `static/css/app.css`: - -```jinja2 -{{/* get_url(path="css/app.css") */}} -``` - -By default, assets will not have a trailing slash. You can force one by passing `trailing_slash=true` to the `get_url` function. -An example is: - -```jinja2 -{{/* get_url(path="css/app.css", trailing_slash=true) */}} -``` - -In the case of non-internal links, you can also add a cachebust of the format `?h=` at the end of a URL -by passing `cachebust=true` to the `get_url` function. - -### `get_file_hash` - -Returns the hash digest of a static file. Supported hashing algorithms are -SHA-256, SHA-384 (default) and SHA-512. Requires `path`. The `sha_type` -parameter is optional and must be one of 256, 384 or 512. - -```jinja2 -{{/* get_file_hash(path="js/app.js", sha_type=256) */}} -``` - -The function can also output a base64-encoded hash value when its `base64` -parameter is set to `true`. This can be used to implement [subresource -integrity](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity). - -```jinja2 - -``` - -Do note that subresource integrity is typically used when using external -scripts, which `get_file_hash` does not support. - -Whenever hashing files, whether using `get_file_hash` or `get_url(..., -cachebust=true)`, the file is searched for in three places: `static/`, -`content/` and the output path (so e.g. compiled SASS can be hashed, too.) - -### `get_image_metadata` - -Gets metadata for an image. This supports common formats like JPEG, PNG, as well as SVG. -Currently, the only supported keys are `width` and `height`. - -```jinja2 - {% set meta = get_image_metadata(path="...") %} - Our image is {{ meta.width }}x{{ meta.height }} -``` - ### `get_taxonomy_url` Gets the permalink for the taxonomy item found. @@ -208,12 +151,88 @@ items: Array; See the [Taxonomies documentation](@/documentation/templates/taxonomies.md) for a full documentation of those types. +### `get_url` +Gets the permalink for the given path. +If the path starts with `@/`, it will be treated as an internal link like the ones used in Markdown, +starting from the root `content` directory as well as validated. + +```jinja2 +{% set url = get_url(path="@/blog/_index.md") %} +``` + +It accepts an optional parameter `lang` in order to compute a *language-aware URL* in multilingual websites. Assuming `config.base_url` is `"http://example.com"`, the following snippet will: + +- return `"http://example.com/blog/"` if `config.default_language` is `"en"` +- return `"http://example.com/en/blog/"` if `config.default_language` is **not** `"en"` and `"en"` appears in `config.languages` +- fail otherwise, with the error message `"'en' is not an authorized language (check config.languages)."` + +```jinja2 +{% set url = get_url(path="@/blog/_index.md", lang="en") %} +``` + +This can also be used to get the permalinks for static assets, for example if +we want to link to the file that is located at `static/css/app.css`: + +```jinja2 +{{/* get_url(path="static/css/app.css") */}} +``` + +By default, assets will not have a trailing slash. You can force one by passing `trailing_slash=true` to the `get_url` function. +An example is: + +```jinja2 +{{/* get_url(path="static/css/app.css", trailing_slash=true) */}} +``` + +In the case of non-internal links, you can also add a cachebust of the format `?h=` at the end of a URL +by passing `cachebust=true` to the `get_url` function. In this case, the path will need to resolve to an actual file. +See [File Searching Logic](@/documentation/templates/overview.md#file-searching-logic) for details. + +### `get_file_hash` + +Returns the hash digest (SHA-256, SHA-384 or SHA-512) of a file. + +It can take the following arguments: +- `path`: mandatory, see [File Searching Logic](@/documentation/templates/overview.md#file-searching-logic) for details +- `sha_type`: optional, one of `256`, `384` or `512`, defaults to `384` +- `base64`: optional, `true` or `false`, defaults to `true`. Whether to encode the hash as base64 + +```jinja2 +{{/* get_file_hash(path="static/js/app.js", sha_type=256) */}} +``` + +The function can also output a base64-encoded hash value when its `base64` +parameter is set to `true`. This can be used to implement [subresource +integrity](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity). + +```jinja2 + +``` + +Do note that subresource integrity is typically used when using external scripts, which `get_file_hash` does not support. + +### `get_image_metadata` + +Gets metadata for an image. This supports common formats like JPEG, PNG, WebP, BMP, GIF as well as SVG. + +It can take the following arguments: + +- `path`: mandatory, see [File Searching Logic](@/documentation/templates/overview.md#file-searching-logic) for details +- `allow_missing`: optional, `true` or `false`, defaults to `false`. Whether a missing file should raise an error or not. + +The method returns a map containing `width`, `height` and `format` (the lowercased value as string). + +```jinja2 + {% set meta = get_image_metadata(path="...") %} + Our image (.{{meta.format}}) has format is {{ meta.width }}x{{ meta.height }} +``` + ### `load_data` Loads data from a file or URL. Supported file types include *toml*, *json*, *csv* and *bibtex*. Any other file type will be loaded as plain text. -The `path` argument specifies the path to the data file relative to your base directory, where your `config.toml` is. -As a security precaution, if this file is outside the main site directory, your site will fail to build. +The `path` argument specifies the path to the data file, according to the [File Searching Logic](@/documentation/templates/overview.md#file-searching-logic). ```jinja2 {% set data = load_data(path="content/blog/story/data.toml") %}