Update docs + sandbox files

CHANGELOG.md

@@ -6,14 +6,19 @@
 
 - Newlines are now required after the closing `+++` of front-matter
 - `resize_image` now returns an object: `{url, static_path}` instead of just the URL so you can follow up with other functions on the new file if needed
-- `get_file_hash` now has the `base64` option set to `true` by default (from `false`) since it's mainly used for integrity hashes which are base64  
+- `get_file_hash` now has the `base64` option set to `true` by default (from `false`) since it's mainly used for integrity hashes which are base64
-- `get_url` does not automatically strip leading `/` from paths anymore  
 - i18n rework: languages now have their sections in `config.toml` to set up all their options
   1. taxonomies don't have a `lang` anymore in the config, you need to declare them in their respective language section
   2. the `config` variable in templates has been changed and is now a stripped down language aware version of the previous `config`
   object
   3. Search settings are now language specific
   4. Translations are now nested in the languages table
+- Paths unification: 
+  1. `get_url` does not load automatically from the `static` folder anymore
+  2. New path resolving logic for all on-disk files: replace `@/` by `content/`, trim leading `/` and 
+     search in $BASE_DIR + $path, $BASE_DIR + static + $path and $BASE_DIR + content + $path
+  3. `get_file_hash` now returns base64 encoded hash by default
+  4. all functions working on files can now only load files in the Zola directory
 
 ### Other
 

components/templates/src/global_fns/files.rs

@@ -19,7 +19,6 @@ where
 {
     let mut hasher = D::new();
     io::copy(&mut file, &mut hasher)?;
-    println!("base64: {}", as_base64);
     if as_base64 {
         Ok(encode_b64(hasher.finalize()))
     } else {
@@ -79,9 +78,6 @@ impl TeraFn for GetUrl {
         let lang = optional_arg!(String, args.get("lang"), "`get_url`: `lang` must be a string.")
             .unwrap_or_else(|| self.config.default_language.clone());
 
-        // TODO: handle rss files with langs
-        // https://zola.discourse.group/t/rss-and-languages-do-not-work/878
-        // TODO: clean up everything
         // if it starts with @/, resolve it as an internal link
         if path.starts_with("@/") {
             let path_with_lang = match make_path_with_lang(path, &lang, &self.config) {
@@ -91,10 +87,11 @@ impl TeraFn for GetUrl {
 
             match resolve_internal_link(&path_with_lang, &self.permalinks) {
                 Ok(resolved) => Ok(to_value(resolved.permalink).unwrap()),
-                Err(_) => {
+                Err(_) => Err(format!(
-                    Err(format!("Could not resolve URL for link `{}` not found.", path_with_lang)
+                    "`get_url`: could not resolve URL for link `{}` not found.",
-                        .into())
+                    path_with_lang
-                }
+                )
+                .into()),
             }
         } else {
             // anything else
@@ -115,6 +112,7 @@ impl TeraFn for GetUrl {
 
             if cachebust {
                 match search_for_file(&self.base_path, &path_with_lang)
+                    .map_err(|e| format!("`get_url`: {}", e))?
                     .and_then(|(p, _)| fs::File::open(&p).ok())
                     .and_then(|f| compute_file_hash::<Sha256>(f, false).ok())
                 {
@@ -122,7 +120,11 @@ impl TeraFn for GetUrl {
                         permalink = format!("{}?h={}", permalink, hash);
                     }
                     None => {
-                        return Err(format!("Could not find or open file {}", path_with_lang).into())
+                        return Err(format!(
+                            "`get_url`: Could not find or open file {}",
+                            path_with_lang
+                        )
+                        .into())
                     }
                 };
             }
@@ -166,7 +168,9 @@ impl TeraFn for GetFileHash {
         )
         .unwrap_or(true);
 
-        let file_path = match search_for_file(&self.base_path, &path) {
+        let file_path = match search_for_file(&self.base_path, &path)
+            .map_err(|e| format!("`get_file_hash`: {}", e))?
+        {
             Some((f, _)) => f,
             None => {
                 return Err(format!("`get_file_hash`: Cannot find file: {}", path).into());
@@ -262,6 +266,10 @@ title = "A title"
         let mut args = HashMap::new();
         args.insert("path".to_string(), to_value("app.css").unwrap());
         assert_eq!(static_fn.call(&args).unwrap(), "http://a-website.com/app.css");
+
+        let mut args = HashMap::new();
+        args.insert("path".to_string(), to_value("/app.css").unwrap());
+        assert_eq!(static_fn.call(&args).unwrap(), "http://a-website.com/app.css");
     }
 
     #[test]
@@ -431,7 +439,6 @@ title = "A title"
         let mut args = HashMap::new();
         args.insert("path".to_string(), to_value("doesnt-exist").unwrap());
         let err = format!("{}", static_fn.call(&args).unwrap_err());
-        println!("{:?}", err);
 
         assert!(err.contains("Cannot find file"));
     }

components/templates/src/global_fns/helpers.rs

@@ -1,6 +1,9 @@
 use std::borrow::Cow;
 use std::path::{Path, PathBuf};
 
+use errors::{bail, Result};
+use utils::fs::is_path_in_directory;
+
 /// This is used by a few Tera functions to search for files on the filesystem.
 /// This does try to find the file in 3 different spots:
 /// 1. base_path + path
@@ -9,16 +12,22 @@ use std::path::{Path, PathBuf};
 /// A path starting with @/ will replace it with `content/` and a path starting with `/` will have
 /// it removed.
 /// It also returns the unified path so it can be used as unique hash for a given file.
-pub fn search_for_file(base_path: &Path, path: &str) -> Option<(PathBuf, String)> {
+/// It will error if the file is not contained in the Zola directory.
+pub fn search_for_file(base_path: &Path, path: &str) -> Result<Option<(PathBuf, String)>> {
     let search_paths = [base_path.join("static"), base_path.join("content")];
     let actual_path = if path.starts_with("@/") {
         Cow::Owned(path.replace("@/", "content/"))
     } else {
         Cow::Borrowed(path.trim_start_matches('/'))
     };
+
     let mut file_path = base_path.join(&*actual_path);
     let mut file_exists = file_path.exists();
 
+    if file_exists && !is_path_in_directory(base_path, &file_path)? {
+        bail!("{:?} is not inside the base site directory {:?}", path, base_path);
+    }
+
     if !file_exists {
         // we need to search in both search folders now
         for dir in &search_paths {
@@ -32,8 +41,8 @@ pub fn search_for_file(base_path: &Path, path: &str) -> Option<(PathBuf, String)
     }
 
     if file_exists {
-        Some((file_path, actual_path.into_owned()))
+        Ok(Some((file_path, actual_path.into_owned())))
     } else {
-        None
+        Ok(None)
     }
 }

components/templates/src/global_fns/images.rs

@@ -55,7 +55,9 @@ impl TeraFn for ResizeImage {
         }
 
         let mut imageproc = self.imageproc.lock().unwrap();
-        let (file_path, unified_path) = match search_for_file(&self.base_path, &path) {
+        let (file_path, unified_path) = match search_for_file(&self.base_path, &path)
+            .map_err(|e| format!("`resize_image`: {}", e))?
+        {
             Some(f) => f,
             None => {
                 return Err(format!("`resize_image`: Cannot find file: {}", path).into());
@@ -95,14 +97,15 @@ impl TeraFn for GetImageMetadata {
             "`get_image_metadata`: `allow_missing` must be a boolean (true or false)"
         )
         .unwrap_or(false);
-        let (src_path, _) = match search_for_file(&self.base_path, &path) {
+        let (src_path, _) = match search_for_file(&self.base_path, &path)
+            .map_err(|e| format!("`get_image_metadata`: {}", e))?
+        {
             Some((f, p)) => (f, p),
             None => {
                 if allow_missing {
-                    println!("Image at path {} could not be found or loaded", path);
                     return Ok(Value::Null);
                 }
-                return Err(format!("`resize_image`: Cannot find path: {}", path).into());
+                return Err(format!("`get_image_metadata`: Cannot find path: {}", path).into());
             }
         };
 

components/templates/src/global_fns/load_data.rs

@@ -11,7 +11,7 @@ use reqwest::{blocking::Client, header};
 use tera::{from_value, to_value, Error, Function as TeraFn, Map, Result, Value};
 use url::Url;
 use utils::de::fix_toml_dates;
-use utils::fs::{get_file_time, is_path_in_directory, read_file};
+use utils::fs::{get_file_time, read_file};
 
 use crate::global_fns::helpers::search_for_file;
 
@@ -94,7 +94,9 @@ impl DataSource {
         }
 
         if let Some(path) = path_arg {
-            return match search_for_file(&base_path, &path) {
+            return match search_for_file(&base_path, &path)
+                .map_err(|e| format!("`load_data`: {}", e))?
+            {
                 Some((f, _)) => Ok(Some(DataSource::Path(f))),
                 None => Ok(None),
             };
@@ -104,7 +106,7 @@ impl DataSource {
             return Url::parse(&url)
                 .map(DataSource::Url)
                 .map(Some)
-                .map_err(|e| format!("Failed to parse {} as url: {}", url, e).into());
+                .map_err(|e| format!("`load_data`: Failed to parse {} as url: {}", url, e).into());
         }
 
         Err(GET_DATA_ARGUMENT_ERROR_MESSAGE.into())
@@ -139,21 +141,6 @@ impl Hash for DataSource {
     }
 }
 
-fn read_local_data_file(base_path: &Path, full_path: PathBuf) -> Result<String> {
-    if !is_path_in_directory(&base_path, &full_path)
-        .map_err(|e| format!("Failed to read data file {}: {}", full_path.display(), e))?
-    {
-        return Err(format!(
-            "{:?} is not inside the base site directory {:?}",
-            full_path, base_path
-        )
-        .into());
-    }
-
-    read_file(&full_path)
-        .map_err(|e| format!("`load_data`: error reading file {:?}: {}", full_path, e).into())
-}
-
 fn get_output_format_from_args(
     format_arg: Option<String>,
     data_source: &DataSource,
@@ -235,21 +222,24 @@ impl TeraFn for LoadData {
 
         // If the file doesn't exist, source is None
         let data_source =
-            match (DataSource::from_args(path_arg.clone(), url_arg, &self.base_path)?, required) {
+            match (DataSource::from_args(path_arg.clone(), url_arg, &self.base_path), required) {
                 // If the file was not required, return a Null value to the template
-                (None, false) => {
+                (Ok(None), false) | (Err(_), false) => {
                     return Ok(Value::Null);
                 }
+                (Err(e), true) => {
+                    return Err(e);
+                }
                 // If the file was required, error
-                (None, true) => {
+                (Ok(None), true) => {
                     // source is None only with path_arg (not URL), so path_arg is safely unwrap
                     return Err(format!(
-                        "{} doesn't exist",
+                        "`load_data`: {} doesn't exist",
                         &self.base_path.join(path_arg.unwrap()).display()
                     )
                     .into());
                 }
-                (Some(data_source), _) => data_source,
+                (Ok(Some(data_source)), _) => data_source,
             };
 
         let file_format = get_output_format_from_args(format_arg, &data_source)?;
@@ -262,7 +252,8 @@ impl TeraFn for LoadData {
         }
 
         let data = match data_source {
-            DataSource::Path(path) => read_local_data_file(&self.base_path, path),
+            DataSource::Path(path) => read_file(&path)
+                .map_err(|e| format!("`load_data`: error reading file {:?}: {}", path, e)),
             DataSource::Url(url) => {
                 let response_client = self.client.lock().expect("response client lock");
                 let req = match method {
@@ -280,7 +271,7 @@ impl TeraFn for LoadData {
                                 }
                                 Err(_) => {
                                     return Err(format!(
-                                        "{} is an illegal content type",
+                                        "`load_data`: {} is an illegal content type",
                                         &content_type
                                     )
                                     .into());
@@ -296,7 +287,7 @@ impl TeraFn for LoadData {
 
                 match req.send().and_then(|res| res.error_for_status()) {
                     Ok(r) => r.text().map_err(|e| {
-                        format!("Failed to parse response from {}: {:?}", url, e).into()
+                        format!("`load_data`: Failed to parse response from {}: {:?}", url, e)
                     }),
                     Err(e) => {
                         if !required {
@@ -305,10 +296,14 @@ impl TeraFn for LoadData {
                             return Ok(Value::Null);
                         }
                         Err(match e.status() {
-                            Some(status) => format!("Failed to request {}: {}", url, status),
+                            Some(status) => {
-                            None => format!("Could not get response status for url: {}", url),
+                                format!("`load_data`: Failed to request {}: {}", url, status)
-                        }
+                            }
-                        .into())
+                            None => format!(
+                                "`load_data`: Could not get response status for url: {}",
+                                url
+                            ),
+                        })
                     }
                 }
             }
@@ -634,13 +629,14 @@ mod tests {
     }
 
     #[test]
-    fn cannot_load_outside_content_dir() {
+    fn cannot_load_outside_base_dir() {
         let static_fn = LoadData::new(PathBuf::from(PathBuf::from("../utils")));
         let mut args = HashMap::new();
         args.insert("path".to_string(), to_value("../../README.md").unwrap());
         args.insert("format".to_string(), to_value("plain").unwrap());
         let result = static_fn.call(&args);
         assert!(result.is_err());
+        println!("{:?} {:?}", std::env::current_dir(), result);
         assert!(result.unwrap_err().to_string().contains("is not inside the base site directory"));
     }
 
@@ -736,7 +732,7 @@ mod tests {
         assert!(result.is_err());
         assert_eq!(
             result.unwrap_err().to_string(),
-            format!("Failed to request {}: 404 Not Found", url)
+            format!("`load_data`: Failed to request {}: 404 Not Found", url)
         );
     }
 

docs/content/documentation/templates/overview.md

@@ -82,21 +82,36 @@ Encode the variable to base64.
 Decode the variable from base64.
 
 
-## Built-in global functions
+## Built-in functions
 
-Zola adds a few global functions to [those in Tera](https://tera.netlify.com/docs#built-in-functions)
+Zola adds a few Tera functions to [those built-in in Tera](https://tera.netlify.com/docs#built-in-functions)
 to make it easier to develop complex sites.
 
+### File searching logic
+For functions that are searching for a file on disk other than through `get_page` and `get_section`, the following
+logic applies.
+
+1. The base directory is the Zola root directory, where the `config.toml` is
+2. For the given path: if it starts with `@/`, replace that with `content/` instead and trim any leading `/`
+3. Search in the following 3 locations in this order, returning the first where the file exists:
+   a. $base_directory + $path
+   b. $base_directory + "static/" + $path
+   c. $base_directory + "content/" + $path
+
+In practice this means that `@/some/image.jpg`, `/content/some/image.jpg` and `content/some/image.jpg` will point to the
+same thing.
+
+It will error if the path is outside the Zola directory.
 
 ### `get_page`
-Takes a path to an `.md` file and returns the associated page.
+Takes a path to an `.md` file and returns the associated page. The base path is the `content` directory.
 
 ```jinja2
 {% set page = get_page(path="blog/page2.md") %}
 ```
 
 ### `get_section`
-Takes a path to an `_index.md` file and returns the associated section.
+Takes a path to an `_index.md` file and returns the associated section. The base path is the `content` directory.
 
 ```jinja2
 {% set section = get_section(path="blog/_index.md") %}
@@ -108,78 +123,6 @@ If you only need the metadata of the section, you can pass `metadata_only=true`
 {% set section = get_section(path="blog/_index.md", metadata_only=true) %}
 ```
 
-### `get_url`
-Gets the permalink for the given path.
-If the path starts with `@/`, it will be treated as an internal
-link like the ones used in Markdown, starting from the root `content` directory.
-
-```jinja2
-{% set url = get_url(path="@/blog/_index.md") %}
-```
-
-It accepts an optional parameter `lang` in order to compute a *language-aware URL* in multilingual websites. Assuming `config.base_url` is `"http://example.com"`, the following snippet will:
-
-- return `"http://example.com/blog/"` if `config.default_language` is `"en"`
-- return `"http://example.com/en/blog/"` if `config.default_language` is **not** `"en"` and `"en"` appears in `config.languages`
-- fail otherwise, with the error message `"'en' is not an authorized language (check config.languages)."`
-
-```jinja2
-{% set url = get_url(path="@/blog/_index.md", lang="en") %}
-```
-
-This can also be used to get the permalinks for static assets, for example if
-we want to link to the file that is located at `static/css/app.css`:
-
-```jinja2
-{{/* get_url(path="css/app.css") */}}
-```
-
-By default, assets will not have a trailing slash. You can force one by passing `trailing_slash=true` to the `get_url` function.
-An example is:
-
-```jinja2
-{{/* get_url(path="css/app.css", trailing_slash=true) */}}
-```
-
-In the case of non-internal links, you can also add a cachebust of the format `?h=<sha256>` at the end of a URL
-by passing `cachebust=true` to the `get_url` function.
-
-### `get_file_hash`
-
-Returns the hash digest of a static file. Supported hashing algorithms are
-SHA-256, SHA-384 (default) and SHA-512. Requires `path`. The `sha_type`
-parameter is optional and must be one of 256, 384 or 512.
-
-```jinja2
-{{/* get_file_hash(path="js/app.js", sha_type=256) */}}
-```
-
-The function can also output a base64-encoded hash value when its `base64`
-parameter is set to `true`. This can be used to implement [subresource
-integrity](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity).
-
-```jinja2
-<script src="{{/* get_url(path="js/app.js") */}}"
-  integrity="sha384-{{ get_file_hash(path="js/app.js", sha_type=384, base64=true) | safe }}"></script>
-```
-
-Do note that subresource integrity is typically used when using external
-scripts, which `get_file_hash` does not support.
-
-Whenever hashing files, whether using `get_file_hash` or `get_url(...,
-cachebust=true)`, the file is searched for in three places: `static/`,
-`content/` and the output path (so e.g. compiled SASS can be hashed, too.)
-
-### `get_image_metadata`
-
-Gets metadata for an image. This supports common formats like JPEG, PNG, as well as SVG.
-Currently, the only supported keys are `width` and `height`.
-
-```jinja2
-  {% set meta = get_image_metadata(path="...") %}
-  Our image is {{ meta.width }}x{{ meta.height }}
-```
-
 ### `get_taxonomy_url`
 Gets the permalink for the taxonomy item found.
 
@@ -208,12 +151,88 @@ items: Array<TaxonomyTerm>;
 
 See the [Taxonomies documentation](@/documentation/templates/taxonomies.md) for a full documentation of those types.
 
+### `get_url`
+Gets the permalink for the given path.
+If the path starts with `@/`, it will be treated as an internal link like the ones used in Markdown, 
+starting from the root `content` directory as well as validated.
+
+```jinja2
+{% set url = get_url(path="@/blog/_index.md") %}
+```
+
+It accepts an optional parameter `lang` in order to compute a *language-aware URL* in multilingual websites. Assuming `config.base_url` is `"http://example.com"`, the following snippet will:
+
+- return `"http://example.com/blog/"` if `config.default_language` is `"en"`
+- return `"http://example.com/en/blog/"` if `config.default_language` is **not** `"en"` and `"en"` appears in `config.languages`
+- fail otherwise, with the error message `"'en' is not an authorized language (check config.languages)."`
+
+```jinja2
+{% set url = get_url(path="@/blog/_index.md", lang="en") %}
+```
+
+This can also be used to get the permalinks for static assets, for example if
+we want to link to the file that is located at `static/css/app.css`:
+
+```jinja2
+{{/* get_url(path="static/css/app.css") */}}
+```
+
+By default, assets will not have a trailing slash. You can force one by passing `trailing_slash=true` to the `get_url` function.
+An example is:
+
+```jinja2
+{{/* get_url(path="static/css/app.css", trailing_slash=true) */}}
+```
+
+In the case of non-internal links, you can also add a cachebust of the format `?h=<sha256>` at the end of a URL
+by passing `cachebust=true` to the `get_url` function. In this case, the path will need to resolve to an actual file. 
+See [File Searching Logic](@/documentation/templates/overview.md#file-searching-logic) for details.
+
+### `get_file_hash`
+
+Returns the hash digest (SHA-256, SHA-384 or SHA-512) of a file.
+
+It can take the following arguments:
+- `path`: mandatory, see [File Searching Logic](@/documentation/templates/overview.md#file-searching-logic) for details
+- `sha_type`: optional, one of `256`, `384` or `512`, defaults to `384`
+- `base64`: optional, `true` or `false`, defaults to `true`. Whether to encode the hash as base64
+
+```jinja2
+{{/* get_file_hash(path="static/js/app.js", sha_type=256) */}}
+```
+
+The function can also output a base64-encoded hash value when its `base64`
+parameter is set to `true`. This can be used to implement [subresource
+integrity](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity).
+
+```jinja2
+<script src="{{/* get_url(path="static/js/app.js") */}}"
+  integrity="sha384-{{ get_file_hash(path="static/js/app.js", sha_type=384, base64=true) | safe }}"></script>
+```
+
+Do note that subresource integrity is typically used when using external scripts, which `get_file_hash` does not support.
+
+### `get_image_metadata`
+
+Gets metadata for an image. This supports common formats like JPEG, PNG, WebP, BMP, GIF as well as SVG.
+
+It can take the following arguments:
+
+- `path`: mandatory, see [File Searching Logic](@/documentation/templates/overview.md#file-searching-logic) for details
+- `allow_missing`: optional, `true` or `false`, defaults to `false`. Whether a missing file should raise an error or not.
+
+The method returns a map containing `width`, `height` and `format` (the lowercased value as string).
+
+```jinja2
+  {% set meta = get_image_metadata(path="...") %}
+  Our image (.{{meta.format}}) has format is {{ meta.width }}x{{ meta.height }}
+```
+
 ### `load_data`
 Loads data from a file or URL. Supported file types include *toml*, *json*, *csv* and *bibtex*.
 Any other file type will be loaded as plain text.
 
-The `path` argument specifies the path to the data file relative to your base directory, where your `config.toml` is.
+The `path` argument specifies the path to the data file, according to the [File Searching Logic](@/documentation/templates/overview.md#file-searching-logic).
-As a security precaution, if this file is outside the main site directory, your site will fail to build.
 
 ```jinja2
 {% set data = load_data(path="content/blog/story/data.toml") %}