diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bb9f93b..f95ed95d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,9 @@ rendering it and not anymore on the `page`/`section` variable - Fix sections with render=false being shown in sitemap - Sitemap is now split when there are more than 30 000 links in it - Add link to sitemap in robots.txt -- Markdown rendering is now fully CommonMark commpliant +- Markdown rendering is now fully CommonMark compliant +- `load_data` now defaults to loading file as plain text, unless `format` is passed +or the extension matches csv/toml/json ## 0.5.1 (2018-12-14) diff --git a/components/rendering/src/markdown.rs b/components/rendering/src/markdown.rs index 5994f7de..6c34a585 100644 --- a/components/rendering/src/markdown.rs +++ b/components/rendering/src/markdown.rs @@ -14,7 +14,7 @@ use table_of_contents::{make_table_of_contents, Header}; use utils::site::resolve_internal_link; use utils::vec::InsertMany; -use self::cmark::{Event, Options, Parser, Tag, LinkType}; +use self::cmark::{Event, LinkType, Options, Parser, Tag}; const CONTINUE_READING: &str = "

\n"; @@ -200,11 +200,7 @@ pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result { if is_colocated_asset_link(&src) { let link = format!("{}{}", context.current_page_permalink, &*src); - return Event::Start(Tag::Image( - link_type, - link.into(), - title, - )); + return Event::Start(Tag::Image(link_type, link.into(), title)); } Event::Start(Tag::Image(link_type, src, title)) diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index f64dbd2e..bd41dae1 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -246,10 +246,12 @@ impl Site { self.add_page(p, false)?; } + // taxonomy Tera fns are loaded in `register_early_global_fns` + // so we do need to populate it first. + self.populate_taxonomies()?; self.register_early_global_fns(); self.populate_sections(); self.render_markdown()?; - self.populate_taxonomies()?; self.register_tera_global_fns(); Ok(()) @@ -347,10 +349,7 @@ impl Site { "resize_image", global_fns::ResizeImage::new(self.imageproc.clone()), ); - self.tera.register_function( - "load_data", - global_fns::LoadData::new(self.base_path.clone()), - ); + self.tera.register_function("load_data", global_fns::LoadData::new(self.base_path.clone())); self.tera.register_function("trans", global_fns::Trans::new(self.config.clone())); self.tera.register_function( "get_taxonomy_url", @@ -862,7 +861,6 @@ impl Site { taxonomies.push(terms); } - let mut all_sitemap_entries = HashSet::new(); for p in pages { all_sitemap_entries.insert(p); @@ -886,25 +884,32 @@ impl Site { context.insert("entries", &all_sitemap_entries); let sitemap = &render_template("sitemap.xml", &self.tera, context, &self.config.theme)?; create_file(&self.output_path.join("sitemap.xml"), sitemap)?; - return Ok(()) + return Ok(()); } // Create multiple sitemaps (max 30000 urls each) let mut sitemap_index = Vec::new(); - for (i, chunk) in all_sitemap_entries.iter().collect::>().chunks(sitemap_limit).enumerate() { + for (i, chunk) in + all_sitemap_entries.iter().collect::>().chunks(sitemap_limit).enumerate() + { let mut context = Context::new(); context.insert("entries", &chunk); let sitemap = &render_template("sitemap.xml", &self.tera, context, &self.config.theme)?; - let file_name = format!("sitemap{}.xml", i+1); + let file_name = format!("sitemap{}.xml", i + 1); create_file(&self.output_path.join(&file_name), sitemap)?; - let mut sitemap_url:String = self.config.make_permalink(&file_name); + let mut sitemap_url: String = self.config.make_permalink(&file_name); sitemap_url.pop(); // Remove trailing slash sitemap_index.push(sitemap_url); } // Create main sitemap that reference numbered sitemaps let mut main_context = Context::new(); main_context.insert("sitemaps", &sitemap_index); - let sitemap = &render_template("split_sitemap_index.xml", &self.tera, main_context, &self.config.theme)?; + let sitemap = &render_template( + "split_sitemap_index.xml", + &self.tera, + main_context, + &self.config.theme, + )?; create_file(&self.output_path.join("sitemap.xml"), sitemap)?; Ok(()) diff --git a/components/templates/src/global_fns/load_data.rs b/components/templates/src/global_fns/load_data.rs index 5bc506fd..9e6449c1 100644 --- a/components/templates/src/global_fns/load_data.rs +++ b/components/templates/src/global_fns/load_data.rs @@ -151,7 +151,7 @@ fn get_output_format_from_args( let format_arg = optional_arg!( String, args.get("format"), - "`load_data`: `format` needs to be an argument with a string value, being one of the supported `load_data` file types (csv, json, toml)" + "`load_data`: `format` needs to be an argument with a string value, being one of the supported `load_data` file types (csv, json, toml, plain)" ); if let Some(format) = format_arg { @@ -159,11 +159,7 @@ fn get_output_format_from_args( } let from_extension = if let DataSource::Path(path) = data_source { - let extension_result: Result<&str> = - path.extension().map(|extension| extension.to_str().unwrap()).ok_or_else(|| { - format!("Could not determine format for {} from extension", path.display()).into() - }); - extension_result? + path.extension().map(|extension| extension.to_str().unwrap()).unwrap_or_else(|| "plain") } else { "plain" }; @@ -332,8 +328,7 @@ mod tests { #[test] fn fails_when_missing_file() { - let static_fn = - LoadData::new(PathBuf::from("../utils")); + let static_fn = LoadData::new(PathBuf::from("../utils")); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("../../../READMEE.md").unwrap()); let result = static_fn.call(&args); @@ -343,8 +338,7 @@ mod tests { #[test] fn cant_load_outside_content_dir() { - let static_fn = - LoadData::new(PathBuf::from(PathBuf::from("../utils"))); + let static_fn = LoadData::new(PathBuf::from(PathBuf::from("../utils"))); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("../../README.md").unwrap()); args.insert("format".to_string(), to_value("plain").unwrap()); @@ -421,9 +415,7 @@ mod tests { #[test] fn can_load_toml() { - let static_fn = LoadData::new( - PathBuf::from("../utils/test-files"), - ); + let static_fn = LoadData::new(PathBuf::from("../utils/test-files")); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("test.toml").unwrap()); let result = static_fn.call(&args.clone()).unwrap(); @@ -442,9 +434,7 @@ mod tests { #[test] fn can_load_csv() { - let static_fn = LoadData::new( - PathBuf::from("../utils/test-files"), - ); + let static_fn = LoadData::new(PathBuf::from("../utils/test-files")); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("test.csv").unwrap()); let result = static_fn.call(&args.clone()).unwrap(); @@ -464,9 +454,7 @@ mod tests { // Test points to bad csv file with uneven row lengths #[test] fn bad_csv_should_result_in_error() { - let static_fn = LoadData::new( - PathBuf::from("../utils/test-files"), - ); + let static_fn = LoadData::new(PathBuf::from("../utils/test-files")); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("uneven_rows.csv").unwrap()); let result = static_fn.call(&args.clone()); @@ -486,9 +474,7 @@ mod tests { #[test] fn can_load_json() { - let static_fn = LoadData::new( - PathBuf::from("../utils/test-files"), - ); + let static_fn = LoadData::new(PathBuf::from("../utils/test-files")); let mut args = HashMap::new(); args.insert("path".to_string(), to_value("test.json").unwrap()); let result = static_fn.call(&args.clone()).unwrap(); diff --git a/components/templates/src/lib.rs b/components/templates/src/lib.rs index eec3b1f3..a1b61139 100644 --- a/components/templates/src/lib.rs +++ b/components/templates/src/lib.rs @@ -35,7 +35,10 @@ lazy_static! { ("__zola_builtins/rss.xml", include_str!("builtins/rss.xml")), ("__zola_builtins/sitemap.xml", include_str!("builtins/sitemap.xml")), ("__zola_builtins/robots.txt", include_str!("builtins/robots.txt")), - ("__zola_builtins/split_sitemap_index.xml", include_str!("builtins/split_sitemap_index.xml")), + ( + "__zola_builtins/split_sitemap_index.xml", + include_str!("builtins/split_sitemap_index.xml"), + ), ("__zola_builtins/anchor-link.html", include_str!("builtins/anchor-link.html")), ( "__zola_builtins/shortcodes/youtube.html", diff --git a/docs/content/documentation/templates/overview.md b/docs/content/documentation/templates/overview.md index bf9aad72..02349c83 100644 --- a/docs/content/documentation/templates/overview.md +++ b/docs/content/documentation/templates/overview.md @@ -146,34 +146,37 @@ Gets the whole taxonomy of a specific kind. ``` ### `load_data` -Loads data from a file or URL. Supported file types include *toml*, *json* and *csv*. +Loads data from a file or URL. Supported file types include *toml*, *json* and *csv*. +Any other file type will be loaded as plain text. -The `path` argument specifies the path to the data file relative to your content directory. +The `path` argument specifies the path to the data file relative to your base directory, where your `config.toml` is. As a security precaution, If this file is outside of the main site directory, your site will fail to build. ```jinja2 -{% set data = load_data(path="blog/story/data.toml") %} +{% set data = load_data(path="content/blog/story/data.toml") %} ``` The optional `format` argument allows you to specify and override which data type is contained -within the file specified in the `path` argument. Valid entries are *"toml"*, *"json"*, *"csv"* -or *"plain"*. If the `format` argument isn't specified, then the paths extension is used. +within the file specified in the `path` argument. Valid entries are `toml`, `json`, `csv` +or `plain`. If the `format` argument isn't specified, then the paths extension is used. ```jinja2 -{% set data = load_data(path="blog/story/data.txt", format="json") %} +{% set data = load_data(path="content/blog/story/data.txt", format="json") %} ``` +Use the `plain` format for when your file has a toml/json/csv extension but you want to load it as plain text. + For *toml* and *json* the data is loaded into a structure matching the original data file, -however for *csv* there is no native notion of such a structure. Instead the data is seperated +however for *csv* there is no native notion of such a structure. Instead the data is separated into a data structure containing *headers* and *records*. See the example below to see how this works. In the template: ```jinja2 -{% set data = load_data(path="blog/story/data.csv") %} +{% set data = load_data(path="content/blog/story/data.csv") %} ``` -In the *blog/story/data.csv* file: +In the *content/blog/story/data.csv* file: ```csv Number, Title 1,Gutenberg @@ -211,7 +214,9 @@ By default, the response body will be returned with no parsing. This can be chan #### Data Caching -Data file loading and remote requests are cached in memory during build, so multiple requests aren't made to the same endpoint. URLs are cached based on the URL, and data files are cached based on the files modified time. The format is also taken into account when caching, so a request will be sent twice if it's loaded with 2 different formats. +Data file loading and remote requests are cached in memory during build, so multiple requests aren't made to the same endpoint. +URLs are cached based on the URL, and data files are cached based on the files modified time. +The format is also taken into account when caching, so a request will be sent twice if it's loaded with 2 different formats. ### `trans` Gets the translation of the given `key`, for the `default_language` or the `language given