emelie/zola
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

load_data now defaults to plain type + fix bug with get_taxonomy fn

Browse source
This commit is contained in:
Vincent Prouillet 2019-03-16 10:01:11 +01:00
parent 3eaf13d49b
commit c63b7fde44
6 changed files with 48 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -20,7 +20,9 @@ rendering it and not anymore on the `page`/`section` variable
- Fix sections with render=false being shown in sitemap
- Sitemap is now split when there are more than 30 000 links in it
- Add link to sitemap in robots.txt
- Markdown rendering is now fully CommonMark commpliant
- Markdown rendering is now fully CommonMark compliant
- `load_data` now defaults to loading file as plain text, unless `format` is passed
or the extension matches csv/toml/json
## 0.5.1 (2018-12-14)

8
components/rendering/src/markdown.rs
View file

@ -14,7 +14,7 @@ use table_of_contents::{make_table_of_contents, Header};
use utils::site::resolve_internal_link;
use utils::vec::InsertMany;
use self::cmark::{Event, Options, Parser, Tag, LinkType};
use self::cmark::{Event, LinkType, Options, Parser, Tag};
const CONTINUE_READING: &str =
"<p id=\"zola-continue-reading\"><a name=\"continue-reading\"></a></p>\n";
@ -200,11 +200,7 @@ pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<Render
Event::Start(Tag::Image(link_type, src, title)) => {
if is_colocated_asset_link(&src) {
let link = format!("{}{}", context.current_page_permalink, &*src);
return Event::Start(Tag::Image(
link_type,
link.into(),
title,
));
return Event::Start(Tag::Image(link_type, link.into(), title));
}
Event::Start(Tag::Image(link_type, src, title))

27
components/site/src/lib.rs
View file

@ -246,10 +246,12 @@ impl Site {
self.add_page(p, false)?;
}
// taxonomy Tera fns are loaded in `register_early_global_fns`
// so we do need to populate it first.
self.populate_taxonomies()?;
self.register_early_global_fns();
self.populate_sections();
self.render_markdown()?;
self.populate_taxonomies()?;
self.register_tera_global_fns();
Ok(())
@ -347,10 +349,7 @@ impl Site {
"resize_image",
global_fns::ResizeImage::new(self.imageproc.clone()),
);
self.tera.register_function(
"load_data",
global_fns::LoadData::new(self.base_path.clone()),
);
self.tera.register_function("load_data", global_fns::LoadData::new(self.base_path.clone()));
self.tera.register_function("trans", global_fns::Trans::new(self.config.clone()));
self.tera.register_function(
"get_taxonomy_url",
@ -862,7 +861,6 @@ impl Site {
taxonomies.push(terms);
}
let mut all_sitemap_entries = HashSet::new();
for p in pages {
all_sitemap_entries.insert(p);
@ -886,25 +884,32 @@ impl Site {
context.insert("entries", &all_sitemap_entries);
let sitemap = &render_template("sitemap.xml", &self.tera, context, &self.config.theme)?;
create_file(&self.output_path.join("sitemap.xml"), sitemap)?;
return Ok(())
return Ok(());
}
// Create multiple sitemaps (max 30000 urls each)
let mut sitemap_index = Vec::new();
for (i, chunk) in all_sitemap_entries.iter().collect::<Vec<_>>().chunks(sitemap_limit).enumerate() {
for (i, chunk) in
all_sitemap_entries.iter().collect::<Vec<_>>().chunks(sitemap_limit).enumerate()
{
let mut context = Context::new();
context.insert("entries", &chunk);
let sitemap = &render_template("sitemap.xml", &self.tera, context, &self.config.theme)?;
let file_name = format!("sitemap{}.xml", i+1);
let file_name = format!("sitemap{}.xml", i + 1);
create_file(&self.output_path.join(&file_name), sitemap)?;
let mut sitemap_url:String = self.config.make_permalink(&file_name);
let mut sitemap_url: String = self.config.make_permalink(&file_name);
sitemap_url.pop(); // Remove trailing slash
sitemap_index.push(sitemap_url);
}
// Create main sitemap that reference numbered sitemaps
let mut main_context = Context::new();
main_context.insert("sitemaps", &sitemap_index);
let sitemap = &render_template("split_sitemap_index.xml", &self.tera, main_context, &self.config.theme)?;
let sitemap = &render_template(
"split_sitemap_index.xml",
&self.tera,
main_context,
&self.config.theme,
)?;
create_file(&self.output_path.join("sitemap.xml"), sitemap)?;
Ok(())

30
components/templates/src/global_fns/load_data.rs
View file

@ -151,7 +151,7 @@ fn get_output_format_from_args(
let format_arg = optional_arg!(
String,
args.get("format"),
"`load_data`: `format` needs to be an argument with a string value, being one of the supported `load_data` file types (csv, json, toml)"
"`load_data`: `format` needs to be an argument with a string value, being one of the supported `load_data` file types (csv, json, toml, plain)"
);
if let Some(format) = format_arg {
@ -159,11 +159,7 @@ fn get_output_format_from_args(
}
let from_extension = if let DataSource::Path(path) = data_source {
let extension_result: Result<&str> =
path.extension().map(|extension| extension.to_str().unwrap()).ok_or_else(|| {
format!("Could not determine format for {} from extension", path.display()).into()
});
extension_result?
path.extension().map(|extension| extension.to_str().unwrap()).unwrap_or_else(|| "plain")
} else {
"plain"
};
@ -332,8 +328,7 @@ mod tests {
#[test]
fn fails_when_missing_file() {
let static_fn =
LoadData::new(PathBuf::from("../utils"));
let static_fn = LoadData::new(PathBuf::from("../utils"));
let mut args = HashMap::new();
args.insert("path".to_string(), to_value("../../../READMEE.md").unwrap());
let result = static_fn.call(&args);
@ -343,8 +338,7 @@ mod tests {
#[test]
fn cant_load_outside_content_dir() {
let static_fn =
LoadData::new(PathBuf::from(PathBuf::from("../utils")));
let static_fn = LoadData::new(PathBuf::from(PathBuf::from("../utils")));
let mut args = HashMap::new();
args.insert("path".to_string(), to_value("../../README.md").unwrap());
args.insert("format".to_string(), to_value("plain").unwrap());
@ -421,9 +415,7 @@ mod tests {
#[test]
fn can_load_toml() {
let static_fn = LoadData::new(
PathBuf::from("../utils/test-files"),
);
let static_fn = LoadData::new(PathBuf::from("../utils/test-files"));
let mut args = HashMap::new();
args.insert("path".to_string(), to_value("test.toml").unwrap());
let result = static_fn.call(&args.clone()).unwrap();
@ -442,9 +434,7 @@ mod tests {
#[test]
fn can_load_csv() {
let static_fn = LoadData::new(
PathBuf::from("../utils/test-files"),
);
let static_fn = LoadData::new(PathBuf::from("../utils/test-files"));
let mut args = HashMap::new();
args.insert("path".to_string(), to_value("test.csv").unwrap());
let result = static_fn.call(&args.clone()).unwrap();
@ -464,9 +454,7 @@ mod tests {
// Test points to bad csv file with uneven row lengths
#[test]
fn bad_csv_should_result_in_error() {
let static_fn = LoadData::new(
PathBuf::from("../utils/test-files"),
);
let static_fn = LoadData::new(PathBuf::from("../utils/test-files"));
let mut args = HashMap::new();
args.insert("path".to_string(), to_value("uneven_rows.csv").unwrap());
let result = static_fn.call(&args.clone());
@ -486,9 +474,7 @@ mod tests {
#[test]
fn can_load_json() {
let static_fn = LoadData::new(
PathBuf::from("../utils/test-files"),
);
let static_fn = LoadData::new(PathBuf::from("../utils/test-files"));
let mut args = HashMap::new();
args.insert("path".to_string(), to_value("test.json").unwrap());
let result = static_fn.call(&args.clone()).unwrap();

5
components/templates/src/lib.rs
View file

@ -35,7 +35,10 @@ lazy_static! {
("__zola_builtins/rss.xml", include_str!("builtins/rss.xml")),
("__zola_builtins/sitemap.xml", include_str!("builtins/sitemap.xml")),
("__zola_builtins/robots.txt", include_str!("builtins/robots.txt")),
("__zola_builtins/split_sitemap_index.xml", include_str!("builtins/split_sitemap_index.xml")),
(
"__zola_builtins/split_sitemap_index.xml",
include_str!("builtins/split_sitemap_index.xml"),
),
("__zola_builtins/anchor-link.html", include_str!("builtins/anchor-link.html")),
(
"__zola_builtins/shortcodes/youtube.html",

25
docs/content/documentation/templates/overview.md
View file

@ -146,34 +146,37 @@ Gets the whole taxonomy of a specific kind.
```
### `load_data`
Loads data from a file or URL. Supported file types include *toml*, *json* and *csv*.
Loads data from a file or URL. Supported file types include *toml*, *json* and *csv*.
Any other file type will be loaded as plain text.
The `path` argument specifies the path to the data file relative to your content directory.
The `path` argument specifies the path to the data file relative to your base directory, where your `config.toml` is.
As a security precaution, If this file is outside of the main site directory, your site will fail to build.
```jinja2
{% set data = load_data(path="blog/story/data.toml") %}
{% set data = load_data(path="content/blog/story/data.toml") %}
```
The optional `format` argument allows you to specify and override which data type is contained
within the file specified in the `path` argument. Valid entries are *"toml"*, *"json"*, *"csv"*
or *"plain"*. If the `format` argument isn't specified, then the paths extension is used.
within the file specified in the `path` argument. Valid entries are `toml`, `json`, `csv`
or `plain`. If the `format` argument isn't specified, then the paths extension is used.
```jinja2
{% set data = load_data(path="blog/story/data.txt", format="json") %}
{% set data = load_data(path="content/blog/story/data.txt", format="json") %}
```
Use the `plain` format for when your file has a toml/json/csv extension but you want to load it as plain text.
For *toml* and *json* the data is loaded into a structure matching the original data file,
however for *csv* there is no native notion of such a structure. Instead the data is seperated
however for *csv* there is no native notion of such a structure. Instead the data is separated
into a data structure containing *headers* and *records*. See the example below to see
how this works.
In the template:
```jinja2
{% set data = load_data(path="blog/story/data.csv") %}
{% set data = load_data(path="content/blog/story/data.csv") %}
```
In the *blog/story/data.csv* file:
In the *content/blog/story/data.csv* file:
```csv
Number, Title
1,Gutenberg
@ -211,7 +214,9 @@ By default, the response body will be returned with no parsing. This can be chan
#### Data Caching
Data file loading and remote requests are cached in memory during build, so multiple requests aren't made to the same endpoint. URLs are cached based on the URL, and data files are cached based on the files modified time. The format is also taken into account when caching, so a request will be sent twice if it's loaded with 2 different formats.
Data file loading and remote requests are cached in memory during build, so multiple requests aren't made to the same endpoint.
URLs are cached based on the URL, and data files are cached based on the files modified time.
The format is also taken into account when caching, so a request will be sent twice if it's loaded with 2 different formats.
### `trans`
Gets the translation of the given `key`, for the `default_language` or the `language given