Allow emitting newlines and whitespace in shortcodes and introduce markdown shortcodes (#1085)

* Replace hack for newline support in shortcodes with new hack

* Be a bit more space efficient/accurate with naming

* Boil newline/whitespace shortcode test down to the essentials

* Make sure the new \n and \s chars in old tests are properly represented

* Support markdown templates and shortcodes

* Refactoring .md/.html shortcode behaviour

* Add test for markdown shortcodes

* Add an html output test for markdown based shortcodes

* Add documentation for Markdown based shortcodes

components/rendering/src/lib.rs

@@ -14,7 +14,9 @@ pub fn render_content(content: &str, context: &RenderContext) -> Result<markdown
     // Don't do shortcodes if there is nothing like a shortcode in the content
     if content.contains("{{") || content.contains("{%") {
         let rendered = render_shortcodes(content, context)?;
-        return markdown_to_html(&rendered, context);
+        let mut html = markdown_to_html(&rendered, context)?;
+        html.body = html.body.replace("<!--\\n-->", "\n");
+        return Ok(html);
     }
 
     markdown_to_html(&content, context)

components/rendering/src/shortcode.rs

@@ -17,7 +17,6 @@ const _GRAMMAR: &str = include_str!("content.pest");
 pub struct ContentParser;
 
 lazy_static! {
-    static ref MULTIPLE_NEWLINE_RE: Regex = Regex::new(r"\n\s*\n").unwrap();
     static ref OUTER_NEWLINE_RE: Regex = Regex::new(r"^\s*\n|\n\s*$").unwrap();
 }
 
@@ -115,20 +114,28 @@ fn render_shortcode(
     }
     tera_context.extend(context.tera_context.clone());
 
-    let template_name = format!("shortcodes/{}.html", name);
+    let mut template_name = format!("shortcodes/{}.md", name);
+    if !context.tera.templates.contains_key(&template_name) {
+        template_name = format!("shortcodes/{}.html", name);
+    }
 
     let res = utils::templates::render_template(&template_name, &context.tera, tera_context, &None)
         .map_err(|e| Error::chain(format!("Failed to render {} shortcode", name), e))?;
 
-    // Small hack to avoid having multiple blank lines because of Tera tags for example
-    // A blank like will cause the markdown parser to think we're out of HTML and start looking
-    // at indentation, making the output a code block.
-    let res = MULTIPLE_NEWLINE_RE.replace_all(&res, "\n");
-
     let res = OUTER_NEWLINE_RE.replace_all(&res, "");
 
+    // A blank line will cause the markdown parser to think we're out of HTML and start looking
+    // at indentation, making the output a code block. To avoid this, newlines are replaced with
+    // "<!--\n-->" at this stage, which will be undone after markdown rendering in lib.rs. Since
+    // that is an HTML comment, it shouldn't be rendered anyway. and not cause problems unless
+    // someone wants to include that comment in their content. This behaviour is unwanted in when
+    // rendering markdown shortcodes.
+    if template_name.ends_with(".html") {
+        Ok(res.replace('\n', "<!--\\n-->").to_string())
+    } else {
         Ok(res.to_string())
     }
+}
 
 pub fn render_shortcodes(content: &str, context: &RenderContext) -> Result<String> {
     let mut res = String::with_capacity(content.len());
@@ -413,8 +420,8 @@ Some body {{ hello() }}{%/* end */%}"#,
     fn shortcodes_with_body_do_not_eat_newlines() {
         let mut tera = Tera::default();
         tera.add_raw_template("shortcodes/youtube.html", "{{body | safe}}").unwrap();
-        let res = render_shortcodes("Body\n {% youtube() %}\nHello \n World{% end %}", &tera);
-        assert_eq!(res, "Body\n Hello \n World");
+        let res = render_shortcodes("Body\n {% youtube() %}\nHello \n \n\n World{% end %}", &tera);
+        assert_eq!(res, "Body\n Hello <!--\\n--> <!--\\n--><!--\\n--> World");
     }
 
     #[test]
@@ -432,4 +439,12 @@ Some body {{ hello() }}{%/* end */%}"#,
         let res = render_shortcodes("\n{{ youtube() }}\n", &tera);
         assert_eq!(res, "\n  Hello, Zola.  \n");
     }
+
+    #[test]
+    fn shortcodes_that_emit_markdown() {
+        let mut tera = Tera::default();
+        tera.add_raw_template("shortcodes/youtube.md", "{% for i in [1,2,3] %}\n* {{ i }}\n{%- endfor %}").unwrap();
+        let res = render_shortcodes("{{ youtube() }}", &tera);
+        assert_eq!(res, "* 1\n* 2\n* 3");
+    }
 }

components/rendering/tests/markdown.rs

@@ -788,10 +788,7 @@ fn doesnt_try_to_highlight_content_from_shortcode() {
 
     let markdown_string = r#"{{ figure(src="spherecluster.png", caption="Some spheres.") }}"#;
 
-    let expected = r#"<figure>
-     <img src="/images/spherecluster.png" alt="Some spheres." />
-     <figcaption>Some spheres.</figcaption>
-</figure>"#;
+    let expected = "<figure>\n     \n     <img src=\"/images/spherecluster.png\" alt=\"Some spheres.\" />\n     \n\n     <figcaption>Some spheres.</figcaption>\n</figure>";
 
     tera.add_raw_template(&format!("shortcodes/{}.html", "figure"), shortcode).unwrap();
     let config = Config::default();
@@ -801,6 +798,28 @@ fn doesnt_try_to_highlight_content_from_shortcode() {
     assert_eq!(res.body, expected);
 }
 
+#[test]
+fn can_emit_newlines_and_whitespace_with_shortcode() {
+    let permalinks_ctx = HashMap::new();
+    let mut tera = Tera::default();
+    tera.extend(&ZOLA_TERA).unwrap();
+
+    let shortcode = r#"<pre>
+{{ body }}
+</pre>"#;
+
+    let markdown_string = "{% preformatted() %}\nHello\n    \n    Zola\n   \n  !\n{% end %}";
+
+    let expected = "<pre>\nHello\n    \n    Zola\n   \n  !\n</pre>";
+
+    tera.add_raw_template(&format!("shortcodes/{}.html", "preformatted"), shortcode).unwrap();
+    let config = Config::default();
+    let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
+
+    let res = render_content(markdown_string, &context).unwrap();
+    assert_eq!(res.body, expected);
+}
+
 // TODO: re-enable once it's fixed in Tera
 // https://github.com/Keats/tera/issues/373
 //#[test]
@@ -885,3 +904,44 @@ fn stops_with_an_error_on_an_empty_link() {
     assert!(res.is_err());
     assert_eq!(res.unwrap_err().to_string(), expected);
 }
+
+#[test]
+fn can_passthrough_markdown_from_shortcode() {
+    let permalinks_ctx = HashMap::new();
+    let mut tera = Tera::default();
+    tera.extend(&ZOLA_TERA).unwrap();
+
+    let shortcode = r#"{% for line in body | split(pat="\n") %}
+> {{ line }}
+{%- endfor %}
+
+-- {{ author }}
+"#;
+    let markdown_string = r#"
+Hello
+
+{% quote(author="Vincent") %}
+# Passing through
+
+*to* **the** document
+{% end %}
+
+Bla bla"#;
+
+    let expected = r#"<p>Hello</p>
+<blockquote>
+<h1 id="passing-through">Passing through</h1>
+<p><em>to</em> <strong>the</strong> document</p>
+</blockquote>
+<p>-- Vincent</p>
+<p>Bla bla</p>
+"#;
+
+    tera.add_raw_template(&format!("shortcodes/{}.md", "quote"), shortcode).unwrap();
+    let config = Config::default();
+    let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
+
+    let res = render_content(markdown_string, &context).unwrap();
+    println!("{:?}", res);
+    assert_eq!(res.body, expected);
+}

components/site/src/tpls.rs

@@ -10,7 +10,7 @@ use utils::templates::rewrite_theme_paths;
 
 pub fn load_tera(path: &Path, config: &Config) -> Result<Tera> {
     let tpl_glob =
-        format!("{}/{}", path.to_string_lossy().replace("\\", "/"), "templates/**/*.*ml");
+        format!("{}/{}", path.to_string_lossy().replace("\\", "/"), "templates/**/*.{*ml,md}");
 
     // Only parsing as we might be extending templates from themes and that would error
     // as we haven't loaded them yet
@@ -27,7 +27,7 @@ pub fn load_tera(path: &Path, config: &Config) -> Result<Tera> {
         let theme_tpl_glob = format!(
             "{}/{}",
             path.to_string_lossy().replace("\\", "/"),
-            format!("themes/{}/templates/**/*.*ml", theme)
+            format!("themes/{}/templates/**/*.{{*ml,md}}", theme)
         );
         let mut tera_theme = Tera::parse(&theme_tpl_glob)
             .map_err(|e| Error::chain("Error parsing templates from themes", e))?;

docs/content/documentation/content/shortcodes.md

@@ -3,14 +3,20 @@ title = "Shortcodes"
 weight = 40
 +++
 
-Although Markdown is good for writing, it isn't great when you need write inline
-HTML to add some styling for example.
-
-To solve this, Zola borrows the concept of [shortcodes](https://codex.wordpress.org/Shortcode_API)
-from WordPress.
+Zola borrows the concept of [shortcodes](https://codex.wordpress.org/Shortcode_API) from WordPress.
 In our case, a shortcode corresponds to a template defined in the `templates/shortcodes` directory or
-a built-in one that can be used in a Markdown file. If you want to use something similar to shortcodes in your templates, try [Tera macros](https://tera.netlify.com/docs#macros).
+a built-in one that can be used in a Markdown file. If you want to use something similar to shortcodes in your templates,
+try [Tera macros](https://tera.netlify.com/docs#macros).
 
+Broadly speaking, Zola's shortcodes cover two distinct use cases:
+
+* Inject more complex HTML: Markdown is good for writing, but it isn't great when you need add inline HTML or styling.
+* Ease repetitive data based tasks: when you have [external data](@/documentation/templates/overview.md#load-data) that you
+  want to display in your page's body.
+
+The latter may also be solved by writing HTML, however Zola allows the use of Markdown based shortcodes which end in `.md`
+rather than `.html`. This may be particularly useful if you want to include headings generated by the shortcode in the
+[table of contents](@/documentation/content/table-of-contents.md).
 
 ## Writing a shortcode
 Let's write a shortcode to embed YouTube videos as an example.
@@ -37,9 +43,24 @@ That's it. Zola will now recognise this template as a shortcode named `youtube`
 The Markdown renderer will wrap an inline HTML node such as `<a>` or `<span>` into a paragraph.
 If you want to disable this behaviour, wrap your shortcode in a `<div>`.
 
-Shortcodes are rendered before the Markdown is parsed so they don't have access to the table of contents. Because of that,
-you also cannot use the `get_page`/`get_section`/`get_taxonomy` global functions. It might work while running
-`zola serve` because it has been loaded but it will fail during `zola build`.
+A Markdown based shortcode in turn will be treated as if what it returned was part of the page's body. If we create
+`books.md` in `templates/shortcodes` for example:
+
+```jinja2
+{% set data = load_data(path=path) -%}
+{% for book in data.books %}
+### {{ book.title }}
+
+{{ book.description | safe }}
+{% endfor %}
+```
+
+This will create a shortcode `books` with the argument `path` pointing to a `.toml` file where it loads lists of books with
+titles and descriptions. They will flow with the rest of the document in which `books` is called.
+
+Shortcodes are rendered before the page's Markdown is parsed so they don't have access to the page's table of contents.
+Because of that, you also cannot use the `get_page`/`get_section`/`get_taxonomy` global functions. It might work while
+running `zola serve` because it has been loaded but it will fail during `zola build`.
 
 ## Using shortcodes
 

docs/content/documentation/content/table-of-contents.md

@@ -3,7 +3,7 @@ title = "Table of Contents"
 weight = 60
 +++
 
-Each page/section will automatically generate a table of contents for itself based on the headers present.
+Each page/section will automatically generate a table of contents for itself based on the headers generated with markdown.
 
 It is available in the template through the `page.toc` or `section.toc` variable.
 You can view the [template variables](@/documentation/templates/pages-sections.md#table-of-contents)