Handle markdown parser potentially splitting shortcodes

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## 0.2.2 (unreleased)
 
 - Fix shortcodes without arguments being ignored
+- Fix shortcodes with markdown chars (_, *, etc) in name and args being ignored
 
 
 ## 0.2.1 (2017-10-17)

components/rendering/src/markdown.rs

@@ -28,6 +28,11 @@ pub fn markdown_to_html(content: &str, context: &Context) -> Result<(String, Vec
     // Set while parsing
     let mut error = None;
     let mut highlighter: Option<HighlightLines> = None;
+    // the markdown parser will send several Text event if a markdown character
+    // is present in it, for example `hello_test` will be split in 2: hello and _test.
+    // Since we can use those chars in shortcode arguments, we need to collect
+    // the full shortcode somehow first
+    let mut current_shortcode = String::new();
     let mut shortcode_block = None;
     // shortcodes live outside of paragraph so we need to ensure we don't close
     // a paragraph that has already been closed
@@ -72,7 +77,7 @@ pub fn markdown_to_html(content: &str, context: &Context) -> Result<(String, Vec
 
     {
         let parser = Parser::new_ext(content, opts).map(|event| match event {
-            Event::Text(text) => {
+            Event::Text(mut text) => {
                 // Header first
                 if in_header {
                     if header_created {
@@ -101,6 +106,23 @@ pub fn markdown_to_html(content: &str, context: &Context) -> Result<(String, Vec
                     return Event::Text(text);
                 }
 
+                // Are we in the middle of a shortcode that somehow got cut off
+                // by the markdown parser?
+                if current_shortcode.is_empty() {
+                    if text.starts_with("{{") && !text.ends_with("}}") {
+                        current_shortcode += &text;
+                    } else if text.starts_with("{%") && !text.ends_with("%}") {
+                        current_shortcode += &text;
+                    }
+                } else {
+                    current_shortcode += &text;
+                }
+
+                if current_shortcode.ends_with("}}") || current_shortcode.ends_with("%}") {
+                    text = Owned(current_shortcode.clone());
+                    current_shortcode = String::new();
+                }
+
                 // Shortcode without body
                 if shortcode_block.is_none() && text.starts_with("{{") && text.ends_with("}}") && SHORTCODE_RE.is_match(&text) {
                     let (name, args) = parse_shortcode(&text);
@@ -254,6 +276,10 @@ pub fn markdown_to_html(content: &str, context: &Context) -> Result<(String, Vec
         cmark::html::push_html(&mut html, parser);
     }
 
+    if !current_shortcode.is_empty() {
+        return Err(format!("A shortcode was not closed properly:\n{:?}", current_shortcode).into());
+    }
+
     match error {
         Some(e) => Err(e),
         None => Ok((html.replace("<p></p>", ""), make_table_of_contents(&headers))),

components/rendering/src/short_code.rs

@@ -6,7 +6,7 @@ use tera::{Tera, Context};
 use errors::{Result, ResultExt};
 
 lazy_static!{
-    pub static ref SHORTCODE_RE: Regex = Regex::new(r#"\{(?:%|\{)\s+([[:alnum:]]+?)\(([[:alnum:]]+?="?.+?"?)?\)\s+(?:%|\})\}"#).unwrap();
+    pub static ref SHORTCODE_RE: Regex = Regex::new(r#"\{(?:%|\{)\s+([[:word:]]+?)\(([[:word:]]+?="?.+?"?)?\)\s+(?:%|\})\}"#).unwrap();
 }
 
 /// A shortcode that has a body
@@ -75,7 +75,28 @@ pub fn render_simple_shortcode(tera: &Tera, name: &str, args: &HashMap<String, S
 
 #[cfg(test)]
 mod tests {
-    use super::parse_shortcode;
+    use super::{parse_shortcode, SHORTCODE_RE};
+
+    #[test]
+    fn can_match_all_kinds_of_shortcode() {
+        let inputs = vec![
+            "{{ basic() }}",
+            "{{ basic(ho=1) }}",
+            "{{ basic(ho=\"hey\") }}",
+            "{{ basic(ho=\"hey_underscore\") }}",
+            "{{ basic(ho=\"hey-dash\") }}",
+            "{% basic(ho=\"hey-dash\") %}",
+            "{% basic(ho=\"hey_underscore\") %}",
+            "{% basic() %}",
+            "{% quo_te(author=\"Bob\") %}",
+            "{{ quo_te(author=\"Bob\") }}",
+        ];
+
+        for i in inputs {
+            println!("{}", i);
+            assert!(SHORTCODE_RE.is_match(i));
+        }
+    }
 
     #[test]
     fn can_parse_simple_shortcode_no_arg() {

components/rendering/tests/markdown.rs

@@ -84,6 +84,59 @@ Hello
     assert!(res.0.contains(r#"<iframe src="https://www.youtube.com/embed/ub36ffWAqgQ""#));
 }
 
+#[test]
+fn can_render_shortcode_with_markdown_char_in_args_name() {
+    let permalinks_ctx = HashMap::new();
+    let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
+    let input = vec![
+        "name",
+        "na_me",
+        "n_a_me",
+        "n1",
+    ];
+    for i in input {
+        let res = markdown_to_html(&format!("{{{{ youtube(id=\"hey\", {}=1) }}}}", i), &context).unwrap();
+        assert!(res.0.contains(r#"<iframe src="https://www.youtube.com/embed/hey""#));
+    }
+}
+
+#[test]
+fn can_render_shortcode_with_markdown_char_in_args_value() {
+    let permalinks_ctx = HashMap::new();
+    let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
+    let input = vec![
+        "ub36ffWAqgQ-hey",
+        "ub36ffWAqgQ_hey",
+        "ub36ffWAqgQ_he_y",
+        "ub36ffWAqgQ*hey",
+        "ub36ffWAqgQ#hey",
+    ];
+    for i in input {
+        let res = markdown_to_html(&format!("{{{{ youtube(id=\"{}\") }}}}", i), &context).unwrap();
+        assert!(res.0.contains(&format!(r#"<iframe src="https://www.youtube.com/embed/{}""#, i)));
+    }
+}
+
+#[test]
+fn can_render_body_shortcode_with_markdown_char_in_name() {
+    let permalinks_ctx = HashMap::new();
+    let mut tera = Tera::default();
+    tera.extend(&GUTENBERG_TERA).unwrap();
+    let input = vec![
+        "quo_te",
+        "qu_o_te",
+    ];
+
+    for i in input {
+        tera.add_raw_template(&format!("shortcodes/{}.html", i), "<blockquote>{{ body }} - {{ author}}</blockquote>").unwrap();
+        let context = Context::new(&tera, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
+
+        let res = markdown_to_html(&format!("{{% {}(author=\"Bob\") %}}\nhey\n{{% end %}}", i), &context).unwrap();
+        println!("{:?}", res);
+        assert!(res.0.contains("<blockquote>hey - Bob</blockquote>"));
+    }
+}
+
 #[test]
 fn can_render_several_shortcode_in_row() {
     let permalinks_ctx = HashMap::new();

components/site/test_site/content/posts/python.md

@@ -9,5 +9,6 @@ Same filename but different path
 {{ basic() }}
 
 {{ pirate(name="Bob") }}
+{{ pirate(name="Bob_Sponge") }}
 
 

components/site/tests/site.rs

@@ -110,6 +110,7 @@ fn can_build_site_without_live_reload() {
     // Shortcodes work
     assert!(file_contains!(public, "posts/python/index.html", "Basic shortcode"));
     assert!(file_contains!(public, "posts/python/index.html", "Arrrh Bob"));
+    assert!(file_contains!(public, "posts/python/index.html", "Arrrh Bob_Sponge"));
     assert!(file_exists!(public, "posts/tutorials/devops/nix/index.html"));
     assert!(file_exists!(public, "posts/with-assets/index.html"));
     assert!(file_exists!(public, "posts/no-section/simple/index.html"));

docs/content/documentation/content/shortcodes.md

@@ -43,6 +43,10 @@ In both cases, their arguments must be named and they will all be passed to the
 
 Any shortcodes in code blocks will be ignored.
 
+Lastly, a shortcode name (and thus the corresponding `.html` file) as well as the arguments name 
+can only contain numbers, letters and underscores, or in Regex terms the following: `[0-9A-Za-z_]`.
+While theoretically an argument name could be a number, it will not be possible to use in the template.
+
 ### Shortcodes without body
 
 On a new line, call the shortcode as if it was a Tera function in a variable block. All the examples below are valid
@@ -78,7 +82,8 @@ A quote
 {% end %}
 ```
 
-The body of the shortcode will be automatically passed down to the rendering context as the `body` variable.
+The body of the shortcode will be automatically passed down to the rendering context as the `body` variable and needs
+to be in a newline.
 
 ## Built-in shortcodes