From 85e13483ebc0514d7f577546de96a16ae15fb222 Mon Sep 17 00:00:00 2001 From: Vincent Prouillet Date: Sun, 6 May 2018 22:58:39 +0200 Subject: [PATCH] Use the new shortcode parser --- Cargo.lock | 5 +- README.md | 4 +- components/content/src/page.rs | 14 +- components/content/src/section.rs | 9 +- components/front_matter/src/page.rs | 2 +- .../examples/generate_sublime.rs | 0 components/highlighting/src/lib.rs | 15 +- components/rendering/Cargo.toml | 2 - components/rendering/benches/all.rs | 43 ++- components/rendering/src/context.rs | 28 +- components/rendering/src/lib.rs | 22 +- components/rendering/src/markdown.rs | 361 ++++++------------ components/rendering/src/short_code.rs | 190 --------- components/rendering/src/shortcode.rs | 30 +- components/rendering/src/table_of_contents.rs | 12 +- components/rendering/tests/markdown.rs | 164 ++++---- sublime_syntaxes/newlines.packdump | Bin 413042 -> 416076 bytes sublime_syntaxes/nonewlines.packdump | Bin 413083 -> 416145 bytes 18 files changed, 322 insertions(+), 579 deletions(-) rename components/{rendering => highlighting}/examples/generate_sublime.rs (100%) delete mode 100644 components/rendering/src/short_code.rs diff --git a/Cargo.lock b/Cargo.lock index 9023ba27..513d8662 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1113,12 +1113,13 @@ dependencies = [ name = "rendering" version = "0.1.0" dependencies = [ + "config 0.1.0", "errors 0.1.0", "front_matter 0.1.0", "highlighting 0.1.0", - "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "pest 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "pest_derive 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", "slug 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/README.md b/README.md index 95bb0f52..23454eb7 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ If you want a feature added or modified, please open an issue to discuss it befo Syntax highlighting depends on submodules so ensure you load them first: ```bash -$ git submodule update --init +$ git submodule update --init ``` Gutenberg only works with syntaxes in the `.sublime-syntax` format. If your syntax @@ -75,7 +75,7 @@ You can check for any updates to the current packages by running: $ git submodule update --remote --merge ``` -And finally from the root of the components/rendering crate run the following command: +And finally from the root of the components/highlighting crate run the following command: ```bash $ cargo run --example generate_sublime synpack ../../sublime_syntaxes ../../sublime_syntaxes/newlines.packdump ../../sublime_syntaxes/nonewlines.packdump diff --git a/components/content/src/page.rs b/components/content/src/page.rs index b3cd820b..f977c464 100644 --- a/components/content/src/page.rs +++ b/components/content/src/page.rs @@ -14,7 +14,7 @@ use utils::fs::{read_file, find_related_assets}; use utils::site::get_reading_analytics; use utils::templates::render_template; use front_matter::{PageFrontMatter, InsertAnchor, split_page_content}; -use rendering::{Context, Header, markdown_to_html}; +use rendering::{RenderContext, Header, render_content}; use file_info::FileInfo; @@ -162,21 +162,23 @@ impl Page { /// We need access to all pages url to render links relative to content /// so that can't happen at the same time as parsing pub fn render_markdown(&mut self, permalinks: &HashMap, tera: &Tera, config: &Config, anchor_insert: InsertAnchor) -> Result<()> { - let context = Context::new( + let context = RenderContext::new( tera, - config.highlight_code, - config.highlight_theme.clone(), + config, &self.permalink, permalinks, anchor_insert ); - let res = markdown_to_html(&self.raw_content.replacen("", "", 1), &context)?; + let res = render_content( + &self.raw_content.replacen("", "", 1), + &context + )?; self.content = res.0; self.toc = res.1; if self.raw_content.contains("") { self.summary = Some({ let summary = self.raw_content.splitn(2, "").collect::>()[0]; - markdown_to_html(summary, &context)?.0 + render_content(summary, &context)?.0 }) } diff --git a/components/content/src/section.rs b/components/content/src/section.rs index b75581c6..c864318e 100644 --- a/components/content/src/section.rs +++ b/components/content/src/section.rs @@ -11,7 +11,7 @@ use errors::{Result, ResultExt}; use utils::fs::read_file; use utils::templates::render_template; use utils::site::get_reading_analytics; -use rendering::{Context, Header, markdown_to_html}; +use rendering::{RenderContext, Header, render_content}; use page::Page; use file_info::FileInfo; @@ -98,15 +98,14 @@ impl Section { /// We need access to all pages url to render links relative to content /// so that can't happen at the same time as parsing pub fn render_markdown(&mut self, permalinks: &HashMap, tera: &Tera, config: &Config) -> Result<()> { - let context = Context::new( + let context = RenderContext::new( tera, - config.highlight_code, - config.highlight_theme.clone(), + config, &self.permalink, permalinks, self.meta.insert_anchor_links, ); - let res = markdown_to_html(&self.raw_content, &context)?; + let res = render_content(&self.raw_content, &context)?; self.content = res.0; self.toc = res.1; Ok(()) diff --git a/components/front_matter/src/page.rs b/components/front_matter/src/page.rs index 86cde2a9..0b33a3a2 100644 --- a/components/front_matter/src/page.rs +++ b/components/front_matter/src/page.rs @@ -21,7 +21,7 @@ fn from_toml_datetime<'de, D>(deserializer: D) -> StdResult, D::E fn convert_toml_date(table: Map) -> Value { let mut new = Map::new(); - for (k, v) in table.into_iter() { + for (k, v) in table { if k == "$__toml_private_datetime" { return v; } diff --git a/components/rendering/examples/generate_sublime.rs b/components/highlighting/examples/generate_sublime.rs similarity index 100% rename from components/rendering/examples/generate_sublime.rs rename to components/highlighting/examples/generate_sublime.rs diff --git a/components/highlighting/src/lib.rs b/components/highlighting/src/lib.rs index f0a80b83..10834dbe 100644 --- a/components/highlighting/src/lib.rs +++ b/components/highlighting/src/lib.rs @@ -4,7 +4,8 @@ extern crate syntect; use syntect::dumps::from_binary; use syntect::parsing::SyntaxSet; -use syntect::highlighting::ThemeSet; +use syntect::highlighting::{ThemeSet, Theme}; +use syntect::easy::HighlightLines; thread_local!{ pub static SYNTAX_SET: SyntaxSet = { @@ -17,3 +18,15 @@ thread_local!{ lazy_static!{ pub static ref THEME_SET: ThemeSet = from_binary(include_bytes!("../../../sublime_themes/all.themedump")); } + + +pub fn get_highlighter<'a>(theme: &'a Theme, info: &str) -> HighlightLines<'a> { + SYNTAX_SET.with(|ss| { + let syntax = info + .split(' ') + .next() + .and_then(|lang| ss.find_syntax_by_token(lang)) + .unwrap_or_else(|| ss.find_syntax_plain_text()); + HighlightLines::new(syntax, theme) + }) +} diff --git a/components/rendering/Cargo.toml b/components/rendering/Cargo.toml index 92ff5c08..029c5056 100644 --- a/components/rendering/Cargo.toml +++ b/components/rendering/Cargo.toml @@ -5,8 +5,6 @@ authors = ["Vincent Prouillet "] [dependencies] tera = "0.11" -regex = "1" -lazy_static = "1" syntect = "2" pulldown-cmark = "0" slug = "0.1" diff --git a/components/rendering/benches/all.rs b/components/rendering/benches/all.rs index e7a968b0..70244521 100644 --- a/components/rendering/benches/all.rs +++ b/components/rendering/benches/all.rs @@ -9,7 +9,7 @@ extern crate front_matter; use std::collections::HashMap; use tera::Tera; -use rendering::{Context, markdown_to_html, render_shortcodes}; +use rendering::{RenderContext, render_content, render_shortcodes}; use front_matter::InsertAnchor; use config::Config; @@ -86,19 +86,36 @@ if __name__ == "__main__": "#; #[bench] -fn bench_markdown_to_html_with_highlighting(b: &mut test::Bencher) { - let tera_ctx = Tera::default(); +fn bench_render_content_with_highlighting(b: &mut test::Bencher) { + let mut tera = Tera::default(); + tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap(); let permalinks_ctx = HashMap::new(); - let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None); - b.iter(|| markdown_to_html(CONTENT, &context)); + let config = Config::default(); + let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None); + b.iter(|| render_content(CONTENT, &context).unwrap()); } #[bench] -fn bench_markdown_to_html_without_highlighting(b: &mut test::Bencher) { - let tera_ctx = Tera::default(); +fn bench_render_content_without_highlighting(b: &mut test::Bencher) { + let mut tera = Tera::default(); + tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap(); let permalinks_ctx = HashMap::new(); - let context = Context::new(&tera_ctx, false, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None); - b.iter(|| markdown_to_html(CONTENT, &context)); + let mut config = Config::default(); + config.highlight_code = false; + let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None); + b.iter(|| render_content(CONTENT, &context).unwrap()); +} + +#[bench] +fn bench_render_content_no_shortcode(b: &mut test::Bencher) { + let tera = Tera::default(); + let content2 = CONTENT.replace(r#"{{ youtube(id="my_youtube_id") }}"#, ""); + let mut config = Config::default(); + config.highlight_code = false; + let permalinks_ctx = HashMap::new(); + let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None); + + b.iter(|| render_content(&content2, &context).unwrap()); } #[bench] @@ -109,11 +126,3 @@ fn bench_render_shortcodes_one_present(b: &mut test::Bencher) { b.iter(|| render_shortcodes(CONTENT, &tera, &Config::default())); } -#[bench] -fn bench_render_shortcodes_none(b: &mut test::Bencher) { - let mut tera = Tera::default(); - tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap(); - let content2 = CONTENT.replace(r#"{{ youtube(id="my_youtube_id") }}"#, ""); - - b.iter(|| render_shortcodes(&content2, &tera, &Config::default())); -} diff --git a/components/rendering/src/context.rs b/components/rendering/src/context.rs index aa866be6..d8a2513f 100644 --- a/components/rendering/src/context.rs +++ b/components/rendering/src/context.rs @@ -1,41 +1,35 @@ use std::collections::HashMap; use tera::Tera; - use front_matter::InsertAnchor; +use config::Config; /// All the information from the gutenberg site that is needed to render HTML from markdown #[derive(Debug)] -pub struct Context<'a> { +pub struct RenderContext<'a> { pub tera: &'a Tera, - pub highlight_code: bool, - pub highlight_theme: String, - pub current_page_permalink: String, + pub config: &'a Config, + pub current_page_permalink: &'a str, pub permalinks: &'a HashMap, pub insert_anchor: InsertAnchor, } -impl<'a> Context<'a> { +impl<'a> RenderContext<'a> { pub fn new( tera: &'a Tera, - highlight_code: bool, - highlight_theme: String, - current_page_permalink: &str, + config: &'a Config, + current_page_permalink: &'a str, permalinks: &'a HashMap, insert_anchor: InsertAnchor, - ) -> Context<'a> { - Context { + ) -> RenderContext<'a> { + RenderContext { tera, - current_page_permalink: current_page_permalink.to_string(), + current_page_permalink, permalinks, insert_anchor, - highlight_code, - highlight_theme, + config, } } - pub fn should_insert_anchor(&self) -> bool { - self.insert_anchor != InsertAnchor::None - } } diff --git a/components/rendering/src/lib.rs b/components/rendering/src/lib.rs index 185c006a..00715af2 100644 --- a/components/rendering/src/lib.rs +++ b/components/rendering/src/lib.rs @@ -1,6 +1,3 @@ -#[macro_use] -extern crate lazy_static; -extern crate regex; extern crate tera; extern crate syntect; extern crate pulldown_cmark; @@ -12,7 +9,7 @@ extern crate pest; #[macro_use] extern crate pest_derive; - +#[macro_use] extern crate errors; extern crate front_matter; extern crate highlighting; @@ -26,9 +23,20 @@ mod context; mod markdown; mod table_of_contents; mod shortcode; -mod short_code; -pub use context::Context; -pub use markdown::markdown_to_html; +use errors::Result; + +use markdown::markdown_to_html; pub use table_of_contents::Header; pub use shortcode::render_shortcodes; +pub use context::RenderContext; + +pub fn render_content(content: &str, context: &RenderContext) -> Result<(String, Vec
)> { + // Don't do anything if there is nothing like a shortcode in the content + if content.contains("{{") || content.contains("{%") { + let rendered = render_shortcodes(content, context.tera, context.config)?; + return markdown_to_html(&rendered, context); + } + + markdown_to_html(&content, context) +} diff --git a/components/rendering/src/markdown.rs b/components/rendering/src/markdown.rs index b993dd70..a6ac28c9 100644 --- a/components/rendering/src/markdown.rs +++ b/components/rendering/src/markdown.rs @@ -8,37 +8,36 @@ use syntect::html::{start_coloured_html_snippet, styles_to_coloured_html, Includ use errors::Result; use utils::site::resolve_internal_link; -use context::Context; -use highlighting::{SYNTAX_SET, THEME_SET}; -use short_code::{SHORTCODE_RE, ShortCode, parse_shortcode, render_simple_shortcode}; +use highlighting::{get_highlighter, THEME_SET}; + use table_of_contents::{TempHeader, Header, make_table_of_contents}; +use context::RenderContext; + +// We might have cases where the slug is already present in our list of anchor +// for example an article could have several titles named Example +// We add a counter after the slug if the slug is already present, which +// means we will have example, example-1, example-2 etc +fn find_anchor(anchors: &[String], name: String, level: u8) -> String { + if level == 0 && !anchors.contains(&name) { + return name.to_string(); + } + + let new_anchor = format!("{}-{}", name, level + 1); + if !anchors.contains(&new_anchor) { + return new_anchor; + } + + find_anchor(anchors, name, level + 1) +} -pub fn markdown_to_html(content: &str, context: &Context) -> Result<(String, Vec
)> { - // We try to be smart about highlighting code as it can be time-consuming - // If the global config disables it, then we do nothing. However, - // if we see a code block in the content, we assume that this page needs - // to be highlighted. It could potentially have false positive if the content - // has ``` in it but that seems kind of unlikely - let should_highlight = if context.highlight_code { - content.contains("```") - } else { - false - }; +pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<(String, Vec
)> { + // the rendered html + let mut html = String::new(); // Set while parsing let mut error = None; + let mut highlighter: Option = None; - // the markdown parser will send several Text event if a markdown character - // is present in it, for example `hello_test` will be split in 2: hello and _test. - // Since we can use those chars in shortcode arguments, we need to collect - // the full shortcode somehow first - let mut current_shortcode = String::new(); - let mut shortcode_block = None; - // shortcodes live outside of paragraph so we need to ensure we don't close - // a paragraph that has already been closed - let mut added_shortcode = false; - // Don't transform things that look like shortcodes in code blocks - let mut in_code_block = false; // If we get text in header, we need to insert the id and a anchor let mut in_header = false; // pulldown_cmark can send several text events for a title if there are markdown @@ -46,254 +45,128 @@ pub fn markdown_to_html(content: &str, context: &Context) -> Result<(String, Vec let mut header_created = false; let mut anchors: Vec = vec![]; - // the rendered html - let mut html = String::new(); - - // We might have cases where the slug is already present in our list of anchor - // for example an article could have several titles named Example - // We add a counter after the slug if the slug is already present, which - // means we will have example, example-1, example-2 etc - fn find_anchor(anchors: &[String], name: String, level: u8) -> String { - if level == 0 && !anchors.contains(&name) { - return name.to_string(); - } - - let new_anchor = format!("{}-{}", name, level + 1); - if !anchors.contains(&new_anchor) { - return new_anchor; - } - - find_anchor(anchors, name, level + 1) - } - let mut headers = vec![]; // Defaults to a 0 level so not a real header // It should be an Option ideally but not worth the hassle to update let mut temp_header = TempHeader::default(); - let mut clear_shortcode_block = false; let mut opts = Options::empty(); opts.insert(OPTION_ENABLE_TABLES); opts.insert(OPTION_ENABLE_FOOTNOTES); { - let parser = Parser::new_ext(content, opts).map(|event| { - if clear_shortcode_block { - clear_shortcode_block = false; - shortcode_block = None; - } - match event { - Event::Text(mut text) => { - // Header first - if in_header { - if header_created { - temp_header.push(&text); + Event::Text(text) => { + // Header first + if in_header { + if header_created { + temp_header.push(&text); + return Event::Html(Owned(String::new())); + } + let id = find_anchor(&anchors, slugify(&text), 0); + anchors.push(id.clone()); + // update the header and add it to the list + temp_header.id = id.clone(); + // += as we might have some or other things already there + temp_header.title += &text; + temp_header.permalink = format!("{}#{}", context.current_page_permalink, id); + header_created = true; return Event::Html(Owned(String::new())); } - let id = find_anchor(&anchors, slugify(&text), 0); - anchors.push(id.clone()); - // update the header and add it to the list - temp_header.id = id.clone(); - // += as we might have some or other things already there - temp_header.title += &text; - temp_header.permalink = format!("{}#{}", context.current_page_permalink, id); - header_created = true; - return Event::Html(Owned(String::new())); - } - // if we are in the middle of a code block - if let Some(ref mut highlighter) = highlighter { - let highlighted = &highlighter.highlight(&text); - let html = styles_to_coloured_html(highlighted, IncludeBackground::Yes); - return Event::Html(Owned(html)); - } - - if in_code_block { - return Event::Text(text); - } - - // Are we in the middle of a shortcode that somehow got cut off - // by the markdown parser? - if current_shortcode.is_empty() { - if text.starts_with("{{") && !text.ends_with("}}") { - current_shortcode += &text; - } else if text.starts_with("{%") && !text.ends_with("%}") { - current_shortcode += &text; + // if we are in the middle of a code block + if let Some(ref mut highlighter) = highlighter { + let highlighted = &highlighter.highlight(&text); + let html = styles_to_coloured_html(highlighted, IncludeBackground::Yes); + return Event::Html(Owned(html)); } - } else { - current_shortcode += &text; - } - if current_shortcode.ends_with("}}") || current_shortcode.ends_with("%}") { - text = Owned(current_shortcode.clone()); - current_shortcode = String::new(); - } - - // Shortcode without body - if shortcode_block.is_none() && text.starts_with("{{") && text.ends_with("}}") && SHORTCODE_RE.is_match(&text) { - let (name, args) = parse_shortcode(&text); - - added_shortcode = true; - match render_simple_shortcode(context.tera, &name, &args) { - // Make before and after cleaning up of extra

/

tags more parallel. - // Or, in other words: - // TERRIBLE HORRIBLE NO GOOD VERY BAD HACK - Ok(s) => return Event::Html(Owned(format!("

{}

", s))), - Err(e) => { - error = Some(e); - return Event::Html(Owned(String::new())); - } + // Business as usual + Event::Text(text) + }, + Event::Start(Tag::CodeBlock(ref info)) => { + if !context.config.highlight_code { + return Event::Html(Owned("

".to_owned()));
                     }
-                }
-
-                // Shortcode with a body
-                if shortcode_block.is_none() && text.starts_with("{%") && text.ends_with("%}") {
-                    if SHORTCODE_RE.is_match(&text) {
-                        let (name, args) = parse_shortcode(&text);
-                        shortcode_block = Some(ShortCode::new(&name, args));
+                    let theme = &THEME_SET.themes[&context.config.highlight_theme];
+                    highlighter = Some(get_highlighter(&theme, info));
+                    let snippet = start_coloured_html_snippet(theme);
+                    Event::Html(Owned(snippet))
+                },
+                Event::End(Tag::CodeBlock(_)) => {
+                    if !context.config.highlight_code {
+                        return Event::Html(Owned("
\n".to_owned())) } - // Don't return anything - return Event::Text(Owned(String::new())); - } - - // If we have some text while in a shortcode, it's either the body - // or the end tag - if shortcode_block.is_some() { - if let Some(ref mut shortcode) = shortcode_block { - if text.trim() == "{% end %}" { - added_shortcode = true; - clear_shortcode_block = true; - match shortcode.render(context.tera) { - Ok(s) => return Event::Html(Owned(format!("

{}", s))), - Err(e) => { - error = Some(e); - return Event::Html(Owned(String::new())); - } + // reset highlight and close the code block + highlighter = None; + Event::Html(Owned("".to_owned())) + }, + // Need to handle relative links + Event::Start(Tag::Link(ref link, ref title)) => { + if in_header { + return Event::Html(Owned("".to_owned())); + } + if link.starts_with("./") { + match resolve_internal_link(link, context.permalinks) { + Ok(url) => { + return Event::Start(Tag::Link(Owned(url), title.clone())); + }, + Err(_) => { + error = Some(format!("Relative link {} not found.", link).into()); + return Event::Html(Owned("".to_string())); } - } else { - shortcode.append(&text); - return Event::Html(Owned(String::new())); - } + }; } - } - // Business as usual - Event::Text(text) - }, - Event::Start(Tag::CodeBlock(ref info)) => { - in_code_block = true; - if !should_highlight { - return Event::Html(Owned("
".to_owned()));
+                    Event::Start(Tag::Link(link.clone(), title.clone()))
+                },
+                Event::End(Tag::Link(_, _)) => {
+                    if in_header {
+                        return Event::Html(Owned("".to_owned()));
+                    }
+                    event
                 }
-                let theme = &THEME_SET.themes[&context.highlight_theme];
-                highlighter = SYNTAX_SET.with(|ss| {
-                    let syntax = info
-                        .split(' ')
-                        .next()
-                        .and_then(|lang| ss.find_syntax_by_token(lang))
-                        .unwrap_or_else(|| ss.find_syntax_plain_text());
-                    Some(HighlightLines::new(syntax, theme))
-                });
-                let snippet = start_coloured_html_snippet(theme);
-                Event::Html(Owned(snippet))
-            },
-            Event::End(Tag::CodeBlock(_)) => {
-                in_code_block = false;
-                if !should_highlight{
-                    return Event::Html(Owned("
\n".to_owned())) - } - // reset highlight and close the code block - highlighter = None; - Event::Html(Owned("".to_owned())) - }, - // Need to handle relative links - Event::Start(Tag::Link(ref link, ref title)) => { - if in_header { - return Event::Html(Owned("".to_owned())); - } - if link.starts_with("./") { - match resolve_internal_link(link, context.permalinks) { - Ok(url) => { - return Event::Start(Tag::Link(Owned(url), title.clone())); - }, - Err(_) => { - error = Some(format!("Relative link {} not found.", link).into()); - return Event::Html(Owned("".to_string())); - } - }; - } - - Event::Start(Tag::Link(link.clone(), title.clone())) - }, - Event::End(Tag::Link(_, _)) => { - if in_header { - return Event::Html(Owned("".to_owned())); - } - event + Event::Start(Tag::Code) => { + if in_header { + temp_header.push(""); + return Event::Html(Owned(String::new())); + } + event + }, + Event::End(Tag::Code) => { + if in_header { + temp_header.push(""); + return Event::Html(Owned(String::new())); + } + event + }, + Event::Start(Tag::Header(num)) => { + in_header = true; + temp_header = TempHeader::new(num); + Event::Html(Owned(String::new())) + }, + Event::End(Tag::Header(_)) => { + // End of a header, reset all the things and return the stringified version of the header + in_header = false; + header_created = false; + let val = temp_header.to_string(context.tera, context.insert_anchor); + headers.push(temp_header.clone()); + temp_header = TempHeader::default(); + Event::Html(Owned(val)) + }, + _ => event, } - // need to know when we are in a code block to disable shortcodes in them - Event::Start(Tag::Code) => { - in_code_block = true; - if in_header { - temp_header.push(""); - return Event::Html(Owned(String::new())); - } - event - }, - Event::End(Tag::Code) => { - in_code_block = false; - if in_header { - temp_header.push(""); - return Event::Html(Owned(String::new())); - } - event - }, - Event::Start(Tag::Header(num)) => { - in_header = true; - temp_header = TempHeader::new(num); - Event::Html(Owned(String::new())) - }, - Event::End(Tag::Header(_)) => { - // End of a header, reset all the things and return the stringified version of the header - in_header = false; - header_created = false; - let val = temp_header.to_string(context); - headers.push(temp_header.clone()); - temp_header = TempHeader::default(); - Event::Html(Owned(val)) - }, - // If we added shortcodes, don't close a paragraph since there's none - Event::End(Tag::Paragraph) => { - if added_shortcode { - added_shortcode = false; - return Event::Html(Owned("".to_owned())); - } - event - }, - // Ignore softbreaks inside shortcodes - Event::SoftBreak => { - if shortcode_block.is_some() { - return Event::Html(Owned("".to_owned())); - } - event - }, - _ => { - // println!("event = {:?}", event); - event - }, - }}); + }); cmark::html::push_html(&mut html, parser); } - if !current_shortcode.is_empty() { - return Err(format!("A shortcode was not closed properly:\n{:?}", current_shortcode).into()); - } - match error { Some(e) => Err(e), - None => Ok((html.replace("

", "").replace("

", "

"), make_table_of_contents(&headers))), + None => Ok(( + html.replace("

", "").replace("

", "

"), + make_table_of_contents(&headers) + )), } } diff --git a/components/rendering/src/short_code.rs b/components/rendering/src/short_code.rs deleted file mode 100644 index f1cd3fbb..00000000 --- a/components/rendering/src/short_code.rs +++ /dev/null @@ -1,190 +0,0 @@ -use std::collections::HashMap; - -use regex::Regex; -use tera::{Tera, Context, Value, to_value}; - -use errors::{Result, ResultExt}; - -lazy_static!{ - // Does this look like a shortcode? - pub static ref SHORTCODE_RE: Regex = Regex::new( - r#"\{(?:%|\{)\s+(\w+?)\((\w+?="?(?:.|\n)+?"?)?\)\s+(?:%|\})\}"# - ).unwrap(); - - // Parse the shortcode args with capture groups named after their type - pub static ref SHORTCODE_ARGS_RE: Regex = Regex::new( - r#"(?P\w+)=\s*((?P".*?")|(?P[-+]?[0-9]+\.[0-9]+)|(?P[-+]?[0-9]+)|(?Ptrue|false))"# - ).unwrap(); -} - -/// A shortcode that has a body -/// Called by having some content like {% ... %} body {% end %} -/// We need the struct to hold the data while we're processing the markdown -#[derive(Debug)] -pub struct ShortCode { - name: String, - args: HashMap, - body: String, -} - -impl ShortCode { - pub fn new(name: &str, args: HashMap) -> ShortCode { - ShortCode { - name: name.to_string(), - args, - body: String::new(), - } - } - - pub fn append(&mut self, text: &str) { - self.body.push_str(text) - } - - pub fn render(&self, tera: &Tera) -> Result { - let mut context = Context::new(); - for (key, value) in &self.args { - context.add(key, value); - } - context.add("body", &self.body); - let tpl_name = format!("shortcodes/{}.html", self.name); - tera.render(&tpl_name, &context) - .chain_err(|| format!("Failed to render {} shortcode", self.name)) - } -} - -/// Parse a shortcode without a body -pub fn parse_shortcode(input: &str) -> (String, HashMap) { - let mut args = HashMap::new(); - let caps = SHORTCODE_RE.captures(input).unwrap(); - // caps[0] is the full match - let name = &caps[1]; - - if let Some(arg_list) = caps.get(2) { - for arg_cap in SHORTCODE_ARGS_RE.captures_iter(arg_list.as_str()) { - let arg_name = arg_cap["name"].trim().to_string(); - - if let Some(arg_val) = arg_cap.name("str") { - args.insert(arg_name, to_value(arg_val.as_str().replace("\"", "")).unwrap()); - continue; - } - - if let Some(arg_val) = arg_cap.name("int") { - args.insert(arg_name, to_value(arg_val.as_str().parse::().unwrap()).unwrap()); - continue; - } - - if let Some(arg_val) = arg_cap.name("float") { - args.insert(arg_name, to_value(arg_val.as_str().parse::().unwrap()).unwrap()); - continue; - } - - if let Some(arg_val) = arg_cap.name("bool") { - args.insert(arg_name, to_value(arg_val.as_str() == "true").unwrap()); - continue; - } - } - } - - (name.to_string(), args) -} - -/// Renders a shortcode or return an error -pub fn render_simple_shortcode(tera: &Tera, name: &str, args: &HashMap) -> Result { - let mut context = Context::new(); - for (key, value) in args.iter() { - context.add(key, value); - } - let tpl_name = format!("shortcodes/{}.html", name); - - tera.render(&tpl_name, &context).chain_err(|| format!("Failed to render {} shortcode", name)) -} - - -#[cfg(test)] -mod tests { - use super::{parse_shortcode, SHORTCODE_RE}; - - #[test] - fn can_match_all_kinds_of_shortcode() { - let inputs = vec![ - "{{ basic() }}", - "{{ basic(ho=1) }}", - "{{ basic(ho=\"hey\") }}", - "{{ basic(ho=\"hey_underscore\") }}", - "{{ basic(ho=\"hey-dash\") }}", - "{% basic(ho=\"hey-dash\") %}", - "{% basic(ho=\"hey_underscore\") %}", - "{% basic() %}", - "{% quo_te(author=\"Bob\") %}", - "{{ quo_te(author=\"Bob\") }}", - // https://github.com/Keats/gutenberg/issues/229 - r#"{{ youtube(id="dQw4w9WgXcQ", - - autoplay=true) }}"#, - ]; - - for i in inputs { - println!("{}", i); - assert!(SHORTCODE_RE.is_match(i)); - } - } - - // https://github.com/Keats/gutenberg/issues/228 - #[test] - fn doesnt_panic_on_invalid_shortcode() { - let (name, args) = parse_shortcode(r#"{{ youtube(id="dQw4w9WgXcQ", autoplay) }}"#); - assert_eq!(name, "youtube"); - assert_eq!(args["id"], "dQw4w9WgXcQ"); - assert!(args.get("autoplay").is_none()); - } - - #[test] - fn can_parse_simple_shortcode_no_arg() { - let (name, args) = parse_shortcode(r#"{{ basic() }}"#); - assert_eq!(name, "basic"); - assert!(args.is_empty()); - } - - #[test] - fn can_parse_simple_shortcode_one_arg() { - let (name, args) = parse_shortcode(r#"{{ youtube(id="w7Ft2ymGmfc") }}"#); - assert_eq!(name, "youtube"); - assert_eq!(args["id"], "w7Ft2ymGmfc"); - } - - #[test] - fn can_parse_simple_shortcode_several_arg() { - let (name, args) = parse_shortcode(r#"{{ youtube(id="w7Ft2ymGmfc", autoplay=true) }}"#); - assert_eq!(name, "youtube"); - assert_eq!(args["id"], "w7Ft2ymGmfc"); - assert_eq!(args["autoplay"], true); - } - - #[test] - fn can_parse_block_shortcode_several_arg() { - let (name, args) = parse_shortcode(r#"{% youtube(id="w7Ft2ymGmfc", autoplay=true) %}"#); - assert_eq!(name, "youtube"); - assert_eq!(args["id"], "w7Ft2ymGmfc"); - assert_eq!(args["autoplay"], true); - } - - #[test] - fn can_parse_shortcode_number() { - let (name, args) = parse_shortcode(r#"{% test(int=42, float=42.0, autoplay=false) %}"#); - assert_eq!(name, "test"); - assert_eq!(args["int"], 42); - assert_eq!(args["float"], 42.0); - assert_eq!(args["autoplay"], false); - } - - // https://github.com/Keats/gutenberg/issues/249 - #[test] - fn can_parse_shortcode_with_comma_in_it() { - let (name, args) = parse_shortcode( - r#"{% quote(author="C++ Standard Core Language Defect Reports and Accepted Issues, Revision 82, delete and user-written deallocation function", href="http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#348") %}"# - ); - assert_eq!(name, "quote"); - assert_eq!(args["author"], "C++ Standard Core Language Defect Reports and Accepted Issues, Revision 82, delete and user-written deallocation function"); - assert_eq!(args["href"], "http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#348"); - } -} diff --git a/components/rendering/src/shortcode.rs b/components/rendering/src/shortcode.rs index 009a0cfe..4f0d0f1a 100644 --- a/components/rendering/src/shortcode.rs +++ b/components/rendering/src/shortcode.rs @@ -90,7 +90,8 @@ fn render_shortcode(name: String, args: Map, tera: &Tera, config: context.insert(key, value); } if let Some(ref b) = body { - context.insert("body", b); + // Trimming right to avoid most shortcodes with bodies ending up with a HTML new line + context.insert("body", b.trim_right()); } context.insert("config", config); let tpl_name = format!("shortcodes/{}.html", name); @@ -99,16 +100,25 @@ fn render_shortcode(name: String, args: Map, tera: &Tera, config: } pub fn render_shortcodes(content: &str, tera: &Tera, config: &Config) -> Result { - // Don't do anything if there is nothing like a shortcode in the content - if !content.contains("{{") && !content.contains("{%") { - return Ok(content.to_string()); - } - let mut res = String::with_capacity(content.len()); let mut pairs = match ContentParser::parse(Rule::page, content) { Ok(p) => p, - Err(_) => panic!("TODO"), // TODO: error handling + Err(e) => { + let fancy_e = e.renamed_rules(|rule| { + match *rule { + Rule::int => "an integer".to_string(), + Rule::float => "a float".to_string(), + Rule::string => "a string".to_string(), + Rule::literal => "a literal (int, float, string, bool)".to_string(), + Rule::array => "an array".to_string(), + Rule::kwarg => "a keyword argument".to_string(), + Rule::ident => "an identifier".to_string(), + _ => format!("TODO error: {:?}", rule).to_string(), + } + }); + bail!("{}", fancy_e); + }, }; // We have at least a `page` pair @@ -333,4 +343,10 @@ Hello World let res = render_shortcodes("Body\n {% youtube() %}Hey!{% end %}", &tera, &Config::default()).unwrap(); assert_eq!(res, "Body\n Hey!"); } + + #[test] + fn errors_on_unterminated_shortcode() { + let res = render_shortcodes("{{ youtube(", &Tera::default(), &Config::default()); + assert!(res.is_err()); + } } diff --git a/components/rendering/src/table_of_contents.rs b/components/rendering/src/table_of_contents.rs index 70f75954..688c64d9 100644 --- a/components/rendering/src/table_of_contents.rs +++ b/components/rendering/src/table_of_contents.rs @@ -1,8 +1,6 @@ -use tera::{Context as TeraContext}; +use tera::{Tera, Context as TeraContext}; use front_matter::InsertAnchor; -use context::Context; - #[derive(Debug, PartialEq, Clone, Serialize)] pub struct Header { @@ -50,16 +48,16 @@ impl TempHeader { } /// Transform all the information we have about this header into the HTML string for it - pub fn to_string(&self, context: &Context) -> String { - let anchor_link = if context.should_insert_anchor() { + pub fn to_string(&self, tera: &Tera, insert_anchor: InsertAnchor) -> String { + let anchor_link = if insert_anchor != InsertAnchor::None { let mut c = TeraContext::new(); c.add("id", &self.id); - context.tera.render("anchor-link.html", &c).unwrap() + tera.render("anchor-link.html", &c).unwrap() } else { String::new() }; - match context.insert_anchor { + match insert_anchor { InsertAnchor::None => format!("{t}\n", lvl=self.level, t=self.title, id=self.id), InsertAnchor::Left => format!("{a}{t}\n", lvl=self.level, a=anchor_link, t=self.title, id=self.id), InsertAnchor::Right => format!("{t}{a}\n", lvl=self.level, a=anchor_link, t=self.title, id=self.id), diff --git a/components/rendering/tests/markdown.rs b/components/rendering/tests/markdown.rs index 8dbdcc73..2e547f33 100644 --- a/components/rendering/tests/markdown.rs +++ b/components/rendering/tests/markdown.rs @@ -2,22 +2,25 @@ extern crate tera; extern crate front_matter; extern crate templates; extern crate rendering; +extern crate config; use std::collections::HashMap; use tera::Tera; +use config::Config; use front_matter::InsertAnchor; use templates::GUTENBERG_TERA; -use rendering::{Context, markdown_to_html}; +use rendering::{RenderContext, render_content}; #[test] -fn can_do_markdown_to_html_simple() { +fn can_do_render_content_simple() { let tera_ctx = Tera::default(); let permalinks_ctx = HashMap::new(); - let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None); - let res = markdown_to_html("hello", &context).unwrap(); + let config = Config::default(); + let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None); + let res = render_content("hello", &context).unwrap(); assert_eq!(res.0, "

hello

\n"); } @@ -25,9 +28,10 @@ fn can_do_markdown_to_html_simple() { fn doesnt_highlight_code_block_with_highlighting_off() { let tera_ctx = Tera::default(); let permalinks_ctx = HashMap::new(); - let mut context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None); - context.highlight_code = false; - let res = markdown_to_html("```\n$ gutenberg server\n```", &context).unwrap(); + let mut config = Config::default(); + config.highlight_code = false; + let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None); + let res = render_content("```\n$ gutenberg server\n```", &context).unwrap(); assert_eq!( res.0, "
$ gutenberg server\n
\n" @@ -38,8 +42,9 @@ fn doesnt_highlight_code_block_with_highlighting_off() { fn can_highlight_code_block_no_lang() { let tera_ctx = Tera::default(); let permalinks_ctx = HashMap::new(); - let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None); - let res = markdown_to_html("```\n$ gutenberg server\n$ ping\n```", &context).unwrap(); + let config = Config::default(); + let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None); + let res = render_content("```\n$ gutenberg server\n$ ping\n```", &context).unwrap(); assert_eq!( res.0, "
\n$ gutenberg server\n$ ping\n
" @@ -50,8 +55,9 @@ fn can_highlight_code_block_no_lang() { fn can_highlight_code_block_with_lang() { let tera_ctx = Tera::default(); let permalinks_ctx = HashMap::new(); - let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None); - let res = markdown_to_html("```python\nlist.append(1)\n```", &context).unwrap(); + let config = Config::default(); + let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None); + let res = render_content("```python\nlist.append(1)\n```", &context).unwrap(); assert_eq!( res.0, "
\nlist.append(1)\n
" @@ -62,8 +68,9 @@ fn can_highlight_code_block_with_lang() { fn can_higlight_code_block_with_unknown_lang() { let tera_ctx = Tera::default(); let permalinks_ctx = HashMap::new(); - let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None); - let res = markdown_to_html("```yolo\nlist.append(1)\n```", &context).unwrap(); + let config = Config::default(); + let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None); + let res = render_content("```yolo\nlist.append(1)\n```", &context).unwrap(); // defaults to plain text assert_eq!( res.0, @@ -74,8 +81,9 @@ fn can_higlight_code_block_with_unknown_lang() { #[test] fn can_render_shortcode() { let permalinks_ctx = HashMap::new(); - let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None); - let res = markdown_to_html(r#" + let config = Config::default(); + let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::None); + let res = render_content(r#" Hello {{ youtube(id="ub36ffWAqgQ") }} @@ -87,7 +95,8 @@ Hello #[test] fn can_render_shortcode_with_markdown_char_in_args_name() { let permalinks_ctx = HashMap::new(); - let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None); + let config = Config::default(); + let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::None); let input = vec![ "name", "na_me", @@ -95,7 +104,7 @@ fn can_render_shortcode_with_markdown_char_in_args_name() { "n1", ]; for i in input { - let res = markdown_to_html(&format!("{{{{ youtube(id=\"hey\", {}=1) }}}}", i), &context).unwrap(); + let res = render_content(&format!("{{{{ youtube(id=\"hey\", {}=1) }}}}", i), &context).unwrap(); assert!(res.0.contains(r#"