diff --git a/components/rendering/Cargo.toml b/components/rendering/Cargo.toml index e9a19e29..92ff5c08 100644 --- a/components/rendering/Cargo.toml +++ b/components/rendering/Cargo.toml @@ -12,11 +12,14 @@ pulldown-cmark = "0" slug = "0.1" serde = "1" serde_derive = "1" +pest = "1" +pest_derive = "1" errors = { path = "../errors" } front_matter = { path = "../front_matter" } highlighting = { path = "../highlighting"} utils = { path = "../utils" } +config = { path = "../config" } [dev-dependencies] templates = { path = "../templates" } diff --git a/components/rendering/benches/all.rs b/components/rendering/benches/all.rs index 999268cf..e7a968b0 100644 --- a/components/rendering/benches/all.rs +++ b/components/rendering/benches/all.rs @@ -3,13 +3,15 @@ extern crate test; extern crate tera; extern crate rendering; +extern crate config; extern crate front_matter; use std::collections::HashMap; use tera::Tera; -use rendering::{Context, markdown_to_html}; +use rendering::{Context, markdown_to_html, render_shortcodes}; use front_matter::InsertAnchor; +use config::Config; static CONTENT: &'static str = r#" # Modus cognitius profanam ne duae virtutis mundi @@ -98,3 +100,20 @@ fn bench_markdown_to_html_without_highlighting(b: &mut test::Bencher) { let context = Context::new(&tera_ctx, false, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None); b.iter(|| markdown_to_html(CONTENT, &context)); } + +#[bench] +fn bench_render_shortcodes_one_present(b: &mut test::Bencher) { + let mut tera = Tera::default(); + tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap(); + + b.iter(|| render_shortcodes(CONTENT, &tera, &Config::default())); +} + +#[bench] +fn bench_render_shortcodes_none(b: &mut test::Bencher) { + let mut tera = Tera::default(); + tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap(); + let content2 = CONTENT.replace(r#"{{ youtube(id="my_youtube_id") }}"#, ""); + + b.iter(|| render_shortcodes(&content2, &tera, &Config::default())); +} diff --git a/components/rendering/src/content.pest b/components/rendering/src/content.pest new file mode 100644 index 00000000..edd3403a --- /dev/null +++ b/components/rendering/src/content.pest @@ -0,0 +1,74 @@ +// Partly taken from Tera + +whitespace = _{ " " | "\t" | "\r" | "\n" } + +/// LITERALS +int = @{ "-" ? ~ ("0" | '1'..'9' ~ '0'..'9' * ) } +float = @{ + "-" ? ~ + ( + "0" ~ "." ~ '0'..'9' + | + '1'..'9' ~ '0'..'9' * ~ "." ~ '0'..'9' + + ) +} +// matches anything between 2 double quotes +double_quoted_string = @{ "\"" ~ (!("\"") ~ any)* ~ "\""} +// matches anything between 2 single quotes +single_quoted_string = @{ "\'" ~ (!("\'") ~ any)* ~ "\'"} +// matches anything between 2 backquotes\backticks +backquoted_quoted_string = @{ "`" ~ (!("`") ~ any)* ~ "`"} + +string = @{ + double_quoted_string | + single_quoted_string | + backquoted_quoted_string +} + +boolean = { "true" | "false" } + +literal = { boolean | string | float | int } +array = { "[" ~ (literal ~ ",")* ~ literal? ~ "]"} + +/// Idents + +all_chars = _{'a'..'z' | 'A'..'Z' | "_" | '0'..'9'} +ident = @{ + ('a'..'z' | 'A'..'Z' | "_") ~ + all_chars* +} + +/// Now specific to Gutenberg + +// shortcode is abbreviated to sc to keep things short + +kwarg = { ident ~ "=" ~ (literal | array) } +kwargs = _{ kwarg ~ ("," ~ kwarg )* } +sc_def = _{ ident ~ "(" ~ kwargs* ~ ")" } + +//sc_start = _{ "{{/*" | "{{" | "{%/*" | "{%" } + +inline_shortcode = !{ "{{" ~ sc_def ~ "}}" } +ignored_inline_shortcode = !{ "{{/*" ~ sc_def ~ "*/}}" } + +sc_body_start = !{ "{%" ~ sc_def ~ "%}" } +sc_body_end = !{ "{%" ~ "end" ~ "%}" } +ignored_sc_body_start = !{ "{%/*" ~ sc_def ~ "*/%}" } +ignored_sc_body_end = !{ "{%/*" ~ "end" ~ "*/%}" } + +shortcode_with_body = !{ sc_body_start ~ text_in_body_sc ~ sc_body_end } +ignored_shortcode_with_body = !{ ignored_sc_body_start ~ text_in_ignored_body_sc ~ ignored_sc_body_end } + +text_in_body_sc = ${ (!("{%") ~ any)+ } +text_in_ignored_body_sc = ${ (!("{%/*") ~ any)+ } +text = ${ (!("{{/*" | "{{" | "{%/*" | "{%") ~ any)+ } + +content = _{ + ignored_inline_shortcode | + inline_shortcode | + ignored_shortcode_with_body | + shortcode_with_body | + text +} + + +page = ${ soi ~ content* ~ eoi } diff --git a/components/rendering/src/lib.rs b/components/rendering/src/lib.rs index 1dfd39b5..185c006a 100644 --- a/components/rendering/src/lib.rs +++ b/components/rendering/src/lib.rs @@ -8,20 +8,27 @@ extern crate slug; #[macro_use] extern crate serde_derive; extern crate serde; +extern crate pest; +#[macro_use] +extern crate pest_derive; + extern crate errors; extern crate front_matter; extern crate highlighting; extern crate utils; +extern crate config; #[cfg(test)] extern crate templates; mod context; mod markdown; -mod short_code; mod table_of_contents; +mod shortcode; +mod short_code; pub use context::Context; pub use markdown::markdown_to_html; pub use table_of_contents::Header; +pub use shortcode::render_shortcodes; diff --git a/components/rendering/src/shortcode.rs b/components/rendering/src/shortcode.rs new file mode 100644 index 00000000..009a0cfe --- /dev/null +++ b/components/rendering/src/shortcode.rs @@ -0,0 +1,336 @@ +use pest::Parser; +use pest::iterators::Pair; +use tera::{Tera, Map, Context, Value, to_value}; + +use errors::{Result, ResultExt}; +use config::Config; + +// This include forces recompiling this source file if the grammar file changes. +// Uncomment it when doing changes to the .pest file +// const _GRAMMAR: &str = include_str!("content.pest"); + +#[derive(Parser)] +#[grammar = "content.pest"] +pub struct ContentParser; + + +fn replace_string_markers(input: &str) -> String { + match input.chars().next().unwrap() { + '"' => input.replace('"', "").to_string(), + '\'' => input.replace('\'', "").to_string(), + '`' => input.replace('`', "").to_string(), + _ => unreachable!("How did you even get there"), + } +} + +fn parse_literal(pair: Pair) -> Value { + let mut val = None; + for p in pair.into_inner() { + match p.as_rule() { + Rule::boolean => match p.as_str() { + "true" => val = Some(Value::Bool(true)), + "false" => val = Some(Value::Bool(false)), + _ => unreachable!(), + }, + Rule::string => val = Some(Value::String(replace_string_markers(p.as_str()))), + Rule::float => { + val = Some(to_value(p.as_str().parse::().unwrap()).unwrap()); + }, + Rule::int => { + val = Some(to_value(p.as_str().parse::().unwrap()).unwrap()); + }, + _ => unreachable!("Unknown literal: {:?}", p) + }; + } + + val.unwrap() +} + +/// Returns (shortcode_name, kwargs) +fn parse_shortcode_call(pair: Pair) -> (String, Map) { + let mut name = None; + let mut args = Map::new(); + + for p in pair.into_inner() { + match p.as_rule() { + Rule::ident => { name = Some(p.into_span().as_str().to_string()); }, + Rule::kwarg => { + let mut arg_name = None; + let mut arg_val = None; + for p2 in p.into_inner() { + match p2.as_rule() { + Rule::ident => { arg_name = Some(p2.into_span().as_str().to_string()); }, + Rule::literal => { arg_val = Some(parse_literal(p2)); }, + Rule::array => { + let mut vals = vec![]; + for p3 in p2.into_inner() { + match p3.as_rule() { + Rule::literal => vals.push(parse_literal(p3)), + _ => unreachable!("Got something other than literal in an array: {:?}", p3), + } + } + arg_val = Some(Value::Array(vals)); + }, + _ => unreachable!("Got something unexpected in a kwarg: {:?}", p2), + } + } + + args.insert(arg_name.unwrap(), arg_val.unwrap()); + }, + _ => unreachable!("Got something unexpected in a shortcode: {:?}", p) + } + } + (name.unwrap(), args) +} + + +fn render_shortcode(name: String, args: Map, tera: &Tera, config: &Config, body: Option<&str>) -> Result { + let mut context = Context::new(); + for (key, value) in args.iter() { + context.insert(key, value); + } + if let Some(ref b) = body { + context.insert("body", b); + } + context.insert("config", config); + let tpl_name = format!("shortcodes/{}.html", name); + + tera.render(&tpl_name, &context).chain_err(|| format!("Failed to render {} shortcode", name)) +} + +pub fn render_shortcodes(content: &str, tera: &Tera, config: &Config) -> Result { + // Don't do anything if there is nothing like a shortcode in the content + if !content.contains("{{") && !content.contains("{%") { + return Ok(content.to_string()); + } + + let mut res = String::with_capacity(content.len()); + + let mut pairs = match ContentParser::parse(Rule::page, content) { + Ok(p) => p, + Err(_) => panic!("TODO"), // TODO: error handling + }; + + // We have at least a `page` pair + for p in pairs.next().unwrap().into_inner() { + match p.as_rule() { + Rule::text | Rule::text_in_ignored_body_sc | Rule::text_in_body_sc => res.push_str(p.into_span().as_str()), + Rule::inline_shortcode => { + let (name, args) = parse_shortcode_call(p); + res.push_str(&render_shortcode(name, args, tera, config, None)?); + }, + Rule::shortcode_with_body => { + let mut inner = p.into_inner(); + // 3 items in inner: call, body, end + // we don't care about the closing tag + let (name, args) = parse_shortcode_call(inner.next().unwrap()); + let body = inner.next().unwrap().into_span().as_str(); + res.push_str(&render_shortcode(name, args, tera, config, Some(body))?); + }, + Rule::ignored_inline_shortcode => { + res.push_str( + &p.into_span().as_str() + .replacen("{{/*", "{{", 1) + .replacen("*/}}", "}}", 1) + ); + }, + Rule::ignored_shortcode_with_body => { + for p2 in p.into_inner() { + match p2.as_rule() { + Rule::ignored_sc_body_start | Rule::ignored_sc_body_end => { + res.push_str( + &p2.into_span().as_str() + .replacen("{%/*", "{%", 1) + .replacen("*/%}", "%}", 1) + ); + }, + Rule::text_in_ignored_body_sc => res.push_str(p2.into_span().as_str()), + _ => unreachable!("Got something weird in an ignored shortcode"), + } + } + }, + _ => unreachable!("unexpected page rule: {:?}", p.as_rule()), + } + } + + Ok(res) +} + +#[cfg(test)] +mod tests { + use super::*; + + macro_rules! assert_lex_rule { + ($rule: expr, $input: expr) => { + let res = ContentParser::parse($rule, $input); + println!("{:?}", $input); + println!("{:#?}", res); + if res.is_err() { + println!("{}", res.unwrap_err()); + panic!(); + } + assert!(res.is_ok()); + assert_eq!(res.unwrap().last().unwrap().into_span().end(), $input.len()); + }; + } + + #[test] + fn lex_text() { + let inputs = vec!["Hello world", "HEllo \n world", "Hello 1 2 true false 'hey'"]; + for i in inputs { + assert_lex_rule!(Rule::text, i); + } + } + + #[test] + fn lex_inline_shortcode() { + let inputs = vec![ + "{{ youtube() }}", + "{{ youtube(id=1, autoplay=true, url='hey') }}", + "{{ youtube(id=1, \nautoplay=true, url='hey') }}", + ]; + for i in inputs { + assert_lex_rule!(Rule::inline_shortcode, i); + } + } + + #[test] + fn lex_inline_ignored_shortcode() { + let inputs = vec![ + "{{/* youtube() */}}", + "{{/* youtube(id=1, autoplay=true, url='hey') */}}", + "{{/* youtube(id=1, \nautoplay=true, \nurl='hey') */}}", + ]; + for i in inputs { + assert_lex_rule!(Rule::ignored_inline_shortcode, i); + } + } + + #[test] + fn lex_shortcode_with_body() { + let inputs = vec![ + r#"{% youtube() %} + Some text + {% end %}"#, + + r#"{% youtube(id=1, + autoplay=true, url='hey') %} + Some text + {% end %}"#, + ]; + for i in inputs { + assert_lex_rule!(Rule::shortcode_with_body, i); + } + } + + #[test] + fn lex_ignored_shortcode_with_body() { + let inputs = vec![ + r#"{%/* youtube() */%} + Some text + {%/* end */%}"#, + + r#"{%/* youtube(id=1, + autoplay=true, url='hey') */%} + Some text + {%/* end */%}"#, + ]; + for i in inputs { + assert_lex_rule!(Rule::ignored_shortcode_with_body, i); + } + } + + #[test] + fn lex_page() { + let inputs = vec![ + "Some text and a shortcode `{{/* youtube() */}}`", + "{{ youtube(id=1, autoplay=true, url='hey') }}", + "{{ youtube(id=1, \nautoplay=true, url='hey') }} that's it", + r#" + This is a test + {% hello() %} + Body {{ var }} + {% end %} + "# + ]; + for i in inputs { + assert_lex_rule!(Rule::page, i); + } + } + + #[test] + fn does_nothing_with_no_shortcodes() { + let res = render_shortcodes("Hello World", &Tera::default(), &Config::default()); + assert_eq!(res.unwrap(), "Hello World"); + } + + #[test] + fn can_unignore_inline_shortcode() { + let res = render_shortcodes( + "Hello World {{/* youtube() */}}", + &Tera::default(), + &Config::default(), + ); + assert_eq!(res.unwrap(), "Hello World {{ youtube() }}"); + } + + #[test] + fn can_unignore_shortcode_with_body() { + let res = render_shortcodes(r#" +Hello World +{%/* youtube() */%}Some body {{ hello() }}{%/* end */%}"#, &Tera::default(), &Config::default()); + assert_eq!(res.unwrap(), "\nHello World\n{% youtube() %}Some body {{ hello() }}{% end %}"); + } + + #[test] + fn can_parse_shortcode_arguments() { + let inputs = vec![ + ("{{ youtube() }}", "youtube", Map::new()), + ( + "{{ youtube(id=1, autoplay=true, hello='salut', float=1.2) }}", + "youtube", + { + let mut m = Map::new(); + m.insert("id".to_string(), to_value(1).unwrap()); + m.insert("autoplay".to_string(), to_value(true).unwrap()); + m.insert("hello".to_string(), to_value("salut").unwrap()); + m.insert("float".to_string(), to_value(1.2).unwrap()); + m + } + ), + ( + "{{ gallery(photos=['something', 'else'], fullscreen=true) }}", + "gallery", + { + let mut m = Map::new(); + m.insert("photos".to_string(), to_value(["something", "else"]).unwrap()); + m.insert("fullscreen".to_string(), to_value(true).unwrap()); + m + } + ), + ]; + + for (i, n, a) in inputs { + let mut res = ContentParser::parse(Rule::inline_shortcode, i).unwrap(); + let (name, args) = parse_shortcode_call(res.next().unwrap()); + assert_eq!(name, n); + assert_eq!(args, a); + } + } + + #[test] + fn can_render_inline_shortcodes() { + let mut tera = Tera::default(); + tera.add_raw_template("shortcodes/youtube.html", "Hello {{id}}").unwrap(); + let res = render_shortcodes("Inline {{ youtube(id=1) }}.", &tera, &Config::default()).unwrap(); + assert_eq!(res, "Inline Hello 1."); + } + + #[test] + fn can_render_shortcodes_with_body() { + let mut tera = Tera::default(); + tera.add_raw_template("shortcodes/youtube.html", "{{body}}").unwrap(); + let res = render_shortcodes("Body\n {% youtube() %}Hey!{% end %}", &tera, &Config::default()).unwrap(); + assert_eq!(res, "Body\n Hey!"); + } +}