WIP: new markdown rendering

This commit is contained in:
Vincent Prouillet 2018-05-03 20:50:30 +02:00
parent b892c07ed3
commit 204f514c0e
5 changed files with 441 additions and 2 deletions

View file

@ -12,11 +12,14 @@ pulldown-cmark = "0"
slug = "0.1" slug = "0.1"
serde = "1" serde = "1"
serde_derive = "1" serde_derive = "1"
pest = "1"
pest_derive = "1"
errors = { path = "../errors" } errors = { path = "../errors" }
front_matter = { path = "../front_matter" } front_matter = { path = "../front_matter" }
highlighting = { path = "../highlighting"} highlighting = { path = "../highlighting"}
utils = { path = "../utils" } utils = { path = "../utils" }
config = { path = "../config" }
[dev-dependencies] [dev-dependencies]
templates = { path = "../templates" } templates = { path = "../templates" }

View file

@ -3,13 +3,15 @@ extern crate test;
extern crate tera; extern crate tera;
extern crate rendering; extern crate rendering;
extern crate config;
extern crate front_matter; extern crate front_matter;
use std::collections::HashMap; use std::collections::HashMap;
use tera::Tera; use tera::Tera;
use rendering::{Context, markdown_to_html}; use rendering::{Context, markdown_to_html, render_shortcodes};
use front_matter::InsertAnchor; use front_matter::InsertAnchor;
use config::Config;
static CONTENT: &'static str = r#" static CONTENT: &'static str = r#"
# Modus cognitius profanam ne duae virtutis mundi # Modus cognitius profanam ne duae virtutis mundi
@ -98,3 +100,20 @@ fn bench_markdown_to_html_without_highlighting(b: &mut test::Bencher) {
let context = Context::new(&tera_ctx, false, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None); let context = Context::new(&tera_ctx, false, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
b.iter(|| markdown_to_html(CONTENT, &context)); b.iter(|| markdown_to_html(CONTENT, &context));
} }
#[bench]
fn bench_render_shortcodes_one_present(b: &mut test::Bencher) {
let mut tera = Tera::default();
tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap();
b.iter(|| render_shortcodes(CONTENT, &tera, &Config::default()));
}
#[bench]
fn bench_render_shortcodes_none(b: &mut test::Bencher) {
let mut tera = Tera::default();
tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap();
let content2 = CONTENT.replace(r#"{{ youtube(id="my_youtube_id") }}"#, "");
b.iter(|| render_shortcodes(&content2, &tera, &Config::default()));
}

View file

@ -0,0 +1,74 @@
// Partly taken from Tera
whitespace = _{ " " | "\t" | "\r" | "\n" }
/// LITERALS
int = @{ "-" ? ~ ("0" | '1'..'9' ~ '0'..'9' * ) }
float = @{
"-" ? ~
(
"0" ~ "." ~ '0'..'9' + |
'1'..'9' ~ '0'..'9' * ~ "." ~ '0'..'9' +
)
}
// matches anything between 2 double quotes
double_quoted_string = @{ "\"" ~ (!("\"") ~ any)* ~ "\""}
// matches anything between 2 single quotes
single_quoted_string = @{ "\'" ~ (!("\'") ~ any)* ~ "\'"}
// matches anything between 2 backquotes\backticks
backquoted_quoted_string = @{ "`" ~ (!("`") ~ any)* ~ "`"}
string = @{
double_quoted_string |
single_quoted_string |
backquoted_quoted_string
}
boolean = { "true" | "false" }
literal = { boolean | string | float | int }
array = { "[" ~ (literal ~ ",")* ~ literal? ~ "]"}
/// Idents
all_chars = _{'a'..'z' | 'A'..'Z' | "_" | '0'..'9'}
ident = @{
('a'..'z' | 'A'..'Z' | "_") ~
all_chars*
}
/// Now specific to Gutenberg
// shortcode is abbreviated to sc to keep things short
kwarg = { ident ~ "=" ~ (literal | array) }
kwargs = _{ kwarg ~ ("," ~ kwarg )* }
sc_def = _{ ident ~ "(" ~ kwargs* ~ ")" }
//sc_start = _{ "{{/*" | "{{" | "{%/*" | "{%" }
inline_shortcode = !{ "{{" ~ sc_def ~ "}}" }
ignored_inline_shortcode = !{ "{{/*" ~ sc_def ~ "*/}}" }
sc_body_start = !{ "{%" ~ sc_def ~ "%}" }
sc_body_end = !{ "{%" ~ "end" ~ "%}" }
ignored_sc_body_start = !{ "{%/*" ~ sc_def ~ "*/%}" }
ignored_sc_body_end = !{ "{%/*" ~ "end" ~ "*/%}" }
shortcode_with_body = !{ sc_body_start ~ text_in_body_sc ~ sc_body_end }
ignored_shortcode_with_body = !{ ignored_sc_body_start ~ text_in_ignored_body_sc ~ ignored_sc_body_end }
text_in_body_sc = ${ (!("{%") ~ any)+ }
text_in_ignored_body_sc = ${ (!("{%/*") ~ any)+ }
text = ${ (!("{{/*" | "{{" | "{%/*" | "{%") ~ any)+ }
content = _{
ignored_inline_shortcode |
inline_shortcode |
ignored_shortcode_with_body |
shortcode_with_body |
text
}
page = ${ soi ~ content* ~ eoi }

View file

@ -8,20 +8,27 @@ extern crate slug;
#[macro_use] #[macro_use]
extern crate serde_derive; extern crate serde_derive;
extern crate serde; extern crate serde;
extern crate pest;
#[macro_use]
extern crate pest_derive;
extern crate errors; extern crate errors;
extern crate front_matter; extern crate front_matter;
extern crate highlighting; extern crate highlighting;
extern crate utils; extern crate utils;
extern crate config;
#[cfg(test)] #[cfg(test)]
extern crate templates; extern crate templates;
mod context; mod context;
mod markdown; mod markdown;
mod short_code;
mod table_of_contents; mod table_of_contents;
mod shortcode;
mod short_code;
pub use context::Context; pub use context::Context;
pub use markdown::markdown_to_html; pub use markdown::markdown_to_html;
pub use table_of_contents::Header; pub use table_of_contents::Header;
pub use shortcode::render_shortcodes;

View file

@ -0,0 +1,336 @@
use pest::Parser;
use pest::iterators::Pair;
use tera::{Tera, Map, Context, Value, to_value};
use errors::{Result, ResultExt};
use config::Config;
// This include forces recompiling this source file if the grammar file changes.
// Uncomment it when doing changes to the .pest file
// const _GRAMMAR: &str = include_str!("content.pest");
#[derive(Parser)]
#[grammar = "content.pest"]
pub struct ContentParser;
fn replace_string_markers(input: &str) -> String {
match input.chars().next().unwrap() {
'"' => input.replace('"', "").to_string(),
'\'' => input.replace('\'', "").to_string(),
'`' => input.replace('`', "").to_string(),
_ => unreachable!("How did you even get there"),
}
}
fn parse_literal(pair: Pair<Rule>) -> Value {
let mut val = None;
for p in pair.into_inner() {
match p.as_rule() {
Rule::boolean => match p.as_str() {
"true" => val = Some(Value::Bool(true)),
"false" => val = Some(Value::Bool(false)),
_ => unreachable!(),
},
Rule::string => val = Some(Value::String(replace_string_markers(p.as_str()))),
Rule::float => {
val = Some(to_value(p.as_str().parse::<f64>().unwrap()).unwrap());
},
Rule::int => {
val = Some(to_value(p.as_str().parse::<i64>().unwrap()).unwrap());
},
_ => unreachable!("Unknown literal: {:?}", p)
};
}
val.unwrap()
}
/// Returns (shortcode_name, kwargs)
fn parse_shortcode_call(pair: Pair<Rule>) -> (String, Map<String, Value>) {
let mut name = None;
let mut args = Map::new();
for p in pair.into_inner() {
match p.as_rule() {
Rule::ident => { name = Some(p.into_span().as_str().to_string()); },
Rule::kwarg => {
let mut arg_name = None;
let mut arg_val = None;
for p2 in p.into_inner() {
match p2.as_rule() {
Rule::ident => { arg_name = Some(p2.into_span().as_str().to_string()); },
Rule::literal => { arg_val = Some(parse_literal(p2)); },
Rule::array => {
let mut vals = vec![];
for p3 in p2.into_inner() {
match p3.as_rule() {
Rule::literal => vals.push(parse_literal(p3)),
_ => unreachable!("Got something other than literal in an array: {:?}", p3),
}
}
arg_val = Some(Value::Array(vals));
},
_ => unreachable!("Got something unexpected in a kwarg: {:?}", p2),
}
}
args.insert(arg_name.unwrap(), arg_val.unwrap());
},
_ => unreachable!("Got something unexpected in a shortcode: {:?}", p)
}
}
(name.unwrap(), args)
}
fn render_shortcode(name: String, args: Map<String, Value>, tera: &Tera, config: &Config, body: Option<&str>) -> Result<String> {
let mut context = Context::new();
for (key, value) in args.iter() {
context.insert(key, value);
}
if let Some(ref b) = body {
context.insert("body", b);
}
context.insert("config", config);
let tpl_name = format!("shortcodes/{}.html", name);
tera.render(&tpl_name, &context).chain_err(|| format!("Failed to render {} shortcode", name))
}
pub fn render_shortcodes(content: &str, tera: &Tera, config: &Config) -> Result<String> {
// Don't do anything if there is nothing like a shortcode in the content
if !content.contains("{{") && !content.contains("{%") {
return Ok(content.to_string());
}
let mut res = String::with_capacity(content.len());
let mut pairs = match ContentParser::parse(Rule::page, content) {
Ok(p) => p,
Err(_) => panic!("TODO"), // TODO: error handling
};
// We have at least a `page` pair
for p in pairs.next().unwrap().into_inner() {
match p.as_rule() {
Rule::text | Rule::text_in_ignored_body_sc | Rule::text_in_body_sc => res.push_str(p.into_span().as_str()),
Rule::inline_shortcode => {
let (name, args) = parse_shortcode_call(p);
res.push_str(&render_shortcode(name, args, tera, config, None)?);
},
Rule::shortcode_with_body => {
let mut inner = p.into_inner();
// 3 items in inner: call, body, end
// we don't care about the closing tag
let (name, args) = parse_shortcode_call(inner.next().unwrap());
let body = inner.next().unwrap().into_span().as_str();
res.push_str(&render_shortcode(name, args, tera, config, Some(body))?);
},
Rule::ignored_inline_shortcode => {
res.push_str(
&p.into_span().as_str()
.replacen("{{/*", "{{", 1)
.replacen("*/}}", "}}", 1)
);
},
Rule::ignored_shortcode_with_body => {
for p2 in p.into_inner() {
match p2.as_rule() {
Rule::ignored_sc_body_start | Rule::ignored_sc_body_end => {
res.push_str(
&p2.into_span().as_str()
.replacen("{%/*", "{%", 1)
.replacen("*/%}", "%}", 1)
);
},
Rule::text_in_ignored_body_sc => res.push_str(p2.into_span().as_str()),
_ => unreachable!("Got something weird in an ignored shortcode"),
}
}
},
_ => unreachable!("unexpected page rule: {:?}", p.as_rule()),
}
}
Ok(res)
}
#[cfg(test)]
mod tests {
use super::*;
macro_rules! assert_lex_rule {
($rule: expr, $input: expr) => {
let res = ContentParser::parse($rule, $input);
println!("{:?}", $input);
println!("{:#?}", res);
if res.is_err() {
println!("{}", res.unwrap_err());
panic!();
}
assert!(res.is_ok());
assert_eq!(res.unwrap().last().unwrap().into_span().end(), $input.len());
};
}
#[test]
fn lex_text() {
let inputs = vec!["Hello world", "HEllo \n world", "Hello 1 2 true false 'hey'"];
for i in inputs {
assert_lex_rule!(Rule::text, i);
}
}
#[test]
fn lex_inline_shortcode() {
let inputs = vec![
"{{ youtube() }}",
"{{ youtube(id=1, autoplay=true, url='hey') }}",
"{{ youtube(id=1, \nautoplay=true, url='hey') }}",
];
for i in inputs {
assert_lex_rule!(Rule::inline_shortcode, i);
}
}
#[test]
fn lex_inline_ignored_shortcode() {
let inputs = vec![
"{{/* youtube() */}}",
"{{/* youtube(id=1, autoplay=true, url='hey') */}}",
"{{/* youtube(id=1, \nautoplay=true, \nurl='hey') */}}",
];
for i in inputs {
assert_lex_rule!(Rule::ignored_inline_shortcode, i);
}
}
#[test]
fn lex_shortcode_with_body() {
let inputs = vec![
r#"{% youtube() %}
Some text
{% end %}"#,
r#"{% youtube(id=1,
autoplay=true, url='hey') %}
Some text
{% end %}"#,
];
for i in inputs {
assert_lex_rule!(Rule::shortcode_with_body, i);
}
}
#[test]
fn lex_ignored_shortcode_with_body() {
let inputs = vec![
r#"{%/* youtube() */%}
Some text
{%/* end */%}"#,
r#"{%/* youtube(id=1,
autoplay=true, url='hey') */%}
Some text
{%/* end */%}"#,
];
for i in inputs {
assert_lex_rule!(Rule::ignored_shortcode_with_body, i);
}
}
#[test]
fn lex_page() {
let inputs = vec![
"Some text and a shortcode `{{/* youtube() */}}`",
"{{ youtube(id=1, autoplay=true, url='hey') }}",
"{{ youtube(id=1, \nautoplay=true, url='hey') }} that's it",
r#"
This is a test
{% hello() %}
Body {{ var }}
{% end %}
"#
];
for i in inputs {
assert_lex_rule!(Rule::page, i);
}
}
#[test]
fn does_nothing_with_no_shortcodes() {
let res = render_shortcodes("Hello World", &Tera::default(), &Config::default());
assert_eq!(res.unwrap(), "Hello World");
}
#[test]
fn can_unignore_inline_shortcode() {
let res = render_shortcodes(
"Hello World {{/* youtube() */}}",
&Tera::default(),
&Config::default(),
);
assert_eq!(res.unwrap(), "Hello World {{ youtube() }}");
}
#[test]
fn can_unignore_shortcode_with_body() {
let res = render_shortcodes(r#"
Hello World
{%/* youtube() */%}Some body {{ hello() }}{%/* end */%}"#, &Tera::default(), &Config::default());
assert_eq!(res.unwrap(), "\nHello World\n{% youtube() %}Some body {{ hello() }}{% end %}");
}
#[test]
fn can_parse_shortcode_arguments() {
let inputs = vec![
("{{ youtube() }}", "youtube", Map::new()),
(
"{{ youtube(id=1, autoplay=true, hello='salut', float=1.2) }}",
"youtube",
{
let mut m = Map::new();
m.insert("id".to_string(), to_value(1).unwrap());
m.insert("autoplay".to_string(), to_value(true).unwrap());
m.insert("hello".to_string(), to_value("salut").unwrap());
m.insert("float".to_string(), to_value(1.2).unwrap());
m
}
),
(
"{{ gallery(photos=['something', 'else'], fullscreen=true) }}",
"gallery",
{
let mut m = Map::new();
m.insert("photos".to_string(), to_value(["something", "else"]).unwrap());
m.insert("fullscreen".to_string(), to_value(true).unwrap());
m
}
),
];
for (i, n, a) in inputs {
let mut res = ContentParser::parse(Rule::inline_shortcode, i).unwrap();
let (name, args) = parse_shortcode_call(res.next().unwrap());
assert_eq!(name, n);
assert_eq!(args, a);
}
}
#[test]
fn can_render_inline_shortcodes() {
let mut tera = Tera::default();
tera.add_raw_template("shortcodes/youtube.html", "Hello {{id}}").unwrap();
let res = render_shortcodes("Inline {{ youtube(id=1) }}.", &tera, &Config::default()).unwrap();
assert_eq!(res, "Inline Hello 1.");
}
#[test]
fn can_render_shortcodes_with_body() {
let mut tera = Tera::default();
tera.add_raw_template("shortcodes/youtube.html", "{{body}}").unwrap();
let res = render_shortcodes("Body\n {% youtube() %}Hey!{% end %}", &tera, &Config::default()).unwrap();
assert_eq!(res, "Body\n Hey!");
}
}