Re-use markdown parser for markdown filter

This commit is contained in:
Vincent Prouillet 2020-11-28 13:04:49 +01:00
parent 59d3d8a3da
commit f20c52b872
10 changed files with 129 additions and 99 deletions

86
Cargo.lock generated
View file

@ -343,7 +343,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils 0.8.0",
"crossbeam-utils",
]
[[package]]
@ -354,18 +354,18 @@ checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-epoch",
"crossbeam-utils 0.8.0",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.0"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0f606a85340376eef0d6d8fec399e6d4a544d648386c6645eb6d0653b27d9f"
checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d"
dependencies = [
"cfg-if 1.0.0",
"const_fn",
"crossbeam-utils 0.8.0",
"crossbeam-utils",
"lazy_static",
"memoffset",
"scopeguard",
@ -373,32 +373,20 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.7.2"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [
"autocfg",
"cfg-if 0.1.10",
"lazy_static",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec91540d98355f690a86367e566ecad2e9e579f230230eb7c21398372be73ea5"
checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d"
dependencies = [
"autocfg",
"cfg-if 1.0.0",
"const_fn",
"lazy_static",
]
[[package]]
name = "csv"
version = "1.1.4"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc4666154fd004af3fd6f1da2e81a96fd5a81927fe8ddb6ecc79e2aa6e138b54"
checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97"
dependencies = [
"bstr",
"csv-core",
@ -1026,11 +1014,11 @@ dependencies = [
[[package]]
name = "ignore"
version = "0.4.16"
version = "0.4.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22dcbf2a4a289528dbef21686354904e1c694ac642610a9bff9e7df730d9ec72"
checksum = "b287fb45c60bb826a0dc68ff08742b9d88a2fea13d6e0c286b3172065aaf878c"
dependencies = [
"crossbeam-utils 0.7.2",
"crossbeam-utils",
"globset",
"lazy_static",
"log",
@ -1381,9 +1369,9 @@ checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "memoffset"
version = "0.5.6"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa"
checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
dependencies = [
"autocfg",
]
@ -1467,9 +1455,9 @@ dependencies = [
[[package]]
name = "miow"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d"
dependencies = [
"kernel32-sys",
"net2",
@ -1497,9 +1485,9 @@ dependencies = [
[[package]]
name = "net2"
version = "0.2.35"
version = "0.2.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853"
checksum = "d7cf75f38f16cb05ea017784dc6dbfd354f76c223dba37701734c4f5a9337d02"
dependencies = [
"cfg-if 0.1.10",
"libc",
@ -1668,9 +1656,9 @@ checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
[[package]]
name = "onig"
version = "6.1.0"
version = "6.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a155d13862da85473665694f4c05d77fb96598bdceeaf696433c84ea9567e20"
checksum = "30b46fd9edbc018f0be4e366c24c46db44fac49cd01c039ae85308088b089dd5"
dependencies = [
"bitflags",
"lazy_static",
@ -1680,9 +1668,9 @@ dependencies = [
[[package]]
name = "onig_sys"
version = "69.5.1"
version = "69.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9bff06597a6b17855040955cae613af000fc0bfc8ad49ea68b9479a74e59292d"
checksum = "ed063c96cf4c0f2e5d09324409d158b38a0a85a7b90fbd68c8cad75c495d5775"
dependencies = [
"cc",
"pkg-config",
@ -2049,7 +2037,7 @@ checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils 0.8.0",
"crossbeam-utils",
"lazy_static",
"num_cpus",
]
@ -2169,9 +2157,9 @@ dependencies = [
[[package]]
name = "ring"
version = "0.16.16"
version = "0.16.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b72b84d47e8ec5a4f2872e8262b8f8256c5be1c938a7d6d3a867a3ba8f722f74"
checksum = "70017ed5c555d79ee3538fc63ca09c70ad8f317dcadc1adc2c496b60c22bb24f"
dependencies = [
"cc",
"libc",
@ -2440,11 +2428,11 @@ checksum = "7acad6f34eb9e8a259d3283d1e8c1d34d7415943d4895f65cc73813c7396fc85"
[[package]]
name = "socket2"
version = "0.3.16"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fd8b795c389288baa5f355489c65e71fd48a02104600d15c4cfbc561e9e429d"
checksum = "2c29947abdee2a218277abeca306f25789c938e500ea5a9d4b12a5a504466902"
dependencies = [
"cfg-if 0.1.10",
"cfg-if 1.0.0",
"libc",
"redox_syscall",
"winapi 0.3.9",
@ -2525,9 +2513,9 @@ dependencies = [
[[package]]
name = "syn"
version = "1.0.48"
version = "1.0.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac"
checksum = "6c1e438504729046a5cfae47f97c30d6d083c7d91d94603efdae3477fc070d4c"
dependencies = [
"proc-macro2",
"quote",
@ -2584,7 +2572,7 @@ dependencies = [
"library",
"mockito",
"nom-bibtex",
"pulldown-cmark",
"rendering",
"reqwest",
"serde_json",
"sha2",
@ -2765,13 +2753,13 @@ checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860"
[[package]]
name = "tracing"
version = "0.1.21"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27"
checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3"
dependencies = [
"cfg-if 0.1.10",
"cfg-if 1.0.0",
"log",
"pin-project-lite 0.1.11",
"pin-project-lite 0.2.0",
"tracing-core",
]
@ -2891,9 +2879,9 @@ dependencies = [
[[package]]
name = "unicode-segmentation"
version = "1.7.0"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db8716a166f290ff49dabc18b44aa407cb7c6dbe1aa0971b44b8a24b0ca35aae"
checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
[[package]]
name = "unicode-width"

View file

@ -1,3 +1,4 @@
use std::borrow::Cow;
use std::collections::HashMap;
use config::Config;
@ -7,11 +8,11 @@ use tera::{Context, Tera};
/// All the information from the zola site that is needed to render HTML from markdown
#[derive(Debug)]
pub struct RenderContext<'a> {
pub tera: &'a Tera,
pub tera: Cow<'a, Tera>,
pub config: &'a Config,
pub tera_context: Context,
pub current_page_permalink: &'a str,
pub permalinks: &'a HashMap<String, String>,
pub permalinks: Cow<'a, HashMap<String, String>>,
pub insert_anchor: InsertAnchor,
}
@ -25,13 +26,25 @@ impl<'a> RenderContext<'a> {
) -> RenderContext<'a> {
let mut tera_context = Context::new();
tera_context.insert("config", config);
RenderContext {
tera,
Self {
tera: Cow::Borrowed(tera),
tera_context,
current_page_permalink,
permalinks,
permalinks: Cow::Borrowed(permalinks),
insert_anchor,
config,
}
}
// In use in the markdown filter
pub fn from_config(config: &'a Config) -> RenderContext<'a> {
Self {
tera: Cow::Owned(Tera::default()),
tera_context: Context::new(),
current_page_permalink: "",
permalinks: Cow::Owned(HashMap::new()),
insert_anchor: InsertAnchor::None,
config,
}
}
}

View file

@ -105,7 +105,7 @@ fn fix_link(
// - it could be a link to a co-located asset
// - it could be a normal link
let result = if link.starts_with("@/") {
match resolve_internal_link(&link, context.permalinks) {
match resolve_internal_link(&link, &context.permalinks) {
Ok(resolved) => {
if resolved.anchor.is_some() {
internal_links_with_anchors
@ -384,7 +384,7 @@ pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<Render
let anchor_link = utils::templates::render_template(
&ANCHOR_LINK_TEMPLATE,
context.tera,
&context.tera,
c,
&None,
)

View file

@ -1125,4 +1125,4 @@ fn can_use_smart_punctuation() {
let context = RenderContext::new(&ZOLA_TERA, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content(r#"This -- is "it"..."#, &context).unwrap();
assert_eq!(res.body, "<p>This is “it”…</p>\n");
}
}

View file

@ -5,7 +5,7 @@ use tera::Tera;
use crate::Site;
use config::Config;
use errors::{bail, Error, Result};
use templates::{global_fns, ZOLA_TERA};
use templates::{filters, global_fns, ZOLA_TERA};
use utils::templates::rewrite_theme_paths;
pub fn load_tera(path: &Path, config: &Config) -> Result<Tera> {
@ -50,6 +50,8 @@ pub fn load_tera(path: &Path, config: &Config) -> Result<Tera> {
/// Adds global fns that are to be available to shortcodes while rendering markdown
pub fn register_early_global_fns(site: &mut Site) {
site.tera.register_filter("markdown", filters::MarkdownFilter::new(site.config.clone()));
site.tera.register_function(
"get_url",
global_fns::GetUrl::new(

View file

@ -8,7 +8,6 @@ edition = "2018"
tera = "1"
base64 = "0.13"
lazy_static = "1"
pulldown-cmark = { version = "0.8", default-features = false }
toml = "0.5"
csv = "1"
image = "0.23"
@ -16,13 +15,14 @@ serde_json = "1.0"
sha2 = "0.9"
url = "2"
nom-bibtex = "0.3"
svg_metadata = "0.4.1"
errors = { path = "../errors" }
utils = { path = "../utils" }
library = { path = "../library" }
config = { path = "../config" }
imageproc = { path = "../imageproc" }
svg_metadata = "0.4.1"
rendering = { path = "../rendering" }
[dependencies.reqwest]
version = "0.10"

View file

@ -2,38 +2,44 @@ use std::collections::HashMap;
use std::hash::BuildHasher;
use base64::{decode, encode};
use pulldown_cmark as cmark;
use tera::{to_value, try_get_value, Result as TeraResult, Value};
use config::Config;
use rendering::{render_content, RenderContext};
use tera::{to_value, try_get_value, Filter as TeraFilter, Result as TeraResult, Value};
pub fn markdown<S: BuildHasher>(
value: &Value,
args: &HashMap<String, Value, S>,
) -> TeraResult<Value> {
let s = try_get_value!("markdown", "value", String, value);
let inline = match args.get("inline") {
Some(val) => try_get_value!("markdown", "inline", bool, val),
None => false,
};
#[derive(Debug)]
pub struct MarkdownFilter {
config: Config,
}
let mut opts = cmark::Options::empty();
opts.insert(cmark::Options::ENABLE_TABLES);
opts.insert(cmark::Options::ENABLE_FOOTNOTES);
opts.insert(cmark::Options::ENABLE_STRIKETHROUGH);
opts.insert(cmark::Options::ENABLE_TASKLISTS);
let mut html = String::new();
let parser = cmark::Parser::new_ext(&s, opts);
cmark::html::push_html(&mut html, parser);
if inline {
html = html
.trim_start_matches("<p>")
// pulldown_cmark finishes a paragraph with `</p>\n`
.trim_end_matches("</p>\n")
.to_string();
impl MarkdownFilter {
pub fn new(config: Config) -> Self {
Self { config }
}
}
Ok(to_value(&html).unwrap())
impl TeraFilter for MarkdownFilter {
fn filter(&self, value: &Value, args: &HashMap<String, Value>) -> TeraResult<Value> {
let context = RenderContext::from_config(&self.config);
let s = try_get_value!("markdown", "value", String, value);
let inline = match args.get("inline") {
Some(val) => try_get_value!("markdown", "inline", bool, val),
None => false,
};
let mut html = match render_content(&s, &context) {
Ok(res) => res.body,
Err(e) => return Err(format!("Failed to render markdown filter: {:?}", e).into()),
};
if inline {
html = html
.trim_start_matches("<p>")
// pulldown_cmark finishes a paragraph with `</p>\n`
.trim_end_matches("</p>\n")
.to_string();
}
Ok(to_value(&html).unwrap())
}
}
pub fn base64_encode<S: BuildHasher>(
@ -56,22 +62,24 @@ pub fn base64_decode<S: BuildHasher>(
mod tests {
use std::collections::HashMap;
use tera::to_value;
use tera::{to_value, Filter};
use super::{base64_decode, base64_encode, markdown};
use super::{base64_decode, base64_encode, MarkdownFilter};
use config::Config;
#[test]
fn markdown_filter() {
let result = markdown(&to_value(&"# Hey").unwrap(), &HashMap::new());
let result = MarkdownFilter::new(Config::default())
.filter(&to_value(&"# Hey").unwrap(), &HashMap::new());
assert!(result.is_ok());
assert_eq!(result.unwrap(), to_value(&"<h1>Hey</h1>\n").unwrap());
assert_eq!(result.unwrap(), to_value(&"<h1 id=\"hey\">Hey</h1>\n").unwrap());
}
#[test]
fn markdown_filter_inline() {
let mut args = HashMap::new();
args.insert("inline".to_string(), to_value(true).unwrap());
let result = markdown(
let result = MarkdownFilter::new(Config::default()).filter(
&to_value(&"Using `map`, `filter`, and `fold` instead of `for`").unwrap(),
&args,
);
@ -84,7 +92,7 @@ mod tests {
fn markdown_filter_inline_tables() {
let mut args = HashMap::new();
args.insert("inline".to_string(), to_value(true).unwrap());
let result = markdown(
let result = MarkdownFilter::new(Config::default()).filter(
&to_value(
&r#"
|id|author_id| timestamp_created|title |content |
@ -100,6 +108,26 @@ mod tests {
assert!(result.unwrap().as_str().unwrap().contains("<table>"));
}
#[test]
fn markdown_filter_use_config_options() {
let mut config = Config::default();
config.markdown.highlight_code = true;
config.markdown.smart_punctuation = true;
config.markdown.render_emoji = true;
config.markdown.external_links_target_blank = true;
let md = "Hello <https://google.com> :smile: ...";
let result =
MarkdownFilter::new(config.clone()).filter(&to_value(&md).unwrap(), &HashMap::new());
assert!(result.is_ok());
assert_eq!(result.unwrap(), to_value(&"<p>Hello <a rel=\"noopener\" target=\"_blank\" href=\"https://google.com\">https://google.com</a> 😄 …</p>\n").unwrap());
let md = "```py\ni=0\n```";
let result = MarkdownFilter::new(config).filter(&to_value(&md).unwrap(), &HashMap::new());
assert!(result.is_ok());
assert!(result.unwrap().as_str().unwrap().contains("<pre style"));
}
#[test]
fn base64_encode_filter() {
// from https://tools.ietf.org/html/rfc4648#section-10

View file

@ -168,7 +168,7 @@ fn get_output_format_from_args(
};
// Always default to Plain if we don't know what it is
OutputFormat::from_str(from_extension).or_else(|_| Ok(OutputFormat::Plain))
OutputFormat::from_str(from_extension).or(Ok(OutputFormat::Plain))
}
/// A Tera function to load data from a file or from a URL

View file

@ -39,7 +39,7 @@ impl TeraFn for Trans {
let term = self
.config
.get_translation(lang, key)
.map_err(|e| Error::chain("Failed to retreive term translation", e))?;
.map_err(|e| Error::chain("Failed to retrieve term translation", e))?;
Ok(to_value(term).unwrap())
}
@ -735,7 +735,7 @@ title = "A title"
let config = Config::parse(TRANS_CONFIG).unwrap();
let error = Trans::new(config).call(&args).unwrap_err();
assert_eq!("Failed to retreive term translation", format!("{}", error));
assert_eq!("Failed to retrieve term translation", format!("{}", error));
}
#[test]
@ -746,7 +746,7 @@ title = "A title"
let config = Config::parse(TRANS_CONFIG).unwrap();
let error = Trans::new(config).call(&args).unwrap_err();
assert_eq!("Failed to retreive term translation", format!("{}", error));
assert_eq!("Failed to retrieve term translation", format!("{}", error));
}
#[test]

View file

@ -36,7 +36,6 @@ lazy_static! {
("internal/alias.html", include_str!("builtins/internal/alias.html")),
])
.unwrap();
tera.register_filter("markdown", filters::markdown);
tera.register_filter("base64_encode", filters::base64_encode);
tera.register_filter("base64_decode", filters::base64_decode);
tera