Merge pull request #369 from remexre/feature/runtime-syntax-loading

Runtime syntax loading
This commit is contained in:
Vincent Prouillet 2018-08-15 10:29:48 +02:00 committed by GitHub
commit ce813fab1b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 176 additions and 40 deletions

View file

@ -3,23 +3,22 @@ extern crate serde_derive;
extern crate toml;
#[macro_use]
extern crate errors;
extern crate highlighting;
extern crate chrono;
extern crate globset;
extern crate highlighting;
use std::collections::HashMap;
use std::fs::File;
use std::io::prelude::*;
use std::path::{Path, PathBuf};
use toml::Value as Toml;
use chrono::Utc;
use globset::{Glob, GlobSet, GlobSetBuilder};
use toml::Value as Toml;
use errors::{Result, ResultExt};
use highlighting::THEME_SET;
mod theme;
use theme::Theme;
@ -27,7 +26,6 @@ use theme::Theme;
// We want a default base url for tests
static DEFAULT_BASE_URL: &'static str = "http://a-website.com";
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct Taxonomy {
@ -101,12 +99,15 @@ pub struct Config {
/// Had to remove the PartialEq derive because GlobSet does not implement it. No impact
/// because it's unused anyway (who wants to sort Configs?).
pub ignored_content: Vec<String>,
#[serde(skip_serializing, skip_deserializing)] // not a typo, 2 are needed
#[serde(skip_serializing, skip_deserializing)] // not a typo, 2 are needed
pub ignored_content_globset: Option<GlobSet>,
/// Whether to check all external links for validity
pub check_external_links: bool,
/// A list of directories to search for additional `.sublime-syntax` files in.
pub extra_syntaxes: Vec<String>,
/// All user params set in [extra] in the config
pub extra: HashMap<String, Toml>,
@ -114,14 +115,13 @@ pub struct Config {
pub build_timestamp: Option<i64>,
}
impl Config {
/// Parses a string containing TOML to our Config struct
/// Any extra parameter will end up in the extra field
pub fn parse(content: &str) -> Result<Config> {
let mut config: Config = match toml::from_str(content) {
Ok(c) => c,
Err(e) => bail!(e)
Err(e) => bail!(e),
};
if config.base_url.is_empty() || config.base_url == DEFAULT_BASE_URL {
@ -134,7 +134,6 @@ impl Config {
config.build_timestamp = Some(Utc::now().timestamp());
if !config.ignored_content.is_empty() {
// Convert the file glob strings into a compiled glob set matcher. We want to do this once,
// at program initialization, rather than for every page, for example. We arrange for the
@ -145,11 +144,19 @@ impl Config {
for pat in &config.ignored_content {
let glob = match Glob::new(pat) {
Ok(g) => g,
Err(e) => bail!("Invalid ignored_content glob pattern: {}, error = {}", pat, e)
Err(e) => bail!(
"Invalid ignored_content glob pattern: {}, error = {}",
pat,
e
),
};
glob_set_builder.add(glob);
}
config.ignored_content_globset = Some(glob_set_builder.build().expect("Bad ignored_content in config file."));
config.ignored_content_globset = Some(
glob_set_builder
.build()
.expect("Bad ignored_content in config file."),
);
}
Ok(config)
@ -161,7 +168,12 @@ impl Config {
let path = path.as_ref();
let file_name = path.file_name().unwrap();
File::open(path)
.chain_err(|| format!("No `{:?}` file found. Are you in the right directory?", file_name))?
.chain_err(|| {
format!(
"No `{:?}` file found. Are you in the right directory?",
file_name
)
})?
.read_to_string(&mut content)?;
Config::parse(&content)
@ -169,7 +181,11 @@ impl Config {
/// Makes a url, taking into account that the base url might have a trailing slash
pub fn make_permalink(&self, path: &str) -> String {
let trailing_bit = if path.ends_with('/') || path.is_empty() { "" } else { "/" };
let trailing_bit = if path.ends_with('/') || path.is_empty() {
""
} else {
"/"
};
// Index section with a base url that has a trailing slash
if self.base_url.ends_with('/') && path == "/" {
@ -195,12 +211,16 @@ impl Config {
let original = self.extra.clone();
// 2. inject theme extra values
for (key, val) in &theme.extra {
self.extra.entry(key.to_string()).or_insert_with(|| val.clone());
self.extra
.entry(key.to_string())
.or_insert_with(|| val.clone());
}
// 3. overwrite with original config
for (key, val) in &original {
self.extra.entry(key.to_string()).or_insert_with(|| val.clone());
self.extra
.entry(key.to_string())
.or_insert_with(|| val.clone());
}
Ok(())
@ -233,13 +253,13 @@ impl Default for Config {
ignored_content: Vec::new(),
ignored_content_globset: None,
translations: HashMap::new(),
extra_syntaxes: Vec::new(),
extra: HashMap::new(),
build_timestamp: Some(1),
}
}
}
/// Get and parse the config.
/// If it doesn't succeed, exit
pub fn get_config(path: &Path, filename: &str) -> Config {
@ -253,7 +273,6 @@ pub fn get_config(path: &Path, filename: &str) -> Config {
}
}
#[cfg(test)]
mod tests {
use super::{Config, Theme};
@ -303,7 +322,16 @@ hello = "world"
let config = Config::parse(config);
assert!(config.is_ok());
assert_eq!(config.unwrap().extra.get("hello").unwrap().as_str().unwrap(), "world");
assert_eq!(
config
.unwrap()
.extra
.get("hello")
.unwrap()
.as_str()
.unwrap(),
"world"
);
}
#[test]
@ -313,7 +341,6 @@ hello = "world"
assert_eq!(config.make_permalink(""), "http://vincent.is/");
}
#[test]
fn can_make_url_index_page_with_railing_slash_url() {
let mut config = Config::default();
@ -339,7 +366,10 @@ hello = "world"
fn can_make_url_with_localhost() {
let mut config = Config::default();
config.base_url = "http://127.0.0.1:1111".to_string();
assert_eq!(config.make_permalink("/tags/rust"), "http://127.0.0.1:1111/tags/rust/");
assert_eq!(
config.make_permalink("/tags/rust"),
"http://127.0.0.1:1111/tags/rust/"
);
}
#[test]

View file

@ -166,12 +166,13 @@ impl Page {
/// We need access to all pages url to render links relative to content
/// so that can't happen at the same time as parsing
pub fn render_markdown(&mut self, permalinks: &HashMap<String, String>, tera: &Tera, config: &Config, anchor_insert: InsertAnchor) -> Result<()> {
pub fn render_markdown(&mut self, permalinks: &HashMap<String, String>, tera: &Tera, config: &Config, base_path: &Path, anchor_insert: InsertAnchor) -> Result<()> {
let mut context = RenderContext::new(
tera,
config,
&self.permalink,
permalinks,
base_path,
anchor_insert,
);

View file

@ -123,12 +123,13 @@ impl Section {
/// We need access to all pages url to render links relative to content
/// so that can't happen at the same time as parsing
pub fn render_markdown(&mut self, permalinks: &HashMap<String, String>, tera: &Tera, config: &Config) -> Result<()> {
pub fn render_markdown(&mut self, permalinks: &HashMap<String, String>, tera: &Tera, config: &Config, base_path: &Path) -> Result<()> {
let mut context = RenderContext::new(
tera,
config,
&self.permalink,
permalinks,
base_path,
self.meta.insert_anchor_links,
);

View file

@ -2,16 +2,21 @@
extern crate lazy_static;
extern crate syntect;
use std::cell::RefCell;
use std::path::Path;
use syntect::LoadingError;
use syntect::dumps::from_binary;
use syntect::parsing::SyntaxSet;
use syntect::highlighting::{ThemeSet, Theme};
use syntect::easy::HighlightLines;
thread_local! {
pub static SYNTAX_SET: SyntaxSet = {
/// A pair of the set and whether extras have been added to it.
pub static SYNTAX_SET: RefCell<(SyntaxSet, bool)> = {
let mut ss: SyntaxSet = from_binary(include_bytes!("../../../sublime_syntaxes/newlines.packdump"));
ss.link_syntaxes();
ss
RefCell::new((ss, false))
};
}
@ -19,14 +24,22 @@ lazy_static! {
pub static ref THEME_SET: ThemeSet = from_binary(include_bytes!("../../../sublime_themes/all.themedump"));
}
pub fn get_highlighter<'a>(theme: &'a Theme, info: &str, base_path: &Path, extra_syntaxes: &[String]) -> Result<HighlightLines<'a>, LoadingError> {
SYNTAX_SET.with(|rc| {
let (ss, extras_added) = &mut *rc.borrow_mut();
if !*extras_added {
for dir in extra_syntaxes {
ss.load_syntaxes(base_path.join(dir), true)?;
}
ss.link_syntaxes();
*extras_added = true;
}
pub fn get_highlighter<'a>(theme: &'a Theme, info: &str) -> HighlightLines<'a> {
SYNTAX_SET.with(|ss| {
let syntax = info
.split(' ')
.next()
.and_then(|lang| ss.find_syntax_by_token(lang))
.unwrap_or_else(|| ss.find_syntax_plain_text());
HighlightLines::new(syntax, theme)
Ok(HighlightLines::new(syntax, theme))
})
}

View file

@ -1,4 +1,5 @@
use std::collections::HashMap;
use std::path::Path;
use tera::{Tera, Context};
use front_matter::InsertAnchor;
@ -13,6 +14,7 @@ pub struct RenderContext<'a> {
pub tera_context: Context,
pub current_page_permalink: &'a str,
pub permalinks: &'a HashMap<String, String>,
pub base_path: &'a Path,
pub insert_anchor: InsertAnchor,
}
@ -22,6 +24,7 @@ impl<'a> RenderContext<'a> {
config: &'a Config,
current_page_permalink: &'a str,
permalinks: &'a HashMap<String, String>,
base_path: &'a Path,
insert_anchor: InsertAnchor,
) -> RenderContext<'a> {
let mut tera_context = Context::new();
@ -32,6 +35,7 @@ impl<'a> RenderContext<'a> {
current_page_permalink,
permalinks,
insert_anchor,
base_path,
config,
}
}

View file

@ -97,7 +97,13 @@ pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<(Strin
}
let theme = &THEME_SET.themes[&context.config.highlight_theme];
highlighter = Some(get_highlighter(&theme, info));
match get_highlighter(&theme, info, context.base_path, &context.config.extra_syntaxes) {
Ok(h) => highlighter = Some(h),
Err(err) => {
error = Some(format!("Could not load syntax: {}", err).into());
return Event::Html(Owned(String::new()));
}
}
let snippet = start_coloured_html_snippet(theme);
Event::Html(Owned(snippet))
}

View file

@ -258,6 +258,7 @@ impl Site {
let permalinks = &self.permalinks;
let tera = &self.tera;
let config = &self.config;
let base_path = &self.base_path;
// TODO: avoid the duplication with function above for that part
// This is needed in the first place because of silly borrow checker
@ -269,13 +270,13 @@ impl Site {
self.pages.par_iter_mut()
.map(|(_, page)| {
let insert_anchor = pages_insert_anchors[&page.file.path];
page.render_markdown(permalinks, tera, config, insert_anchor)
page.render_markdown(permalinks, tera, config, base_path, insert_anchor)
})
.fold(|| Ok(()), Result::and)
.reduce(|| Ok(()), Result::and)?;
self.sections.par_iter_mut()
.map(|(_, section)| section.render_markdown(permalinks, tera, config))
.map(|(_, section)| section.render_markdown(permalinks, tera, config, base_path))
.fold(|| Ok(()), Result::and)
.reduce(|| Ok(()), Result::and)?;
@ -318,7 +319,7 @@ impl Site {
if render {
let insert_anchor = self.find_parent_section_insert_anchor(&self.pages[&path].file.parent);
let page = self.pages.get_mut(&path).unwrap();
page.render_markdown(&self.permalinks, &self.tera, &self.config, insert_anchor)?;
page.render_markdown(&self.permalinks, &self.tera, &self.config, &self.base_path, insert_anchor)?;
}
Ok(prev)
@ -335,7 +336,7 @@ impl Site {
if render {
let section = self.sections.get_mut(&path).unwrap();
section.render_markdown(&self.permalinks, &self.tera, &self.config)?;
section.render_markdown(&self.permalinks, &self.tera, &self.config, &self.base_path)?;
}
Ok(prev)

View file

@ -19,7 +19,7 @@ fn can_parse_site() {
site.load().unwrap();
// Correct number of pages (sections are pages too)
assert_eq!(site.pages.len(), 14);
assert_eq!(site.pages.len(), 15);
let posts_path = path.join("content").join("posts");
// Make sure we remove all the pwd + content from the sections
@ -44,7 +44,7 @@ fn can_parse_site() {
let posts_section = &site.sections[&posts_path.join("_index.md")];
assert_eq!(posts_section.subsections.len(), 1);
assert_eq!(posts_section.pages.len(), 6);
assert_eq!(posts_section.pages.len(), 7);
let tutorials_section = &site.sections[&posts_path.join("tutorials").join("_index.md")];
assert_eq!(tutorials_section.subsections.len(), 2);
@ -321,22 +321,41 @@ fn can_build_site_with_pagination_for_section() {
"posts/page/1/index.html",
"http-equiv=\"refresh\" content=\"0;url=https://replace-this-with-your-url.com/posts/\""
));
assert!(file_contains!(public, "posts/index.html", "Num pagers: 3"));
assert!(file_contains!(public, "posts/index.html", "Num pagers: 4"));
assert!(file_contains!(public, "posts/index.html", "Page size: 2"));
assert!(file_contains!(public, "posts/index.html", "Current index: 1"));
assert!(!file_contains!(public, "posts/index.html", "has_prev"));
assert!(file_contains!(public, "posts/index.html", "has_next"));
assert!(file_contains!(public, "posts/index.html", "First: https://replace-this-with-your-url.com/posts/"));
assert!(file_contains!(public, "posts/index.html", "Last: https://replace-this-with-your-url.com/posts/page/3/"));
assert!(file_contains!(public, "posts/index.html", "Last: https://replace-this-with-your-url.com/posts/page/4/"));
assert_eq!(file_contains!(public, "posts/index.html", "has_prev"), false);
assert!(file_exists!(public, "posts/page/2/index.html"));
assert!(file_contains!(public, "posts/page/2/index.html", "Num pagers: 3"));
assert!(file_contains!(public, "posts/page/2/index.html", "Num pagers: 4"));
assert!(file_contains!(public, "posts/page/2/index.html", "Page size: 2"));
assert!(file_contains!(public, "posts/page/2/index.html", "Current index: 2"));
assert!(file_contains!(public, "posts/page/2/index.html", "has_prev"));
assert!(file_contains!(public, "posts/page/2/index.html", "has_next"));
assert!(file_contains!(public, "posts/page/2/index.html", "First: https://replace-this-with-your-url.com/posts/"));
assert!(file_contains!(public, "posts/page/2/index.html", "Last: https://replace-this-with-your-url.com/posts/page/3/"));
assert!(file_contains!(public, "posts/page/2/index.html", "Last: https://replace-this-with-your-url.com/posts/page/4/"));
assert!(file_exists!(public, "posts/page/3/index.html"));
assert!(file_contains!(public, "posts/page/3/index.html", "Num pagers: 4"));
assert!(file_contains!(public, "posts/page/3/index.html", "Page size: 2"));
assert!(file_contains!(public, "posts/page/3/index.html", "Current index: 3"));
assert!(file_contains!(public, "posts/page/3/index.html", "has_prev"));
assert!(file_contains!(public, "posts/page/3/index.html", "has_next"));
assert!(file_contains!(public, "posts/page/3/index.html", "First: https://replace-this-with-your-url.com/posts/"));
assert!(file_contains!(public, "posts/page/3/index.html", "Last: https://replace-this-with-your-url.com/posts/page/4/"));
assert!(file_exists!(public, "posts/page/4/index.html"));
assert!(file_contains!(public, "posts/page/4/index.html", "Num pagers: 4"));
assert!(file_contains!(public, "posts/page/4/index.html", "Page size: 2"));
assert!(file_contains!(public, "posts/page/4/index.html", "Current index: 4"));
assert!(file_contains!(public, "posts/page/4/index.html", "has_prev"));
assert!(!file_contains!(public, "posts/page/4/index.html", "has_next"));
assert!(file_contains!(public, "posts/page/4/index.html", "First: https://replace-this-with-your-url.com/posts/"));
assert!(file_contains!(public, "posts/page/4/index.html", "Last: https://replace-this-with-your-url.com/posts/page/4/"));
}
#[test]
@ -397,10 +416,10 @@ fn can_build_rss_feed() {
assert!(Path::new(&public).exists());
assert!(file_exists!(public, "rss.xml"));
// latest article is posts/simple.md
// latest article is posts/extra-syntax.md
assert!(file_contains!(public, "rss.xml", "Extra Syntax"));
// Next is posts/simple.md
assert!(file_contains!(public, "rss.xml", "Simple article with shortcodes"));
// Next is posts/python.md
assert!(file_contains!(public, "rss.xml", "Python in posts"));
}
@ -420,3 +439,20 @@ fn can_build_search_index() {
assert!(file_exists!(public, "elasticlunr.min.js"));
assert!(file_exists!(public, "search_index.en.js"));
}
#[test]
fn can_build_with_extra_syntaxes() {
let mut path = env::current_dir().unwrap().parent().unwrap().parent().unwrap().to_path_buf();
path.push("test_site");
let mut site = Site::new(&path, "config.toml").unwrap();
site.load().unwrap();
let tmp_dir = tempdir().expect("create temp dir");
let public = &tmp_dir.path().join("public");
site.set_output_path(&public);
site.build().unwrap();
assert!(&public.exists());
assert!(file_exists!(public, "posts/extra-syntax/index.html"));
assert!(file_contains!(public, "posts/extra-syntax/index.html",
r#"<span style="background-color:#2b303b;color:#d08770;">test</span>"#));
}

View file

@ -106,3 +106,23 @@ Here is a full list of the supported languages and the short names you can use:
```
If you want to highlight a language not on that list, please open an issue or a pull request on the [Gutenberg repo](https://github.com/Keats/gutenberg).
Alternatively, the `extra_syntaxes` config option can be used to add additional syntax files.
If your site source is laid out as follows:
```
.
├── config.toml
├── content/
│   └── ...
├── static/
│   └── ...
├── syntaxes/
│   ├── Sublime-Language1/
│   │   └── lang1.sublime-syntax
│   └── lang2.sublime-syntax
└── templates/
└── ...
```
you would set your `extra_syntaxes` to `["syntaxes", "syntaxes/Sublime-Language1"]` in order to load `lang1.sublime-syntax` and `lang2.sublime-syntax`.

View file

@ -70,6 +70,9 @@ check_external_links = false
# ignored_content = ["*.{graphml,xlsx}", "temp.*"]
ignored_content = []
# A list of directories to search for additional `.sublime-syntax` files in.
extra_syntaxes = []
# Optional translation object. The key if present should be a language code
[translations]

View file

@ -10,5 +10,7 @@ taxonomies = [
{name = "categories", rss = true},
]
extra_syntaxes = ["syntaxes"]
[extra.author]
name = "Vincent Prouillet"

View file

@ -0,0 +1,9 @@
+++
title = "Extra Syntax"
description = ""
date = 2018-08-14
+++
```test-syntax
This is a test code snippet.
```

View file

@ -0,0 +1,10 @@
%YAML 1.2
---
file_extensions:
- test-syntax
scope: source.test
contexts:
main:
- match: "test"
scope: constant.language.test