parallelize site rendering with rayon (Keats/gutenberg#70)

This commit is contained in:
William Yao 2017-06-19 23:29:14 -05:00
parent 53b0ec0244
commit b759cf7681
7 changed files with 96 additions and 31 deletions

52
Cargo.lock generated
View file

@ -12,6 +12,7 @@ dependencies = [
"mount 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "mount 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"notify 4.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "notify 4.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"pulldown-cmark 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)", "pulldown-cmark 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
"rayon 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
@ -170,6 +171,15 @@ dependencies = [
"gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)", "gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "coco"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"scopeguard 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "conduit-mime-types" name = "conduit-mime-types"
version = "0.7.3" version = "0.7.3"
@ -192,6 +202,11 @@ name = "dtoa"
version = "0.4.1" version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "either"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "error" name = "error"
version = "0.1.9" version = "0.1.9"
@ -249,6 +264,11 @@ dependencies = [
"libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "futures"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "gcc" name = "gcc"
version = "0.3.51" version = "0.3.51"
@ -629,6 +649,27 @@ dependencies = [
"libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "rayon"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rayon-core 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rayon-core"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.1.18" version = "0.1.18"
@ -675,6 +716,11 @@ dependencies = [
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "scopeguard"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "sequence_trie" name = "sequence_trie"
version = "0.2.1" version = "0.2.1"
@ -1073,9 +1119,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum chrono 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d9123be86fd2a8f627836c235ecdf331fdd067ecf7ac05aa1a68fbcf2429f056" "checksum chrono 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d9123be86fd2a8f627836c235ecdf331fdd067ecf7ac05aa1a68fbcf2429f056"
"checksum clap 2.24.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6b8f69e518f967224e628896b54e41ff6acfb4dcfefc5076325c36525dac900f" "checksum clap 2.24.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6b8f69e518f967224e628896b54e41ff6acfb4dcfefc5076325c36525dac900f"
"checksum cmake 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "b8ebbb35d3dc9cd09497168f33de1acb79b265d350ab0ac34133b98f8509af1f" "checksum cmake 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "b8ebbb35d3dc9cd09497168f33de1acb79b265d350ab0ac34133b98f8509af1f"
"checksum coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c06169f5beb7e31c7c67ebf5540b8b472d23e3eade3b2ec7d1f5b504a85f91bd"
"checksum conduit-mime-types 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "95ca30253581af809925ef68c2641cc140d6183f43e12e0af4992d53768bd7b8" "checksum conduit-mime-types 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "95ca30253581af809925ef68c2641cc140d6183f43e12e0af4992d53768bd7b8"
"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850" "checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
"checksum dtoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80c8b71fd71146990a9742fc06dcbbde19161a267e0ad4e572c35162f4578c90" "checksum dtoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80c8b71fd71146990a9742fc06dcbbde19161a267e0ad4e572c35162f4578c90"
"checksum either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18785c1ba806c258137c937e44ada9ee7e69a37e3c72077542cd2f069d78562a"
"checksum error 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e606f14042bb87cc02ef6a14db6c90ab92ed6f62d87e69377bc759fd7987cc" "checksum error 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e606f14042bb87cc02ef6a14db6c90ab92ed6f62d87e69377bc759fd7987cc"
"checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8" "checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8"
"checksum filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "5363ab8e4139b8568a6237db5248646e5a8a2f89bd5ccb02092182b11fd3e922" "checksum filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "5363ab8e4139b8568a6237db5248646e5a8a2f89bd5ccb02092182b11fd3e922"
@ -1083,6 +1131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344" "checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344"
"checksum fsevent 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "dfe593ebcfc76884138b25426999890b10da8e6a46d01b499d7c54c604672c38" "checksum fsevent 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "dfe593ebcfc76884138b25426999890b10da8e6a46d01b499d7c54c604672c38"
"checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874" "checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874"
"checksum futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "4b63a4792d4f8f686defe3b39b92127fea6344de5d38202b2ee5a11bbbf29d6a"
"checksum gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)" = "120d07f202dcc3f72859422563522b66fe6463a4c513df062874daad05f85f0a" "checksum gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)" = "120d07f202dcc3f72859422563522b66fe6463a4c513df062874daad05f85f0a"
"checksum getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9047cfbd08a437050b363d35ef160452c5fe8ea5187ae0a624708c91581d685" "checksum getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9047cfbd08a437050b363d35ef160452c5fe8ea5187ae0a624708c91581d685"
"checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
@ -1128,6 +1177,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum pulldown-cmark 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9ab1e588ef8efd702c7ed9d2bd774db5e6f4d878bb5a1a9f371828fbdff6973" "checksum pulldown-cmark 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9ab1e588ef8efd702c7ed9d2bd774db5e6f4d878bb5a1a9f371828fbdff6973"
"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a"
"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d" "checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d"
"checksum rayon 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "705cf28d52a26a9ab548930a9a3d9799eb77cf84d66d7cc6e52fa222ca662424"
"checksum rayon-core 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7febc28567082c345f10cddc3612c6ea020fc3297a1977d472cf9fdb73e6e493"
"checksum redox_syscall 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "3041aeb6000db123d2c9c751433f526e1f404b23213bd733167ab770c3989b4d" "checksum redox_syscall 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "3041aeb6000db123d2c9c751433f526e1f404b23213bd733167ab770c3989b4d"
"checksum regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1731164734096285ec2a5ec7fea5248ae2f5485b3feeb0115af4fda2183b2d1b" "checksum regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1731164734096285ec2a5ec7fea5248ae2f5485b3feeb0115af4fda2183b2d1b"
"checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db" "checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db"
@ -1135,6 +1186,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f"
"checksum same-file 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d931a44fdaa43b8637009e7632a02adc4f2b2e0733c08caa4cf00e8da4a117a7" "checksum same-file 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d931a44fdaa43b8637009e7632a02adc4f2b2e0733c08caa4cf00e8da4a117a7"
"checksum scopeguard 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c79eb2c3ac4bc2507cda80e7f3ac5b88bd8eae4c0914d5663e6a8933994be918"
"checksum sequence_trie 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c915714ca833b1d4d6b8f6a9d72a3ff632fe45b40a8d184ef79c81bec6327eed" "checksum sequence_trie 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c915714ca833b1d4d6b8f6a9d72a3ff632fe45b40a8d184ef79c81bec6327eed"
"checksum serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c2f530d36fb84ec48fb7146936881f026cdbf4892028835fd9398475f82c1bb4" "checksum serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c2f530d36fb84ec48fb7146936881f026cdbf4892028835fd9398475f82c1bb4"
"checksum serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "10552fad5500771f3902d0c5ba187c5881942b811b7ba0d8fbbfbf84d80806d3" "checksum serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "10552fad5500771f3902d0c5ba187c5881942b811b7ba0d8fbbfbf84d80806d3"

View file

@ -31,6 +31,7 @@ chrono = "0.3"
toml = "0.4" toml = "0.4"
term-painter = "0.2" term-painter = "0.2"
base64 = "0.6" base64 = "0.6"
rayon = "0.8"
# Below is for the serve cmd # Below is for the serve cmd
staticfile = "0.4" staticfile = "0.4"

View file

@ -16,6 +16,7 @@ extern crate syntect;
extern crate slug; extern crate slug;
extern crate chrono; extern crate chrono;
extern crate base64; extern crate base64;
extern crate rayon;
#[cfg(test)] #[cfg(test)]
extern crate tempdir; extern crate tempdir;

View file

@ -4,9 +4,7 @@ use pulldown_cmark as cmark;
use self::cmark::{Parser, Event, Tag, Options, OPTION_ENABLE_TABLES, OPTION_ENABLE_FOOTNOTES}; use self::cmark::{Parser, Event, Tag, Options, OPTION_ENABLE_TABLES, OPTION_ENABLE_FOOTNOTES};
use regex::Regex; use regex::Regex;
use slug::slugify; use slug::slugify;
use syntect::dumps::from_binary;
use syntect::easy::HighlightLines; use syntect::easy::HighlightLines;
use syntect::parsing::SyntaxSet;
use syntect::html::{start_coloured_html_snippet, styles_to_coloured_html, IncludeBackground}; use syntect::html::{start_coloured_html_snippet, styles_to_coloured_html, IncludeBackground};
use tera::{Context as TeraContext}; use tera::{Context as TeraContext};
@ -15,26 +13,12 @@ use site::resolve_internal_link;
use front_matter::InsertAnchor; use front_matter::InsertAnchor;
use rendering::context::Context; use rendering::context::Context;
use rendering::highlighting::THEME_SET; use rendering::highlighting::THEME_SET;
use rendering::parsing::SYNTAX_SET;
use rendering::short_code::{ShortCode, parse_shortcode, render_simple_shortcode}; use rendering::short_code::{ShortCode, parse_shortcode, render_simple_shortcode};
use content::{TempHeader, Header, make_table_of_contents}; use content::{TempHeader, Header, make_table_of_contents};
// We need to put those in a struct to impl Send and sync
pub struct Setup {
pub syntax_set: SyntaxSet,
}
unsafe impl Send for Setup {}
unsafe impl Sync for Setup {}
lazy_static!{ lazy_static!{
static ref SHORTCODE_RE: Regex = Regex::new(r#"\{(?:%|\{)\s+([[:alnum:]]+?)\(([[:alnum:]]+?="?.+?"?)\)\s+(?:%|\})\}"#).unwrap(); static ref SHORTCODE_RE: Regex = Regex::new(r#"\{(?:%|\{)\s+([[:alnum:]]+?)\(([[:alnum:]]+?="?.+?"?)\)\s+(?:%|\})\}"#).unwrap();
pub static ref SETUP: Setup = Setup {
syntax_set: {
let mut ps: SyntaxSet = from_binary(include_bytes!("../../sublime_syntaxes/newlines.packdump"));
ps.link_syntaxes();
ps
},
};
} }
@ -190,12 +174,14 @@ pub fn markdown_to_html(content: &str, context: &Context) -> Result<(String, Vec
return Event::Html(Owned("<pre><code>".to_owned())); return Event::Html(Owned("<pre><code>".to_owned()));
} }
let theme = &THEME_SET.themes[&context.highlight_theme]; let theme = &THEME_SET.themes[&context.highlight_theme];
highlighter = SYNTAX_SET.with(|ss| {
let syntax = info let syntax = info
.split(' ') .split(' ')
.next() .next()
.and_then(|lang| SETUP.syntax_set.find_syntax_by_token(lang)) .and_then(|lang| ss.find_syntax_by_token(lang))
.unwrap_or_else(|| SETUP.syntax_set.find_syntax_plain_text()); .unwrap_or_else(|| ss.find_syntax_plain_text());
highlighter = Some(HighlightLines::new(syntax, theme)); Some(HighlightLines::new(syntax, theme))
});
let snippet = start_coloured_html_snippet(theme); let snippet = start_coloured_html_snippet(theme);
Event::Html(Owned(snippet)) Event::Html(Owned(snippet))
}, },

View file

@ -1,4 +1,5 @@
pub mod highlighting; pub mod highlighting;
pub mod parsing;
pub mod markdown; pub mod markdown;
pub mod short_code; pub mod short_code;
pub mod context; pub mod context;

10
src/rendering/parsing.rs Normal file
View file

@ -0,0 +1,10 @@
use syntect::parsing::SyntaxSet;
use syntect::dumps::from_binary;
thread_local! {
pub static SYNTAX_SET: SyntaxSet = {
let mut ss: SyntaxSet = from_binary(include_bytes!("../../sublime_syntaxes/newlines.packdump"));
ss.link_syntaxes();
ss
};
}

View file

@ -13,6 +13,7 @@ use content::{Page, Section, Paginator, SortBy, Taxonomy, populate_previous_and_
use templates::{GUTENBERG_TERA, global_fns, render_redirect_template}; use templates::{GUTENBERG_TERA, global_fns, render_redirect_template};
use front_matter::InsertAnchor; use front_matter::InsertAnchor;
use rayon::prelude::*;
#[derive(Debug)] #[derive(Debug)]
pub struct Site { pub struct Site {
@ -118,14 +119,27 @@ impl Site {
pages_insert_anchors.insert(page.file.path.clone(), self.find_parent_section_insert_anchor(&page.file.parent.clone())); pages_insert_anchors.insert(page.file.path.clone(), self.find_parent_section_insert_anchor(&page.file.parent.clone()));
} }
// TODO: make that parallel {
for page in self.pages.values_mut() { // Another silly thing needed to not borrow &self in parallel and
// make the borrow checker happy
let permalinks = &self.permalinks;
let tera = &self.tera;
let config = &self.config;
self.pages.par_iter_mut()
.map(|(_, page)| page)
.map(|page| {
let insert_anchor = pages_insert_anchors[&page.file.path]; let insert_anchor = pages_insert_anchors[&page.file.path];
page.render_markdown(&self.permalinks, &self.tera, &self.config, insert_anchor)?; page.render_markdown(&permalinks, &tera, &config, insert_anchor)
} })
// TODO: make that parallel .fold(|| Ok(()), Result::and)
for section in self.sections.values_mut() { .reduce(|| Ok(()), Result::and)?;
section.render_markdown(&self.permalinks, &self.tera, &self.config)?;
self.sections.par_iter_mut()
.map(|(_, section)| section)
.map(|section| section.render_markdown(permalinks, tera, config))
.fold(|| Ok(()), Result::and)
.reduce(|| Ok(()), Result::and)?;
} }
self.populate_sections(); self.populate_sections();