Merge pull request #89 from williamyaoh/add-parallel
Parallelize site rendering using rayon
This commit is contained in:
Vincent Prouillet
GitHub
commit
b45f8c3116
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
## 0.0.8 (unreleased)
|
||||
|
||||
- Parallelize page rendering using rayon
|
||||
|
||||
## 0.0.7 (2017-06-19)
|
||||
|
||||
|
|
|
|||
52
Cargo.lock
generated
52
Cargo.lock
generated
|
|
@ -12,6 +12,7 @@ dependencies = [
|
|||
"mount 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"notify 4.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"pulldown-cmark 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rayon 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -170,6 +171,15 @@ dependencies = [
|
|||
"gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "coco"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"scopeguard 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "conduit-mime-types"
|
||||
version = "0.7.3"
|
||||
|
|
@ -192,6 +202,11 @@ name = "dtoa"
|
|||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "error"
|
||||
version = "0.1.9"
|
||||
|
|
@ -249,6 +264,11 @@ dependencies = [
|
|||
"libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "gcc"
|
||||
version = "0.3.51"
|
||||
|
|
@ -629,6 +649,27 @@ dependencies = [
|
|||
"libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"rayon-core 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num_cpus 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.1.18"
|
||||
|
|
@ -675,6 +716,11 @@ dependencies = [
|
|||
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "sequence_trie"
|
||||
version = "0.2.1"
|
||||
|
|
@ -1073,9 +1119,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum chrono 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d9123be86fd2a8f627836c235ecdf331fdd067ecf7ac05aa1a68fbcf2429f056"
|
||||
"checksum clap 2.24.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6b8f69e518f967224e628896b54e41ff6acfb4dcfefc5076325c36525dac900f"
|
||||
"checksum cmake 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "b8ebbb35d3dc9cd09497168f33de1acb79b265d350ab0ac34133b98f8509af1f"
|
||||
"checksum coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c06169f5beb7e31c7c67ebf5540b8b472d23e3eade3b2ec7d1f5b504a85f91bd"
|
||||
"checksum conduit-mime-types 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "95ca30253581af809925ef68c2641cc140d6183f43e12e0af4992d53768bd7b8"
|
||||
"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
|
||||
"checksum dtoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80c8b71fd71146990a9742fc06dcbbde19161a267e0ad4e572c35162f4578c90"
|
||||
"checksum either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18785c1ba806c258137c937e44ada9ee7e69a37e3c72077542cd2f069d78562a"
|
||||
"checksum error 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e606f14042bb87cc02ef6a14db6c90ab92ed6f62d87e69377bc759fd7987cc"
|
||||
"checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8"
|
||||
"checksum filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "5363ab8e4139b8568a6237db5248646e5a8a2f89bd5ccb02092182b11fd3e922"
|
||||
|
|
@ -1083,6 +1131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344"
|
||||
"checksum fsevent 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "dfe593ebcfc76884138b25426999890b10da8e6a46d01b499d7c54c604672c38"
|
||||
"checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874"
|
||||
"checksum futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "4b63a4792d4f8f686defe3b39b92127fea6344de5d38202b2ee5a11bbbf29d6a"
|
||||
"checksum gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)" = "120d07f202dcc3f72859422563522b66fe6463a4c513df062874daad05f85f0a"
|
||||
"checksum getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9047cfbd08a437050b363d35ef160452c5fe8ea5187ae0a624708c91581d685"
|
||||
"checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
|
||||
|
|
@ -1128,6 +1177,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum pulldown-cmark 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9ab1e588ef8efd702c7ed9d2bd774db5e6f4d878bb5a1a9f371828fbdff6973"
|
||||
"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a"
|
||||
"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d"
|
||||
"checksum rayon 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "705cf28d52a26a9ab548930a9a3d9799eb77cf84d66d7cc6e52fa222ca662424"
|
||||
"checksum rayon-core 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7febc28567082c345f10cddc3612c6ea020fc3297a1977d472cf9fdb73e6e493"
|
||||
"checksum redox_syscall 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "3041aeb6000db123d2c9c751433f526e1f404b23213bd733167ab770c3989b4d"
|
||||
"checksum regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1731164734096285ec2a5ec7fea5248ae2f5485b3feeb0115af4fda2183b2d1b"
|
||||
"checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db"
|
||||
|
|
@ -1135,6 +1186,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
|
||||
"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f"
|
||||
"checksum same-file 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d931a44fdaa43b8637009e7632a02adc4f2b2e0733c08caa4cf00e8da4a117a7"
|
||||
"checksum scopeguard 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c79eb2c3ac4bc2507cda80e7f3ac5b88bd8eae4c0914d5663e6a8933994be918"
|
||||
"checksum sequence_trie 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c915714ca833b1d4d6b8f6a9d72a3ff632fe45b40a8d184ef79c81bec6327eed"
|
||||
"checksum serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c2f530d36fb84ec48fb7146936881f026cdbf4892028835fd9398475f82c1bb4"
|
||||
"checksum serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "10552fad5500771f3902d0c5ba187c5881942b811b7ba0d8fbbfbf84d80806d3"
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ chrono = "0.3"
|
|||
toml = "0.4"
|
||||
term-painter = "0.2"
|
||||
base64 = "0.6"
|
||||
rayon = "0.8"
|
||||
|
||||
# Below is for the serve cmd
|
||||
staticfile = "0.4"
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ extern crate syntect;
|
|||
extern crate slug;
|
||||
extern crate chrono;
|
||||
extern crate base64;
|
||||
extern crate rayon;
|
||||
#[cfg(test)]
|
||||
extern crate tempdir;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,16 @@
|
|||
use syntect::dumps::from_binary;
|
||||
use syntect::parsing::SyntaxSet;
|
||||
use syntect::highlighting::ThemeSet;
|
||||
|
||||
thread_local!{
|
||||
pub static SYNTAX_SET: SyntaxSet = {
|
||||
let mut ss: SyntaxSet = from_binary(include_bytes!("../../sublime_syntaxes/newlines.packdump"));
|
||||
ss.link_syntaxes();
|
||||
ss
|
||||
};
|
||||
}
|
||||
|
||||
lazy_static!{
|
||||
pub static ref THEME_SET: ThemeSet = from_binary(include_bytes!("../../sublime_themes/all.themedump"));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,9 +4,7 @@ use pulldown_cmark as cmark;
|
|||
use self::cmark::{Parser, Event, Tag, Options, OPTION_ENABLE_TABLES, OPTION_ENABLE_FOOTNOTES};
|
||||
use regex::Regex;
|
||||
use slug::slugify;
|
||||
use syntect::dumps::from_binary;
|
||||
use syntect::easy::HighlightLines;
|
||||
use syntect::parsing::SyntaxSet;
|
||||
use syntect::html::{start_coloured_html_snippet, styles_to_coloured_html, IncludeBackground};
|
||||
use tera::{Context as TeraContext};
|
||||
|
||||
|
|
@ -14,27 +12,12 @@ use errors::{Result};
|
|||
use site::resolve_internal_link;
|
||||
use front_matter::InsertAnchor;
|
||||
use rendering::context::Context;
|
||||
use rendering::highlighting::THEME_SET;
|
||||
use rendering::highlighting::{SYNTAX_SET, THEME_SET};
|
||||
use rendering::short_code::{ShortCode, parse_shortcode, render_simple_shortcode};
|
||||
use content::{TempHeader, Header, make_table_of_contents};
|
||||
|
||||
// We need to put those in a struct to impl Send and sync
|
||||
pub struct Setup {
|
||||
pub syntax_set: SyntaxSet,
|
||||
}
|
||||
|
||||
unsafe impl Send for Setup {}
|
||||
unsafe impl Sync for Setup {}
|
||||
|
||||
lazy_static!{
|
||||
static ref SHORTCODE_RE: Regex = Regex::new(r#"\{(?:%|\{)\s+([[:alnum:]]+?)\(([[:alnum:]]+?="?.+?"?)\)\s+(?:%|\})\}"#).unwrap();
|
||||
pub static ref SETUP: Setup = Setup {
|
||||
syntax_set: {
|
||||
let mut ps: SyntaxSet = from_binary(include_bytes!("../../sublime_syntaxes/newlines.packdump"));
|
||||
ps.link_syntaxes();
|
||||
ps
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -190,12 +173,14 @@ pub fn markdown_to_html(content: &str, context: &Context) -> Result<(String, Vec
|
|||
return Event::Html(Owned("<pre><code>".to_owned()));
|
||||
}
|
||||
let theme = &THEME_SET.themes[&context.highlight_theme];
|
||||
let syntax = info
|
||||
.split(' ')
|
||||
.next()
|
||||
.and_then(|lang| SETUP.syntax_set.find_syntax_by_token(lang))
|
||||
.unwrap_or_else(|| SETUP.syntax_set.find_syntax_plain_text());
|
||||
highlighter = Some(HighlightLines::new(syntax, theme));
|
||||
highlighter = SYNTAX_SET.with(|ss| {
|
||||
let syntax = info
|
||||
.split(' ')
|
||||
.next()
|
||||
.and_then(|lang| ss.find_syntax_by_token(lang))
|
||||
.unwrap_or_else(|| ss.find_syntax_plain_text());
|
||||
Some(HighlightLines::new(syntax, theme))
|
||||
});
|
||||
let snippet = start_coloured_html_snippet(theme);
|
||||
Event::Html(Owned(snippet))
|
||||
},
|
||||
|
|
|
|||
30
src/site.rs
30
src/site.rs
|
|
@ -13,6 +13,7 @@ use content::{Page, Section, Paginator, SortBy, Taxonomy, populate_previous_and_
|
|||
use templates::{GUTENBERG_TERA, global_fns, render_redirect_template};
|
||||
use front_matter::InsertAnchor;
|
||||
|
||||
use rayon::prelude::*;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Site {
|
||||
|
|
@ -118,14 +119,27 @@ impl Site {
|
|||
pages_insert_anchors.insert(page.file.path.clone(), self.find_parent_section_insert_anchor(&page.file.parent.clone()));
|
||||
}
|
||||
|
||||
// TODO: make that parallel
|
||||
for page in self.pages.values_mut() {
|
||||
let insert_anchor = pages_insert_anchors[&page.file.path];
|
||||
page.render_markdown(&self.permalinks, &self.tera, &self.config, insert_anchor)?;
|
||||
}
|
||||
// TODO: make that parallel
|
||||
for section in self.sections.values_mut() {
|
||||
section.render_markdown(&self.permalinks, &self.tera, &self.config)?;
|
||||
{
|
||||
// Another silly thing needed to not borrow &self in parallel and
|
||||
// make the borrow checker happy
|
||||
let permalinks = &self.permalinks;
|
||||
let tera = &self.tera;
|
||||
let config = &self.config;
|
||||
|
||||
self.pages.par_iter_mut()
|
||||
.map(|(_, page)| page)
|
||||
.map(|page| {
|
||||
let insert_anchor = pages_insert_anchors[&page.file.path];
|
||||
page.render_markdown(&permalinks, &tera, &config, insert_anchor)
|
||||
})
|
||||
.fold(|| Ok(()), Result::and)
|
||||
.reduce(|| Ok(()), Result::and)?;
|
||||
|
||||
self.sections.par_iter_mut()
|
||||
.map(|(_, section)| section)
|
||||
.map(|section| section.render_markdown(permalinks, tera, config))
|
||||
.fold(|| Ok(()), Result::and)
|
||||
.reduce(|| Ok(()), Result::and)?;
|
||||
}
|
||||
|
||||
self.populate_sections();
|
||||
|
|
|
|||
Loading…
Reference in a new issue