parallelize site rendering with rayon (Keats/gutenberg#70)

Cargo.lock

@@ -12,6 +12,7 @@ dependencies = [
  "mount 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "notify 4.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "pulldown-cmark 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rayon 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -170,6 +171,15 @@ dependencies = [
  "gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "coco"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "scopeguard 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "conduit-mime-types"
 version = "0.7.3"
@@ -192,6 +202,11 @@ name = "dtoa"
 version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "either"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "error"
 version = "0.1.9"
@@ -249,6 +264,11 @@ dependencies = [
  "libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "futures"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "gcc"
 version = "0.3.51"
@@ -629,6 +649,27 @@ dependencies = [
  "libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rayon"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rayon-core 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num_cpus 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.1.18"
@@ -675,6 +716,11 @@ dependencies = [
  "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "scopeguard"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "sequence_trie"
 version = "0.2.1"
@@ -1073,9 +1119,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum chrono 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d9123be86fd2a8f627836c235ecdf331fdd067ecf7ac05aa1a68fbcf2429f056"
 "checksum clap 2.24.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6b8f69e518f967224e628896b54e41ff6acfb4dcfefc5076325c36525dac900f"
 "checksum cmake 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "b8ebbb35d3dc9cd09497168f33de1acb79b265d350ab0ac34133b98f8509af1f"
+"checksum coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c06169f5beb7e31c7c67ebf5540b8b472d23e3eade3b2ec7d1f5b504a85f91bd"
 "checksum conduit-mime-types 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "95ca30253581af809925ef68c2641cc140d6183f43e12e0af4992d53768bd7b8"
 "checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
 "checksum dtoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80c8b71fd71146990a9742fc06dcbbde19161a267e0ad4e572c35162f4578c90"
+"checksum either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18785c1ba806c258137c937e44ada9ee7e69a37e3c72077542cd2f069d78562a"
 "checksum error 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e606f14042bb87cc02ef6a14db6c90ab92ed6f62d87e69377bc759fd7987cc"
 "checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8"
 "checksum filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "5363ab8e4139b8568a6237db5248646e5a8a2f89bd5ccb02092182b11fd3e922"
@@ -1083,6 +1131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344"
 "checksum fsevent 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "dfe593ebcfc76884138b25426999890b10da8e6a46d01b499d7c54c604672c38"
 "checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874"
+"checksum futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "4b63a4792d4f8f686defe3b39b92127fea6344de5d38202b2ee5a11bbbf29d6a"
 "checksum gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)" = "120d07f202dcc3f72859422563522b66fe6463a4c513df062874daad05f85f0a"
 "checksum getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9047cfbd08a437050b363d35ef160452c5fe8ea5187ae0a624708c91581d685"
 "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
@@ -1128,6 +1177,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum pulldown-cmark 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9ab1e588ef8efd702c7ed9d2bd774db5e6f4d878bb5a1a9f371828fbdff6973"
 "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a"
 "checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d"
+"checksum rayon 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "705cf28d52a26a9ab548930a9a3d9799eb77cf84d66d7cc6e52fa222ca662424"
+"checksum rayon-core 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7febc28567082c345f10cddc3612c6ea020fc3297a1977d472cf9fdb73e6e493"
 "checksum redox_syscall 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "3041aeb6000db123d2c9c751433f526e1f404b23213bd733167ab770c3989b4d"
 "checksum regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1731164734096285ec2a5ec7fea5248ae2f5485b3feeb0115af4fda2183b2d1b"
 "checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db"
@@ -1135,6 +1186,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
 "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f"
 "checksum same-file 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d931a44fdaa43b8637009e7632a02adc4f2b2e0733c08caa4cf00e8da4a117a7"
+"checksum scopeguard 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c79eb2c3ac4bc2507cda80e7f3ac5b88bd8eae4c0914d5663e6a8933994be918"
 "checksum sequence_trie 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c915714ca833b1d4d6b8f6a9d72a3ff632fe45b40a8d184ef79c81bec6327eed"
 "checksum serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c2f530d36fb84ec48fb7146936881f026cdbf4892028835fd9398475f82c1bb4"
 "checksum serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "10552fad5500771f3902d0c5ba187c5881942b811b7ba0d8fbbfbf84d80806d3"

Cargo.toml

@@ -31,6 +31,7 @@ chrono = "0.3"
 toml = "0.4"
 term-painter = "0.2"
 base64 = "0.6"
+rayon = "0.8"
 
 # Below is for the serve cmd
 staticfile = "0.4"

src/lib.rs

@@ -16,6 +16,7 @@ extern crate syntect;
 extern crate slug;
 extern crate chrono;
 extern crate base64;
+extern crate rayon;
 #[cfg(test)]
 extern crate tempdir;
 

src/rendering/markdown.rs

@@ -4,9 +4,7 @@ use pulldown_cmark as cmark;
 use self::cmark::{Parser, Event, Tag, Options, OPTION_ENABLE_TABLES, OPTION_ENABLE_FOOTNOTES};
 use regex::Regex;
 use slug::slugify;
-use syntect::dumps::from_binary;
 use syntect::easy::HighlightLines;
-use syntect::parsing::SyntaxSet;
 use syntect::html::{start_coloured_html_snippet, styles_to_coloured_html, IncludeBackground};
 use tera::{Context as TeraContext};
 
@@ -15,26 +13,12 @@ use site::resolve_internal_link;
 use front_matter::InsertAnchor;
 use rendering::context::Context;
 use rendering::highlighting::THEME_SET;
+use rendering::parsing::SYNTAX_SET;
 use rendering::short_code::{ShortCode, parse_shortcode, render_simple_shortcode};
 use content::{TempHeader, Header, make_table_of_contents};
 
-// We need to put those in a struct to impl Send and sync
-pub struct Setup {
-    pub syntax_set: SyntaxSet,
-}
-
-unsafe impl Send for Setup {}
-unsafe impl Sync for Setup {}
-
 lazy_static!{
     static ref SHORTCODE_RE: Regex = Regex::new(r#"\{(?:%|\{)\s+([[:alnum:]]+?)\(([[:alnum:]]+?="?.+?"?)\)\s+(?:%|\})\}"#).unwrap();
-    pub static ref SETUP: Setup = Setup {
-        syntax_set: {
-            let mut ps: SyntaxSet = from_binary(include_bytes!("../../sublime_syntaxes/newlines.packdump"));
-            ps.link_syntaxes();
-            ps
-        },
-    };
 }
 
 
@@ -190,12 +174,14 @@ pub fn markdown_to_html(content: &str, context: &Context) -> Result<(String, Vec
                     return Event::Html(Owned("<pre><code>".to_owned()));
                 }
                 let theme = &THEME_SET.themes[&context.highlight_theme];
+                highlighter = SYNTAX_SET.with(|ss| {
                     let syntax = info
                         .split(' ')
                         .next()
-                    .and_then(|lang| SETUP.syntax_set.find_syntax_by_token(lang))
+                        .and_then(|lang| ss.find_syntax_by_token(lang))
-                    .unwrap_or_else(|| SETUP.syntax_set.find_syntax_plain_text());
+                        .unwrap_or_else(|| ss.find_syntax_plain_text());
-                highlighter = Some(HighlightLines::new(syntax, theme));
+                    Some(HighlightLines::new(syntax, theme))
+                });
                 let snippet = start_coloured_html_snippet(theme);
                 Event::Html(Owned(snippet))
             },

src/rendering/mod.rs

@@ -1,4 +1,5 @@
 pub mod highlighting;
+pub mod parsing;
 pub mod markdown;
 pub mod short_code;
 pub mod context;

src/rendering/parsing.rs

@@ -0,0 +1,10 @@
+use syntect::parsing::SyntaxSet;
+use syntect::dumps::from_binary;
+
+thread_local! {
+    pub static SYNTAX_SET: SyntaxSet = {
+        let mut ss: SyntaxSet = from_binary(include_bytes!("../../sublime_syntaxes/newlines.packdump"));
+        ss.link_syntaxes();
+        ss
+    };
+}

src/site.rs

@@ -13,6 +13,7 @@ use content::{Page, Section, Paginator, SortBy, Taxonomy, populate_previous_and_
 use templates::{GUTENBERG_TERA, global_fns, render_redirect_template};
 use front_matter::InsertAnchor;
 
+use rayon::prelude::*;
 
 #[derive(Debug)]
 pub struct Site {
@@ -118,14 +119,27 @@ impl Site {
             pages_insert_anchors.insert(page.file.path.clone(), self.find_parent_section_insert_anchor(&page.file.parent.clone()));
         }
 
-        // TODO: make that parallel
+        {
-        for page in self.pages.values_mut() {
+            // Another silly thing needed to not borrow &self in parallel and
+            // make the borrow checker happy
+            let permalinks = &self.permalinks;
+            let tera = &self.tera;
+            let config = &self.config;
+
+            self.pages.par_iter_mut()
+                .map(|(_, page)| page)
+                .map(|page| {
                     let insert_anchor = pages_insert_anchors[&page.file.path];
-            page.render_markdown(&self.permalinks, &self.tera, &self.config, insert_anchor)?;
+                    page.render_markdown(&permalinks, &tera, &config, insert_anchor)
-        }
+                })
-        // TODO: make that parallel
+                .fold(|| Ok(()), Result::and)
-        for section in self.sections.values_mut() {
+                .reduce(|| Ok(()), Result::and)?;
-            section.render_markdown(&self.permalinks, &self.tera, &self.config)?;
+
+            self.sections.par_iter_mut()
+                .map(|(_, section)| section)
+                .map(|section| section.render_markdown(permalinks, tera, config))
+                .fold(|| Ok(()), Result::and)
+                .reduce(|| Ok(()), Result::and)?;
         }
 
         self.populate_sections();