From b759cf7681537e8432ae6d30b2ed6cf6939aa049 Mon Sep 17 00:00:00 2001 From: William Yao Date: Mon, 19 Jun 2017 23:29:14 -0500 Subject: [PATCH] parallelize site rendering with rayon (Keats/gutenberg#70) --- Cargo.lock | 52 +++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/lib.rs | 1 + src/rendering/markdown.rs | 32 +++++++----------------- src/rendering/mod.rs | 1 + src/rendering/parsing.rs | 10 ++++++++ src/site.rs | 30 ++++++++++++++++------ 7 files changed, 96 insertions(+), 31 deletions(-) create mode 100644 src/rendering/parsing.rs diff --git a/Cargo.lock b/Cargo.lock index fa3e0fd9..6a856a7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,6 +12,7 @@ dependencies = [ "mount 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "notify 4.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "pulldown-cmark 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -170,6 +171,15 @@ dependencies = [ "gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "coco" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "conduit-mime-types" version = "0.7.3" @@ -192,6 +202,11 @@ name = "dtoa" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "either" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "error" version = "0.1.9" @@ -249,6 +264,11 @@ dependencies = [ "libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "futures" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "gcc" version = "0.3.51" @@ -629,6 +649,27 @@ dependencies = [ "libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rayon" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rayon-core 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rayon-core" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "redox_syscall" version = "0.1.18" @@ -675,6 +716,11 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "scopeguard" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "sequence_trie" version = "0.2.1" @@ -1073,9 +1119,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum chrono 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d9123be86fd2a8f627836c235ecdf331fdd067ecf7ac05aa1a68fbcf2429f056" "checksum clap 2.24.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6b8f69e518f967224e628896b54e41ff6acfb4dcfefc5076325c36525dac900f" "checksum cmake 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "b8ebbb35d3dc9cd09497168f33de1acb79b265d350ab0ac34133b98f8509af1f" +"checksum coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c06169f5beb7e31c7c67ebf5540b8b472d23e3eade3b2ec7d1f5b504a85f91bd" "checksum conduit-mime-types 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "95ca30253581af809925ef68c2641cc140d6183f43e12e0af4992d53768bd7b8" "checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850" "checksum dtoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80c8b71fd71146990a9742fc06dcbbde19161a267e0ad4e572c35162f4578c90" +"checksum either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18785c1ba806c258137c937e44ada9ee7e69a37e3c72077542cd2f069d78562a" "checksum error 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e606f14042bb87cc02ef6a14db6c90ab92ed6f62d87e69377bc759fd7987cc" "checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8" "checksum filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "5363ab8e4139b8568a6237db5248646e5a8a2f89bd5ccb02092182b11fd3e922" @@ -1083,6 +1131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344" "checksum fsevent 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "dfe593ebcfc76884138b25426999890b10da8e6a46d01b499d7c54c604672c38" "checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874" +"checksum futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "4b63a4792d4f8f686defe3b39b92127fea6344de5d38202b2ee5a11bbbf29d6a" "checksum gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)" = "120d07f202dcc3f72859422563522b66fe6463a4c513df062874daad05f85f0a" "checksum getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9047cfbd08a437050b363d35ef160452c5fe8ea5187ae0a624708c91581d685" "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" @@ -1128,6 +1177,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum pulldown-cmark 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9ab1e588ef8efd702c7ed9d2bd774db5e6f4d878bb5a1a9f371828fbdff6973" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d" +"checksum rayon 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "705cf28d52a26a9ab548930a9a3d9799eb77cf84d66d7cc6e52fa222ca662424" +"checksum rayon-core 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7febc28567082c345f10cddc3612c6ea020fc3297a1977d472cf9fdb73e6e493" "checksum redox_syscall 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "3041aeb6000db123d2c9c751433f526e1f404b23213bd733167ab770c3989b4d" "checksum regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1731164734096285ec2a5ec7fea5248ae2f5485b3feeb0115af4fda2183b2d1b" "checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db" @@ -1135,6 +1186,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" "checksum same-file 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d931a44fdaa43b8637009e7632a02adc4f2b2e0733c08caa4cf00e8da4a117a7" +"checksum scopeguard 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c79eb2c3ac4bc2507cda80e7f3ac5b88bd8eae4c0914d5663e6a8933994be918" "checksum sequence_trie 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c915714ca833b1d4d6b8f6a9d72a3ff632fe45b40a8d184ef79c81bec6327eed" "checksum serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c2f530d36fb84ec48fb7146936881f026cdbf4892028835fd9398475f82c1bb4" "checksum serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "10552fad5500771f3902d0c5ba187c5881942b811b7ba0d8fbbfbf84d80806d3" diff --git a/Cargo.toml b/Cargo.toml index 0292be9d..87d498fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ chrono = "0.3" toml = "0.4" term-painter = "0.2" base64 = "0.6" +rayon = "0.8" # Below is for the serve cmd staticfile = "0.4" diff --git a/src/lib.rs b/src/lib.rs index 01f809d3..75dc9673 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ extern crate syntect; extern crate slug; extern crate chrono; extern crate base64; +extern crate rayon; #[cfg(test)] extern crate tempdir; diff --git a/src/rendering/markdown.rs b/src/rendering/markdown.rs index 19826ff6..a4288055 100644 --- a/src/rendering/markdown.rs +++ b/src/rendering/markdown.rs @@ -4,9 +4,7 @@ use pulldown_cmark as cmark; use self::cmark::{Parser, Event, Tag, Options, OPTION_ENABLE_TABLES, OPTION_ENABLE_FOOTNOTES}; use regex::Regex; use slug::slugify; -use syntect::dumps::from_binary; use syntect::easy::HighlightLines; -use syntect::parsing::SyntaxSet; use syntect::html::{start_coloured_html_snippet, styles_to_coloured_html, IncludeBackground}; use tera::{Context as TeraContext}; @@ -15,26 +13,12 @@ use site::resolve_internal_link; use front_matter::InsertAnchor; use rendering::context::Context; use rendering::highlighting::THEME_SET; +use rendering::parsing::SYNTAX_SET; use rendering::short_code::{ShortCode, parse_shortcode, render_simple_shortcode}; use content::{TempHeader, Header, make_table_of_contents}; -// We need to put those in a struct to impl Send and sync -pub struct Setup { - pub syntax_set: SyntaxSet, -} - -unsafe impl Send for Setup {} -unsafe impl Sync for Setup {} - lazy_static!{ static ref SHORTCODE_RE: Regex = Regex::new(r#"\{(?:%|\{)\s+([[:alnum:]]+?)\(([[:alnum:]]+?="?.+?"?)\)\s+(?:%|\})\}"#).unwrap(); - pub static ref SETUP: Setup = Setup { - syntax_set: { - let mut ps: SyntaxSet = from_binary(include_bytes!("../../sublime_syntaxes/newlines.packdump")); - ps.link_syntaxes(); - ps - }, - }; } @@ -190,12 +174,14 @@ pub fn markdown_to_html(content: &str, context: &Context) -> Result<(String, Vec return Event::Html(Owned("
".to_owned()));
                 }
                 let theme = &THEME_SET.themes[&context.highlight_theme];
-                let syntax = info
-                    .split(' ')
-                    .next()
-                    .and_then(|lang| SETUP.syntax_set.find_syntax_by_token(lang))
-                    .unwrap_or_else(|| SETUP.syntax_set.find_syntax_plain_text());
-                highlighter = Some(HighlightLines::new(syntax, theme));
+                highlighter = SYNTAX_SET.with(|ss| {
+                    let syntax = info
+                        .split(' ')
+                        .next()
+                        .and_then(|lang| ss.find_syntax_by_token(lang))
+                        .unwrap_or_else(|| ss.find_syntax_plain_text());
+                    Some(HighlightLines::new(syntax, theme))
+                });
                 let snippet = start_coloured_html_snippet(theme);
                 Event::Html(Owned(snippet))
             },
diff --git a/src/rendering/mod.rs b/src/rendering/mod.rs
index 53b696d5..b51351bb 100644
--- a/src/rendering/mod.rs
+++ b/src/rendering/mod.rs
@@ -1,4 +1,5 @@
 pub mod highlighting;
+pub mod parsing;
 pub mod markdown;
 pub mod short_code;
 pub mod context;
diff --git a/src/rendering/parsing.rs b/src/rendering/parsing.rs
new file mode 100644
index 00000000..9ff5e5fb
--- /dev/null
+++ b/src/rendering/parsing.rs
@@ -0,0 +1,10 @@
+use syntect::parsing::SyntaxSet;
+use syntect::dumps::from_binary;
+
+thread_local! {
+    pub static SYNTAX_SET: SyntaxSet = {
+        let mut ss: SyntaxSet = from_binary(include_bytes!("../../sublime_syntaxes/newlines.packdump"));
+        ss.link_syntaxes();
+        ss
+    };
+}
diff --git a/src/site.rs b/src/site.rs
index 78a6dcb6..decae1b6 100644
--- a/src/site.rs
+++ b/src/site.rs
@@ -13,6 +13,7 @@ use content::{Page, Section, Paginator, SortBy, Taxonomy, populate_previous_and_
 use templates::{GUTENBERG_TERA, global_fns, render_redirect_template};
 use front_matter::InsertAnchor;
 
+use rayon::prelude::*;
 
 #[derive(Debug)]
 pub struct Site {
@@ -118,14 +119,27 @@ impl Site {
             pages_insert_anchors.insert(page.file.path.clone(), self.find_parent_section_insert_anchor(&page.file.parent.clone()));
         }
 
-        // TODO: make that parallel
-        for page in self.pages.values_mut() {
-            let insert_anchor = pages_insert_anchors[&page.file.path];
-            page.render_markdown(&self.permalinks, &self.tera, &self.config, insert_anchor)?;
-        }
-        // TODO: make that parallel
-        for section in self.sections.values_mut() {
-            section.render_markdown(&self.permalinks, &self.tera, &self.config)?;
+        {
+            // Another silly thing needed to not borrow &self in parallel and
+            // make the borrow checker happy
+            let permalinks = &self.permalinks;
+            let tera = &self.tera;
+            let config = &self.config;
+
+            self.pages.par_iter_mut()
+                .map(|(_, page)| page)
+                .map(|page| {
+                    let insert_anchor = pages_insert_anchors[&page.file.path];
+                    page.render_markdown(&permalinks, &tera, &config, insert_anchor)
+                })
+                .fold(|| Ok(()), Result::and)
+                .reduce(|| Ok(()), Result::and)?;
+
+            self.sections.par_iter_mut()
+                .map(|(_, section)| section)
+                .map(|section| section.render_markdown(permalinks, tera, config))
+                .fold(|| Ok(()), Result::and)
+                .reduce(|| Ok(()), Result::and)?;
         }
 
         self.populate_sections();