Read off disk in parallel

CHANGELOG.md

@@ -2,7 +2,7 @@
 
 ## 0.0.8 (unreleased)
 
-- Parallelize page rendering using rayon
+- Parallelize all the things
 
 ## 0.0.7 (2017-06-19)
 

benches/site_load.rs

@@ -1,13 +1,11 @@
-//! Benchmarking generated sites of various sizes
+//! Benchmarking loading/markdown rendering of generated sites of various sizes
 
 #![feature(test)]
 extern crate test;
 extern crate gutenberg;
-extern crate tempdir;
 
 use std::env;
 
-use tempdir::TempDir;
 use gutenberg::Site;
 
 
@@ -42,16 +40,16 @@ fn bench_loading_medium_blog(b: &mut test::Bencher) {
     b.iter(|| site.load().unwrap());
 }
 
-//#[bench]
+#[bench]
-//fn bench_loading_medium_blog_with_syntax_highlighting(b: &mut test::Bencher) {
+fn bench_loading_medium_blog_with_syntax_highlighting(b: &mut test::Bencher) {
-//    let mut path = env::current_dir().unwrap().to_path_buf();
+    let mut path = env::current_dir().unwrap().to_path_buf();
-//    path.push("benches");
+    path.push("benches");
-//    path.push("medium-blog");
+    path.push("medium-blog");
-//    let mut site = Site::new(&path, "config.toml").unwrap();
+    let mut site = Site::new(&path, "config.toml").unwrap();
-//    site.config.highlight_code = Some(true);
+    site.config.highlight_code = Some(true);
-//
+
-//    b.iter(|| site.load().unwrap());
+    b.iter(|| site.load().unwrap());
-//}
+}
 
 #[bench]
 fn bench_loading_big_blog(b: &mut test::Bencher) {
@@ -74,26 +72,26 @@ fn bench_loading_big_blog_with_syntax_highlighting(b: &mut test::Bencher) {
     b.iter(|| site.load().unwrap());
 }
 
-#[bench]
+//#[bench]
-fn bench_loading_huge_blog(b: &mut test::Bencher) {
+//fn bench_loading_huge_blog(b: &mut test::Bencher) {
-    let mut path = env::current_dir().unwrap().to_path_buf();
+//    let mut path = env::current_dir().unwrap().to_path_buf();
-    path.push("benches");
+//    path.push("benches");
-    path.push("huge-blog");
+//    path.push("huge-blog");
-    let mut site = Site::new(&path, "config.toml").unwrap();
+//    let mut site = Site::new(&path, "config.toml").unwrap();
-
+//
-    b.iter(|| site.load().unwrap());
+//    b.iter(|| site.load().unwrap());
-}
+//}
-
+//
-#[bench]
+//#[bench]
-fn bench_loading_huge_blog_with_syntax_highlighting(b: &mut test::Bencher) {
+//fn bench_loading_huge_blog_with_syntax_highlighting(b: &mut test::Bencher) {
-    let mut path = env::current_dir().unwrap().to_path_buf();
+//    let mut path = env::current_dir().unwrap().to_path_buf();
-    path.push("benches");
+//    path.push("benches");
-    path.push("huge-blog");
+//    path.push("huge-blog");
-    let mut site = Site::new(&path, "config.toml").unwrap();
+//    let mut site = Site::new(&path, "config.toml").unwrap();
-    site.config.highlight_code = Some(true);
+//    site.config.highlight_code = Some(true);
-
+//
-    b.iter(|| site.load().unwrap());
+//    b.iter(|| site.load().unwrap());
-}
+//}
 
 #[bench]
 fn bench_loading_small_kb(b: &mut test::Bencher) {
@@ -137,23 +135,23 @@ fn bench_loading_medium_kb_with_syntax_highlighting(b: &mut test::Bencher) {
     b.iter(|| site.load().unwrap());
 }
 
-#[bench]
+//#[bench]
-fn bench_loading_huge_kb(b: &mut test::Bencher) {
+//fn bench_loading_huge_kb(b: &mut test::Bencher) {
-    let mut path = env::current_dir().unwrap().to_path_buf();
+//    let mut path = env::current_dir().unwrap().to_path_buf();
-    path.push("benches");
+//    path.push("benches");
-    path.push("huge-kb");
+//    path.push("huge-kb");
-    let mut site = Site::new(&path, "config.toml").unwrap();
+//    let mut site = Site::new(&path, "config.toml").unwrap();
-
+//
-    b.iter(|| site.load().unwrap());
+//    b.iter(|| site.load().unwrap());
-}
+//}
-
+//
-#[bench]
+//#[bench]
-fn bench_loading_huge_kb_with_syntax_highlighting(b: &mut test::Bencher) {
+//fn bench_loading_huge_kb_with_syntax_highlighting(b: &mut test::Bencher) {
-    let mut path = env::current_dir().unwrap().to_path_buf();
+//    let mut path = env::current_dir().unwrap().to_path_buf();
-    path.push("benches");
+//    path.push("benches");
-    path.push("huge-kb");
+//    path.push("huge-kb");
-    let mut site = Site::new(&path, "config.toml").unwrap();
+//    let mut site = Site::new(&path, "config.toml").unwrap();
-    site.config.highlight_code = Some(true);
+//    site.config.highlight_code = Some(true);
-
+//
-    b.iter(|| site.load().unwrap());
+//    b.iter(|| site.load().unwrap());
-}
+//}

benches/unit.rs

@@ -1,3 +0,0 @@
-//! Benchmarking individual functions of Gutenberg
-
-

src/bin/rebuild.rs

@@ -130,7 +130,8 @@ pub fn after_content_change(site: &mut Site, path: &Path) -> Result<()> {
 
     // A section was edited
     if is_section {
-        match site.add_section(path, true)? {
+        let section = Section::from_file(path, &site.config)?;
+        match site.add_section(section, true)? {
             Some(prev) => {
                 // Updating a section
                 let current_meta = site.sections[path].meta.clone();
@@ -166,7 +167,8 @@ pub fn after_content_change(site: &mut Site, path: &Path) -> Result<()> {
     }
 
     // A page was edited
-    match site.add_page(path, true)? {
+    let page = Page::from_file(path, &site.config)?;
+    match site.add_page(page, true)? {
         Some(prev) => {
             site.register_get_url_fn();
             // Updating a page

src/site.rs

@@ -96,14 +96,46 @@ impl Site {
         let base_path = self.base_path.to_string_lossy().replace("\\", "/");
         let content_glob = format!("{}/{}", base_path, "content/**/*.md");
 
-        for entry in glob(&content_glob).unwrap().filter_map(|e| e.ok()) {
+        let (section_entries, page_entries): (Vec<_>, Vec<_>) = glob(&content_glob)
-            let path = entry.as_path();
+            .unwrap()
-            if path.file_name().unwrap() == "_index.md" {
+            .filter_map(|e| e.ok())
-                self.add_section(path, false)?;
+            .partition(|ref entry| entry.as_path().file_name().unwrap() == "_index.md");
-            } else {
+
-                self.add_page(path, false)?;
+        let sections = {
-            }
+            let config = &self.config;
+
+            section_entries
+                .into_par_iter()
+                .filter(|entry| entry.as_path().file_name().unwrap() == "_index.md")
+                .map(|entry| {
+                    let path = entry.as_path();
+                    Section::from_file(path, &config)
+                }).collect::<Vec<_>>()
+        };
+
+        let pages = {
+            let config = &self.config;
+
+            page_entries
+                .into_par_iter()
+                .filter(|entry| entry.as_path().file_name().unwrap() != "_index.md")
+                .map(|entry| {
+                    let path = entry.as_path();
+                    Page::from_file(path, &config)
+                }).collect::<Vec<_>>()
+        };
+
+        // Kinda duplicated code for add_section/add_page but necessary to do it that
+        // way because of the borrow checker
+        for section in sections {
+            let s = section?;
+            self.add_section(s, false)?;
         }
+        for page in pages {
+            let p = page?;
+            self.add_page(p, false)?;
+        }
+
         // Insert a default index section if necessary so we don't need to create
         // a _index.md to render the index page
         let index_path = self.base_path.join("content").join("_index.md");
@@ -161,14 +193,14 @@ impl Site {
     /// The `render` parameter is used in the serve command, when rebuilding a page.
     /// If `true`, it will also render the markdown for that page
     /// Returns the previous page struct if there was one
-    pub fn add_page(&mut self, path: &Path, render: bool) -> Result<Option<Page>> {
+    pub fn add_page(&mut self, page: Page, render: bool) -> Result<Option<Page>> {
-        let page = Page::from_file(&path, &self.config)?;
+        let path = page.file.path.clone();
         self.permalinks.insert(page.file.relative.clone(), page.permalink.clone());
         let prev = self.pages.insert(page.file.path.clone(), page);
 
         if render {
-            let insert_anchor = self.find_parent_section_insert_anchor(&self.pages[path].file.parent);
+            let insert_anchor = self.find_parent_section_insert_anchor(&self.pages[&path].file.parent);
-            let mut page = self.pages.get_mut(path).unwrap();
+            let mut page = self.pages.get_mut(&path).unwrap();
             page.render_markdown(&self.permalinks, &self.tera, &self.config, insert_anchor)?;
         }
 
@@ -179,13 +211,13 @@ impl Site {
     /// The `render` parameter is used in the serve command, when rebuilding a page.
     /// If `true`, it will also render the markdown for that page
     /// Returns the previous section struct if there was one
-    pub fn add_section(&mut self, path: &Path, render: bool) -> Result<Option<Section>> {
+    pub fn add_section(&mut self, section: Section, render: bool) -> Result<Option<Section>> {
-        let section = Section::from_file(path, &self.config)?;
+        let path = section.file.path.clone();
         self.permalinks.insert(section.file.relative.clone(), section.permalink.clone());
         let prev = self.sections.insert(section.file.path.clone(), section);
 
         if render {
-            let mut section = self.sections.get_mut(path).unwrap();
+            let mut section = self.sections.get_mut(&path).unwrap();
             section.render_markdown(&self.permalinks, &self.tera, &self.config)?;
         }