emelie/zola
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Read off disk in parallel

Browse source
This commit is contained in:
Vincent Prouillet 2017-06-22 12:01:45 +09:00
parent b45f8c3116
commit b158ca7952
5 changed files with 102 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CHANGELOG.md
View file

@ -2,7 +2,7 @@
## 0.0.8 (unreleased)
- Parallelize page rendering using rayon
- Parallelize all the things
## 0.0.7 (2017-06-19)

104
benches/integration.rs → benches/site_load.rs
View file

@ -1,13 +1,11 @@
//! Benchmarking generated sites of various sizes
//! Benchmarking loading/markdown rendering of generated sites of various sizes
#![feature(test)]
extern crate test;
extern crate gutenberg;
extern crate tempdir;
use std::env;
use tempdir::TempDir;
use gutenberg::Site;
@ -42,16 +40,16 @@ fn bench_loading_medium_blog(b: &mut test::Bencher) {
b.iter(|| site.load().unwrap());
}
//#[bench]
//fn bench_loading_medium_blog_with_syntax_highlighting(b: &mut test::Bencher) {
// let mut path = env::current_dir().unwrap().to_path_buf();
// path.push("benches");
// path.push("medium-blog");
// let mut site = Site::new(&path, "config.toml").unwrap();
// site.config.highlight_code = Some(true);
//
// b.iter(|| site.load().unwrap());
//}
#[bench]
fn bench_loading_medium_blog_with_syntax_highlighting(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("medium-blog");
let mut site = Site::new(&path, "config.toml").unwrap();
site.config.highlight_code = Some(true);
b.iter(|| site.load().unwrap());
}
#[bench]
fn bench_loading_big_blog(b: &mut test::Bencher) {
@ -74,26 +72,26 @@ fn bench_loading_big_blog_with_syntax_highlighting(b: &mut test::Bencher) {
b.iter(|| site.load().unwrap());
}
#[bench]
fn bench_loading_huge_blog(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("huge-blog");
let mut site = Site::new(&path, "config.toml").unwrap();
b.iter(|| site.load().unwrap());
}
#[bench]
fn bench_loading_huge_blog_with_syntax_highlighting(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("huge-blog");
let mut site = Site::new(&path, "config.toml").unwrap();
site.config.highlight_code = Some(true);
b.iter(|| site.load().unwrap());
}
//#[bench]
//fn bench_loading_huge_blog(b: &mut test::Bencher) {
// let mut path = env::current_dir().unwrap().to_path_buf();
// path.push("benches");
// path.push("huge-blog");
// let mut site = Site::new(&path, "config.toml").unwrap();
//
// b.iter(|| site.load().unwrap());
//}
//
//#[bench]
//fn bench_loading_huge_blog_with_syntax_highlighting(b: &mut test::Bencher) {
// let mut path = env::current_dir().unwrap().to_path_buf();
// path.push("benches");
// path.push("huge-blog");
// let mut site = Site::new(&path, "config.toml").unwrap();
// site.config.highlight_code = Some(true);
//
// b.iter(|| site.load().unwrap());
//}
#[bench]
fn bench_loading_small_kb(b: &mut test::Bencher) {
@ -137,23 +135,23 @@ fn bench_loading_medium_kb_with_syntax_highlighting(b: &mut test::Bencher) {
b.iter(|| site.load().unwrap());
}
#[bench]
fn bench_loading_huge_kb(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("huge-kb");
let mut site = Site::new(&path, "config.toml").unwrap();
b.iter(|| site.load().unwrap());
}
#[bench]
fn bench_loading_huge_kb_with_syntax_highlighting(b: &mut test::Bencher) {
let mut path = env::current_dir().unwrap().to_path_buf();
path.push("benches");
path.push("huge-kb");
let mut site = Site::new(&path, "config.toml").unwrap();
site.config.highlight_code = Some(true);
b.iter(|| site.load().unwrap());
}
//#[bench]
//fn bench_loading_huge_kb(b: &mut test::Bencher) {
// let mut path = env::current_dir().unwrap().to_path_buf();
// path.push("benches");
// path.push("huge-kb");
// let mut site = Site::new(&path, "config.toml").unwrap();
//
// b.iter(|| site.load().unwrap());
//}
//
//#[bench]
//fn bench_loading_huge_kb_with_syntax_highlighting(b: &mut test::Bencher) {
// let mut path = env::current_dir().unwrap().to_path_buf();
// path.push("benches");
// path.push("huge-kb");
// let mut site = Site::new(&path, "config.toml").unwrap();
// site.config.highlight_code = Some(true);
//
// b.iter(|| site.load().unwrap());
//}

3
benches/unit.rs
View file

@ -1,3 +0,0 @@
//! Benchmarking individual functions of Gutenberg

6
src/bin/rebuild.rs
View file

@ -130,7 +130,8 @@ pub fn after_content_change(site: &mut Site, path: &Path) -> Result<()> {
// A section was edited
if is_section {
match site.add_section(path, true)? {
let section = Section::from_file(path, &site.config)?;
match site.add_section(section, true)? {
Some(prev) => {
// Updating a section
let current_meta = site.sections[path].meta.clone();
@ -166,7 +167,8 @@ pub fn after_content_change(site: &mut Site, path: &Path) -> Result<()> {
}
// A page was edited
match site.add_page(path, true)? {
let page = Page::from_file(path, &site.config)?;
match site.add_page(page, true)? {
Some(prev) => {
site.register_get_url_fn();
// Updating a page

60
src/site.rs
View file

@ -96,14 +96,46 @@ impl Site {
let base_path = self.base_path.to_string_lossy().replace("\\", "/");
let content_glob = format!("{}/{}", base_path, "content/**/*.md");
for entry in glob(&content_glob).unwrap().filter_map(|e| e.ok()) {
let path = entry.as_path();
if path.file_name().unwrap() == "_index.md" {
self.add_section(path, false)?;
} else {
self.add_page(path, false)?;
}
let (section_entries, page_entries): (Vec<_>, Vec<_>) = glob(&content_glob)
.unwrap()
.filter_map(|e| e.ok())
.partition(|ref entry| entry.as_path().file_name().unwrap() == "_index.md");
let sections = {
let config = &self.config;
section_entries
.into_par_iter()
.filter(|entry| entry.as_path().file_name().unwrap() == "_index.md")
.map(|entry| {
let path = entry.as_path();
Section::from_file(path, &config)
}).collect::<Vec<_>>()
};
let pages = {
let config = &self.config;
page_entries
.into_par_iter()
.filter(|entry| entry.as_path().file_name().unwrap() != "_index.md")
.map(|entry| {
let path = entry.as_path();
Page::from_file(path, &config)
}).collect::<Vec<_>>()
};
// Kinda duplicated code for add_section/add_page but necessary to do it that
// way because of the borrow checker
for section in sections {
let s = section?;
self.add_section(s, false)?;
}
for page in pages {
let p = page?;
self.add_page(p, false)?;
}
// Insert a default index section if necessary so we don't need to create
// a _index.md to render the index page
let index_path = self.base_path.join("content").join("_index.md");
@ -161,14 +193,14 @@ impl Site {
/// The `render` parameter is used in the serve command, when rebuilding a page.
/// If `true`, it will also render the markdown for that page
/// Returns the previous page struct if there was one
pub fn add_page(&mut self, path: &Path, render: bool) -> Result<Option<Page>> {
let page = Page::from_file(&path, &self.config)?;
pub fn add_page(&mut self, page: Page, render: bool) -> Result<Option<Page>> {
let path = page.file.path.clone();
self.permalinks.insert(page.file.relative.clone(), page.permalink.clone());
let prev = self.pages.insert(page.file.path.clone(), page);
if render {
let insert_anchor = self.find_parent_section_insert_anchor(&self.pages[path].file.parent);
let mut page = self.pages.get_mut(path).unwrap();
let insert_anchor = self.find_parent_section_insert_anchor(&self.pages[&path].file.parent);
let mut page = self.pages.get_mut(&path).unwrap();
page.render_markdown(&self.permalinks, &self.tera, &self.config, insert_anchor)?;
}
@ -179,13 +211,13 @@ impl Site {
/// The `render` parameter is used in the serve command, when rebuilding a page.
/// If `true`, it will also render the markdown for that page
/// Returns the previous section struct if there was one
pub fn add_section(&mut self, path: &Path, render: bool) -> Result<Option<Section>> {
let section = Section::from_file(path, &self.config)?;
pub fn add_section(&mut self, section: Section, render: bool) -> Result<Option<Section>> {
let path = section.file.path.clone();
self.permalinks.insert(section.file.relative.clone(), section.permalink.clone());
let prev = self.sections.insert(section.file.path.clone(), section);
if render {
let mut section = self.sections.get_mut(path).unwrap();
let mut section = self.sections.get_mut(&path).unwrap();
section.render_markdown(&self.permalinks, &self.tera, &self.config)?;
}