From 8473dff23d0016a333aef609e1dbf4a56659bfd0 Mon Sep 17 00:00:00 2001 From: cmal Date: Tue, 7 Aug 2018 12:12:12 +0200 Subject: [PATCH 01/11] Implement assets colocation in section --- components/content/src/section.rs | 112 +++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/components/content/src/section.rs b/components/content/src/section.rs index 58567f32..73727410 100644 --- a/components/content/src/section.rs +++ b/components/content/src/section.rs @@ -8,7 +8,7 @@ use serde::ser::{SerializeStruct, self}; use config::Config; use front_matter::{SectionFrontMatter, split_section_content}; use errors::{Result, ResultExt}; -use utils::fs::read_file; +use utils::fs::{read_file, find_related_assets}; use utils::templates::render_template; use utils::site::get_reading_analytics; use rendering::{RenderContext, Header, render_content}; @@ -33,6 +33,8 @@ pub struct Section { pub raw_content: String, /// The HTML rendered of the page pub content: String, + /// All the non-md files we found next to the .md file + pub assets: Vec, /// All direct pages of that section pub pages: Vec, /// All pages that cannot be sorted in this section @@ -54,6 +56,7 @@ impl Section { components: vec![], permalink: "".to_string(), raw_content: "".to_string(), + assets: vec![], content: "".to_string(), pages: vec![], ignored_pages: vec![], @@ -79,8 +82,35 @@ impl Section { pub fn from_file>(path: P, config: &Config) -> Result
{ let path = path.as_ref(); let content = read_file(path)?; + let mut section = Section::parse(path, &content, config)?; - Section::parse(path, &content, config) + if section.file.name == "_index" { + let parent_dir = path.parent().unwrap(); + let assets = find_related_assets(parent_dir); + + if let Some(ref globset) = config.ignored_content_globset { + // `find_related_assets` only scans the immediate directory (it is not recursive) so our + // filtering only needs to work against the file_name component, not the full suffix. If + // `find_related_assets` was changed to also return files in subdirectories, we could + // use `PathBuf.strip_prefix` to remove the parent directory and then glob-filter + // against the remaining path. Note that the current behaviour effectively means that + // the `ignored_content` setting in the config file is limited to single-file glob + // patterns (no "**" patterns). + section.assets = assets.into_iter() + .filter(|path| + match path.file_name() { + None => true, + Some(file) => !globset.is_match(file) + } + ).collect(); + } else { + section.assets = assets; + } + } else { + section.assets = vec![]; + } + + Ok(section) } pub fn get_template_name(&self) -> String { @@ -146,6 +176,15 @@ impl Section { pub fn is_child_page(&self, path: &PathBuf) -> bool { self.all_pages_path().contains(path) } + + /// Creates a vectors of asset URLs. + fn serialize_assets(&self) -> Vec { + self.assets.iter() + .filter_map(|asset| asset.file_name()) + .filter_map(|filename| filename.to_str()) + .map(|filename| self.path.clone() + filename) + .collect() + } } impl ser::Serialize for Section { @@ -165,6 +204,8 @@ impl ser::Serialize for Section { state.serialize_field("word_count", &word_count)?; state.serialize_field("reading_time", &reading_time)?; state.serialize_field("toc", &self.toc)?; + let assets = self.serialize_assets(); + state.serialize_field("assets", &assets)?; state.end() } } @@ -179,6 +220,7 @@ impl Default for Section { components: vec![], permalink: "".to_string(), raw_content: "".to_string(), + assets: vec![], content: "".to_string(), pages: vec![], ignored_pages: vec![], @@ -187,3 +229,69 @@ impl Default for Section { } } } + +#[cfg(test)] +mod tests { + use std::io::Write; + use std::fs::{File, create_dir}; + + use tempfile::tempdir; + use globset::{Glob, GlobSetBuilder}; + + use config::Config; + use super::Section; + + #[test] + fn section_with_assets_gets_right_info() { + let tmp_dir = tempdir().expect("create temp dir"); + let path = tmp_dir.path(); + create_dir(&path.join("content")).expect("create content temp dir"); + create_dir(&path.join("content").join("posts")).expect("create posts temp dir"); + let nested_path = path.join("content").join("posts").join("with-assets"); + create_dir(&nested_path).expect("create nested temp dir"); + let mut f = File::create(nested_path.join("_index.md")).unwrap(); + f.write_all(b"+++\n+++\n").unwrap(); + File::create(nested_path.join("example.js")).unwrap(); + File::create(nested_path.join("graph.jpg")).unwrap(); + File::create(nested_path.join("fail.png")).unwrap(); + + let res = Section::from_file( + nested_path.join("_index.md").as_path(), + &Config::default(), + ); + assert!(res.is_ok()); + let section = res.unwrap(); + assert_eq!(section.assets.len(), 3); + assert_eq!(section.permalink, "http://a-website.com/posts/with-assets/"); + } + + #[test] + fn section_with_ignored_assets_filters_out_correct_files() { + let tmp_dir = tempdir().expect("create temp dir"); + let path = tmp_dir.path(); + create_dir(&path.join("content")).expect("create content temp dir"); + create_dir(&path.join("content").join("posts")).expect("create posts temp dir"); + let nested_path = path.join("content").join("posts").join("with-assets"); + create_dir(&nested_path).expect("create nested temp dir"); + let mut f = File::create(nested_path.join("_index.md")).unwrap(); + f.write_all(b"+++\nslug=\"hey\"\n+++\n").unwrap(); + File::create(nested_path.join("example.js")).unwrap(); + File::create(nested_path.join("graph.jpg")).unwrap(); + File::create(nested_path.join("fail.png")).unwrap(); + + let mut gsb = GlobSetBuilder::new(); + gsb.add(Glob::new("*.{js,png}").unwrap()); + let mut config = Config::default(); + config.ignored_content_globset = Some(gsb.build().unwrap()); + + let res = Section::from_file( + nested_path.join("_index.md").as_path(), + &config, + ); + + assert!(res.is_ok()); + let page = res.unwrap(); + assert_eq!(page.assets.len(), 1); + assert_eq!(page.assets[0].file_name().unwrap().to_str(), Some("graph.jpg")); + } +} From 15190962ba210342572b955b3caa14adffbaf811 Mon Sep 17 00:00:00 2001 From: cmal Date: Tue, 7 Aug 2018 12:14:59 +0200 Subject: [PATCH 02/11] Copy relevant assets in case of colocation for section --- components/site/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index dd351ef7..5d4236ac 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -837,6 +837,12 @@ impl Site { } } + // Copy any asset we found previously into the same directory as the index.html + for asset in §ion.assets { + let asset_path = asset.as_path(); + copy(&asset_path, &output_path.join(asset_path.file_name().unwrap()))?; + } + if render_pages { section .pages From c7156a84f0ca4132c3380d75ba993f379a967cb5 Mon Sep 17 00:00:00 2001 From: cmal Date: Wed, 8 Aug 2018 10:51:40 +0200 Subject: [PATCH 03/11] Start implementing _index folder for section content/assets --- components/content/src/section.rs | 10 +++++++++- components/site/src/lib.rs | 16 +++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/components/content/src/section.rs b/components/content/src/section.rs index 73727410..905c532a 100644 --- a/components/content/src/section.rs +++ b/components/content/src/section.rs @@ -82,9 +82,17 @@ impl Section { pub fn from_file>(path: P, config: &Config) -> Result
{ let path = path.as_ref(); let content = read_file(path)?; + println!("Parsing from file {:?}", path); let mut section = Section::parse(path, &content, config)?; - if section.file.name == "_index" { + println!("filename {:?}", section.file.name); + // I don't see any reason why, but section.file.name always is "_index"! ← bug + + let file_name = path.file_name().unwrap(); + + // Is this check really necessary? Should the else case happen at all? + if file_name == "_index.md" || file_name == "index.md" { + // In any case, we're looking for assets inside parent directory let parent_dir = path.parent().unwrap(); let assets = find_related_assets(parent_dir); diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index 5d4236ac..b293b0ee 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -180,14 +180,20 @@ impl Site { let (section_entries, page_entries): (Vec<_>, Vec<_>) = glob(&content_glob) .unwrap() .filter_map(|e| e.ok()) - .partition(|entry| entry.as_path().file_name().unwrap() == "_index.md"); + // If file is _index.md and it doesn't have a sibling folder called _index containing an index.md (which have priority), we have a section + // If parent folder is _index and file is index.md then we have a section + .partition(|entry| ( (entry.as_path().file_name().unwrap() == "_index.md" && !entry.as_path().parent().unwrap().join("_index/index.md").is_file()) || (entry.as_path().parent().unwrap().file_name().unwrap() == "_index" && entry.as_path().file_name().unwrap() == "index.md"))); + + println!("Now section_entries"); + println!("{:?}", section_entries); let sections = { let config = &self.config; section_entries .into_par_iter() - .filter(|entry| entry.as_path().file_name().unwrap() == "_index.md") + // Is it really necessary to refilter for _index.md/index.md after the partition took place? + //.filter(|entry| entry.as_path().file_name().unwrap() == "_index.md") .map(|entry| { let path = entry.as_path(); Section::from_file(path, config) @@ -195,11 +201,15 @@ impl Site { .collect::>() }; + println!("Now sections:"); + println!("{:?}", sections); + let pages = { let config = &self.config; page_entries .into_par_iter() + // Same question. Do we have to refilter here given we have already partitioned results from the glob? .filter(|entry| entry.as_path().file_name().unwrap() != "_index.md") .map(|entry| { let path = entry.as_path(); @@ -216,7 +226,7 @@ impl Site { } // Insert a default index section if necessary so we don't need to create - // a _index.md to render the index page + // a _index.md to render the index page at the root of the site let index_path = self.index_section_path(); if let Some(ref index_section) = self.sections.get(&index_path) { if self.config.build_search_index && !index_section.meta.in_search_index { From 3b9c8c71b50954fed4b6840ccde7433eaccb65c3 Mon Sep 17 00:00:00 2001 From: cmal Date: Thu, 9 Aug 2018 11:51:01 +0200 Subject: [PATCH 04/11] Revert "Start implementing _index folder for section content/assets" This reverts commit c7156a84f0ca4132c3380d75ba993f379a967cb5. --- components/content/src/section.rs | 10 +--------- components/site/src/lib.rs | 16 +++------------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/components/content/src/section.rs b/components/content/src/section.rs index 905c532a..73727410 100644 --- a/components/content/src/section.rs +++ b/components/content/src/section.rs @@ -82,17 +82,9 @@ impl Section { pub fn from_file>(path: P, config: &Config) -> Result
{ let path = path.as_ref(); let content = read_file(path)?; - println!("Parsing from file {:?}", path); let mut section = Section::parse(path, &content, config)?; - println!("filename {:?}", section.file.name); - // I don't see any reason why, but section.file.name always is "_index"! ← bug - - let file_name = path.file_name().unwrap(); - - // Is this check really necessary? Should the else case happen at all? - if file_name == "_index.md" || file_name == "index.md" { - // In any case, we're looking for assets inside parent directory + if section.file.name == "_index" { let parent_dir = path.parent().unwrap(); let assets = find_related_assets(parent_dir); diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index b293b0ee..5d4236ac 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -180,20 +180,14 @@ impl Site { let (section_entries, page_entries): (Vec<_>, Vec<_>) = glob(&content_glob) .unwrap() .filter_map(|e| e.ok()) - // If file is _index.md and it doesn't have a sibling folder called _index containing an index.md (which have priority), we have a section - // If parent folder is _index and file is index.md then we have a section - .partition(|entry| ( (entry.as_path().file_name().unwrap() == "_index.md" && !entry.as_path().parent().unwrap().join("_index/index.md").is_file()) || (entry.as_path().parent().unwrap().file_name().unwrap() == "_index" && entry.as_path().file_name().unwrap() == "index.md"))); - - println!("Now section_entries"); - println!("{:?}", section_entries); + .partition(|entry| entry.as_path().file_name().unwrap() == "_index.md"); let sections = { let config = &self.config; section_entries .into_par_iter() - // Is it really necessary to refilter for _index.md/index.md after the partition took place? - //.filter(|entry| entry.as_path().file_name().unwrap() == "_index.md") + .filter(|entry| entry.as_path().file_name().unwrap() == "_index.md") .map(|entry| { let path = entry.as_path(); Section::from_file(path, config) @@ -201,15 +195,11 @@ impl Site { .collect::>() }; - println!("Now sections:"); - println!("{:?}", sections); - let pages = { let config = &self.config; page_entries .into_par_iter() - // Same question. Do we have to refilter here given we have already partitioned results from the glob? .filter(|entry| entry.as_path().file_name().unwrap() != "_index.md") .map(|entry| { let path = entry.as_path(); @@ -226,7 +216,7 @@ impl Site { } // Insert a default index section if necessary so we don't need to create - // a _index.md to render the index page at the root of the site + // a _index.md to render the index page let index_path = self.index_section_path(); if let Some(ref index_section) = self.sections.get(&index_path) { if self.config.build_search_index && !index_section.meta.in_search_index { From 77f8d96c9c3dda5af8146d65413f0804d2db029f Mon Sep 17 00:00:00 2001 From: cmal Date: Thu, 9 Aug 2018 11:53:45 +0200 Subject: [PATCH 05/11] Make comment more explicit --- components/site/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index 5d4236ac..da5f79d4 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -216,7 +216,7 @@ impl Site { } // Insert a default index section if necessary so we don't need to create - // a _index.md to render the index page + // a _index.md to render the index page at the root of the site let index_path = self.index_section_path(); if let Some(ref index_section) = self.sections.get(&index_path) { if self.config.build_search_index && !index_section.meta.in_search_index { From 739c2011a71d129e1ddb86a653fcadc6420e56db Mon Sep 17 00:00:00 2001 From: cmal Date: Thu, 9 Aug 2018 11:55:27 +0200 Subject: [PATCH 06/11] Remove redundant filtering operation (optimization) --- components/site/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index da5f79d4..d2e198b5 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -187,7 +187,6 @@ impl Site { section_entries .into_par_iter() - .filter(|entry| entry.as_path().file_name().unwrap() == "_index.md") .map(|entry| { let path = entry.as_path(); Section::from_file(path, config) @@ -200,7 +199,6 @@ impl Site { page_entries .into_par_iter() - .filter(|entry| entry.as_path().file_name().unwrap() != "_index.md") .map(|entry| { let path = entry.as_path(); Page::from_file(path, config) From 31479ff23bf36c35bd4f5e7fdd188a144742c491 Mon Sep 17 00:00:00 2001 From: cmal Date: Thu, 9 Aug 2018 11:58:09 +0200 Subject: [PATCH 07/11] Remove condition that's always true (optimization) --- components/content/src/section.rs | 40 ++++++++++++++----------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/components/content/src/section.rs b/components/content/src/section.rs index 73727410..6426eb94 100644 --- a/components/content/src/section.rs +++ b/components/content/src/section.rs @@ -84,30 +84,26 @@ impl Section { let content = read_file(path)?; let mut section = Section::parse(path, &content, config)?; - if section.file.name == "_index" { - let parent_dir = path.parent().unwrap(); - let assets = find_related_assets(parent_dir); + let parent_dir = path.parent().unwrap(); + let assets = find_related_assets(parent_dir); - if let Some(ref globset) = config.ignored_content_globset { - // `find_related_assets` only scans the immediate directory (it is not recursive) so our - // filtering only needs to work against the file_name component, not the full suffix. If - // `find_related_assets` was changed to also return files in subdirectories, we could - // use `PathBuf.strip_prefix` to remove the parent directory and then glob-filter - // against the remaining path. Note that the current behaviour effectively means that - // the `ignored_content` setting in the config file is limited to single-file glob - // patterns (no "**" patterns). - section.assets = assets.into_iter() - .filter(|path| - match path.file_name() { - None => true, - Some(file) => !globset.is_match(file) - } - ).collect(); - } else { - section.assets = assets; - } + if let Some(ref globset) = config.ignored_content_globset { + // `find_related_assets` only scans the immediate directory (it is not recursive) so our + // filtering only needs to work against the file_name component, not the full suffix. If + // `find_related_assets` was changed to also return files in subdirectories, we could + // use `PathBuf.strip_prefix` to remove the parent directory and then glob-filter + // against the remaining path. Note that the current behaviour effectively means that + // the `ignored_content` setting in the config file is limited to single-file glob + // patterns (no "**" patterns). + section.assets = assets.into_iter() + .filter(|path| + match path.file_name() { + None => true, + Some(file) => !globset.is_match(file) + } + ).collect(); } else { - section.assets = vec![]; + section.assets = assets; } Ok(section) From 26ffc318506f79777d54a1f887378d6e7ff37941 Mon Sep 17 00:00:00 2001 From: cmal Date: Fri, 10 Aug 2018 15:56:36 +0200 Subject: [PATCH 08/11] Document section assets + add example asset interaction from Markdown --- .../content/documentation/content/overview.md | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/docs/content/documentation/content/overview.md b/docs/content/documentation/content/overview.md index c2e30d1a..969e9665 100644 --- a/docs/content/documentation/content/overview.md +++ b/docs/content/documentation/content/overview.md @@ -40,18 +40,34 @@ While not shown in the example, sections can be nested indefinitely. ## Assets colocation The `content` directory is not limited to markup files though: it's natural to want to co-locate a page and some related -assets. +assets, for instance images or spreadsheets. Gutenberg supports that pattern out of the box for both sections and pages. + +Any non-markdown file you add in the page/section folder will be copied alongside the generated page when building the site, +which allows us to use a relative path to access them. + +For pages to use assets colocation, they should not be placed directly in their section folder (such as `latest-experiment.md`), but as an `index.md` file +in a dedicated folder (`latest-experiment/index.md`), like so: -Gutenberg supports that pattern out of the box: create a folder, add a `index.md` file and as many non-markdown files as you want. -Those assets will be copied in the same folder when building the site which allows you to use a relative path to access them. ```bash -└── with-assets - ├── index.md - └── yavascript.js +└── research + ├── latest-experiment + │ ├── index.md + │ └── yavascript.js + ├── _index.md + └── research.jpg ``` -By default, this page will get the folder name (`with-assets` in this case) as its slug. +In this setup, you may access `research.jpg` from your 'research' section, +and `yavascript.js` from your 'latest-experiment' directly within the Markdown: + +```markdown +Check out the complete program [here](yavascript.js). It's **really cool free-software**! +``` + +By default, this page will get the folder name as its slug. So its permalink would be in the form of `https://example.com/research/latest-experiment/` + +### Excluding files from assets It is possible to ignore selected asset files using the [ignored_content](./documentation/getting-started/configuration.md) setting in the config file. From b8bc13c35124d2885c82c9a9f49ac7a6daba44bb Mon Sep 17 00:00:00 2001 From: cmal Date: Fri, 10 Aug 2018 15:57:41 +0200 Subject: [PATCH 09/11] Make link more specific --- docs/content/documentation/content/page.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/documentation/content/page.md b/docs/content/documentation/content/page.md index 7d61c27c..733f412b 100644 --- a/docs/content/documentation/content/page.md +++ b/docs/content/documentation/content/page.md @@ -20,7 +20,7 @@ content directory `about.md` would also create a page at `[base_url]/about`. As you can see, creating an `about.md` file is exactly equivalent to creating an `about/index.md` file. The only difference between the two methods is that creating the `about` folder allows you to use asset colocation, as discussed in the -[Overview](./documentation/content/overview.md) section of this documentation. +[Overview](./documentation/content/overview.md#assets-colocation) section of this documentation. ## Front-matter From ec65d01a7206eb240f77ffaa150d6a7646be77c0 Mon Sep 17 00:00:00 2001 From: cmal Date: Fri, 10 Aug 2018 15:59:03 +0200 Subject: [PATCH 10/11] Add assets to Sections variables --- docs/content/documentation/templates/pages-sections.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/content/documentation/templates/pages-sections.md b/docs/content/documentation/templates/pages-sections.md index 297c95b3..bb4d7544 100644 --- a/docs/content/documentation/templates/pages-sections.md +++ b/docs/content/documentation/templates/pages-sections.md @@ -78,6 +78,8 @@ word_count: Number; reading_time: Number; // See the Table of contents section below for more details toc: Array
; +// Paths of colocated assets, relative to the content directory +assets: Array; ``` ## Table of contents From 7875387a04289515d70984c4aa6f482c05ccdcb4 Mon Sep 17 00:00:00 2001 From: cmal Date: Fri, 10 Aug 2018 15:59:16 +0200 Subject: [PATCH 11/11] Document assets on sections doc --- docs/content/documentation/content/section.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/content/documentation/content/section.md b/docs/content/documentation/content/section.md index 7baba791..74c32e0a 100644 --- a/docs/content/documentation/content/section.md +++ b/docs/content/documentation/content/section.md @@ -14,6 +14,8 @@ not have any content or metadata. If you would like to add content or metadata, `_index.md` file at the root of the `content` folder and edit it just as you would edit any other `_index.md` file; your `index.html` template will then have access to that content and metadata. +Any non-Markdown file in the section folder is added to the `assets` collection of the section, as explained in the [Content Overview](./documentation/content/overview.md#assets-colocation). These files are then available from the Markdown using relative links. + ## Front-matter The `_index.md` file within a folder defines the content and metadata for that section. To set