diff --git a/.gitignore b/.gitignore index 5085055a..23e585f0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ target .idea/ -components/site/test_site/public +test_site/public docs/public small-blog diff --git a/CHANGELOG.md b/CHANGELOG.md index 46dfb3ff..a75c1e0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ to the public directory - Do not require themes to have a static folder - Now supports indented Sass syntax +- Add search index building ## 0.3.2 (2018-03-05) diff --git a/Cargo.lock b/Cargo.lock index 613da997..7a33afe5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6,6 +6,19 @@ dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ammonia" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "html5ever 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "maplit 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tendril 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ansi_term" version = "0.11.0" @@ -149,7 +162,7 @@ dependencies = [ [[package]] name = "clap" -version = "2.31.1" +version = "2.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -241,6 +254,14 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "debug_unreachable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "dtoa" version = "0.4.2" @@ -262,6 +283,21 @@ name = "either" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "elasticlunr-rs" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "rust-stemmers 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)", + "strum 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "strum_macros 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "error-chain" version = "0.11.0" @@ -354,6 +390,15 @@ name = "fuchsia-zircon-sys" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "futf" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "debug_unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "gcc" version = "0.3.54" @@ -386,7 +431,7 @@ name = "gutenberg" version = "0.3.3" dependencies = [ "chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "clap 2.31.1 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", "content 0.1.0", "ctrlc 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "errors 0.1.0", @@ -412,6 +457,18 @@ dependencies = [ "syntect 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "html5ever" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "markup5ever 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "httparse" version = "1.2.4" @@ -547,6 +604,31 @@ dependencies = [ "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "maplit" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "markup5ever" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)", + "string_cache 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "string_cache_codegen 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tendril 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "matches" version = "0.1.6" @@ -881,6 +963,11 @@ dependencies = [ "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "proc-macro2" version = "0.2.3" @@ -1031,6 +1118,15 @@ dependencies = [ "utils 0.1.0", ] +[[package]] +name = "rust-stemmers" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rustc-demangle" version = "0.1.7" @@ -1073,6 +1169,17 @@ name = "scopeguard" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "search" +version = "0.1.0" +dependencies = [ + "ammonia 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "content 0.1.0", + "elasticlunr-rs 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "errors 0.1.0", + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "sequence_trie" version = "0.3.5" @@ -1146,6 +1253,7 @@ dependencies = [ "pagination 0.1.0", "rayon 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "sass-rs 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "search 0.1.0", "serde 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", "taxonomies 0.1.0", @@ -1153,7 +1261,6 @@ dependencies = [ "templates 0.1.0", "tera 0.11.5 (registry+https://github.com/rust-lang/crates.io-index)", "utils 0.1.0", - "walkdir 2.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1190,11 +1297,55 @@ dependencies = [ "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "string_cache" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "debug_unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", + "precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", + "string_cache_codegen 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "string_cache_codegen" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_generator 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "string_cache_shared" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "strsim" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "strum" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "strum_macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "syn" version = "0.11.11" @@ -1282,6 +1433,16 @@ dependencies = [ "utils 0.1.0", ] +[[package]] +name = "tendril" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futf 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "utf-8 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tera" version = "0.11.5" @@ -1427,6 +1588,14 @@ name = "unidecode" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unreachable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "unreachable" version = "1.0.0" @@ -1453,6 +1622,11 @@ dependencies = [ "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "utf-8" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "utf8-ranges" version = "1.0.0" @@ -1465,6 +1639,7 @@ dependencies = [ "errors 0.1.0", "tempdir 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "tera 0.11.5 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 2.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1563,6 +1738,7 @@ dependencies = [ [metadata] "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" +"checksum ammonia 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fd4c682378117e4186a492b2252b9537990e1617f44aed9788b9a1149de45477" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" "checksum atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "af80143d6f7608d746df1520709e5d141c96f240b0e62b0aa41bdfb53374d9d4" @@ -1582,15 +1758,17 @@ dependencies = [ "checksum cc 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "d9324127e719125ec8a16e6e509abc4c641e773621b50aea695af3f005656d61" "checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" "checksum chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7c20ebe0b2b08b0aeddba49c609fe7957ba2e33449882cb186a180bc60682fa9" -"checksum clap 2.31.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5dc18f6f4005132120d9711636b32c46a233fad94df6217fa1d81c5e97a9f200" +"checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536" "checksum cmake 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)" = "56d741ea7a69e577f6d06b36b7dff4738f680593dc27a701ffa8506b73ce28bb" "checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3" "checksum crossbeam-epoch 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "59796cc6cbbdc6bb319161349db0c3250ec73ec7fcb763a51065ec4e2e158552" "checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9" "checksum ctrlc 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "653abc99aa905f693d89df4797fadc08085baee379db92be9f2496cefe8a6f2c" +"checksum debug_unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9a032eac705ca39214d169f83e3d3da290af06d8d1d344d1baad2fd002dca4b3" "checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab" "checksum duct 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8c553d79f40e74f7f611e49bf3429b6760cff79596b61818291c27cc0b18549d" "checksum either 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "740178ddf48b1a9e878e6d6509a1442a2d42fd2928aae8e7a6f8a36fb01981b3" +"checksum elasticlunr-rs 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18c5413e83a7be7e86ce0afce03b23508db56082edbb2b28a178e6275fa3957d" "checksum error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff511d5dc435d703f4971bc399647c9bc38e20cb41452e3b9feb4765419ed3f3" "checksum filetime 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "714653f3e34871534de23771ac7b26e999651a0a228f47beb324dfdf1dd4b10f" "checksum flate2 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9fac2277e84e5e858483756647a9d0aa8d9a2b7cba517fd84325a0aaa69a0909" @@ -1600,10 +1778,12 @@ dependencies = [ "checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum futf 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "51f93f3de6ba1794dcd5810b3546d004600a59a98266487c8407bc4b24e398f3" "checksum gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)" = "5e33ec290da0d127825013597dbdfc28bee4964690c7ce1166cbc2a7bd08b1bb" "checksum getopts 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "b900c08c1939860ce8b54dc6a89e26e00c04c380fd0e09796799bd7f12861e05" "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" "checksum globset 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1e96ab92362c06811385ae9a34d2698e8a1160745e0c78fbb434a44c8de3fabc" +"checksum html5ever 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e579ac8647178ab915d400d7d22938bda5cd351c6c62e1c294d56884ccfc75fe" "checksum httparse 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c2f407128745b78abc95c0ffbe4e5d37427fdc0d45470710cfef8c44522a2e37" "checksum humansize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6cab2627acfc432780848602f3f558f7e9dd427352224b0d9324025796d2a5e" "checksum hyper 0.10.13 (registry+https://github.com/rust-lang/crates.io-index)" = "368cb56b2740ebf4230520e2b90ebb0461e69034d85d1945febd9b3971426db2" @@ -1622,6 +1802,9 @@ dependencies = [ "checksum linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70fb39025bc7cdd76305867c4eccf2f2dcf6e9a57f5b21a93e1c2d86cd03ec9e" "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" "checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" +"checksum mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +"checksum maplit 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "08cbb6b4fef96b6d77bfc40ec491b1690c779e77b05cd9f07f787ed376fd4c43" +"checksum markup5ever 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfedc97d5a503e96816d10fedcd5b42f760b2e525ce2f7ec71f6a41780548475" "checksum matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376" "checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" @@ -1657,6 +1840,7 @@ dependencies = [ "checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903" "checksum plist 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c61ac2afed2856590ae79d6f358a24b85ece246d2aa134741a66d589519b7503" "checksum plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a6a0dc3910bc8db877ffed8e457763b317cf880df4ae19109b9f77d277cf6e0" +"checksum precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" "checksum proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cd07deb3c6d1d9ff827999c7f9b04cdfd66b1b17ae508e14fe47b620f2282ae0" "checksum pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d6fdf85cda6cadfae5428a54661d431330b312bc767ddbc57adbedc24da66e32" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" @@ -1671,6 +1855,7 @@ dependencies = [ "checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e" "checksum regex-syntax 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b2550876c31dc914696a6c2e01cbce8afba79a93c8ae979d2fe051c0230b3756" "checksum remove_dir_all 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b5d2f806b0fcdabd98acd380dc8daef485e22bcb7cddc811d1337967f2528cf5" +"checksum rust-stemmers 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fbf06149ec391025664a5634200ced1afb489f0f3f8a140d515ebc0eb04b4bc0" "checksum rustc-demangle 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11fb43a206a04116ffd7cfcf9bcb941f8eb6cc7ff667272246b0a1c74259a3cb" "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" "checksum same-file 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cfb6eded0b06a0b512c8ddbcf04089138c9b4362c2f696f3c3d76039d68f3637" @@ -1690,12 +1875,18 @@ dependencies = [ "checksum slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fdeff4cd9ecff59ec7e3744cbca73dfe5ac35c2aedb2cfba8a1c715a18912e9d" "checksum slug 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f5ff4b43cb07b86c5f9236c92714a22cdf9e5a27a7d85e398e2c9403328cb8" "checksum staticfile 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "babd3fa68bb7e3994ce181c5f21ff3ff5fffef7b18b8a10163b45e4dafc6fb86" +"checksum string_cache 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39cb4173bcbd1319da31faa5468a7e3870683d7a237150b0b0aaafd546f6ad12" +"checksum string_cache_codegen 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "479cde50c3539481f33906a387f2bd17c8e87cb848c35b6021d41fb81ff9b4d7" +"checksum string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" +"checksum strum 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "099e21b5dd6dd07b5adcf8c4b723a7c0b7efd7a9359bf963d58c0caae8532545" +"checksum strum_macros 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0dd9bd569e88028750e3ae5c25616b8278ac16a8e61aba4339195c72396d49e1" "checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" "checksum syn 0.12.14 (registry+https://github.com/rust-lang/crates.io-index)" = "8c5bc2d6ff27891209efa5f63e9de78648d7801f085e4653701a692ce938d6fd" "checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" "checksum syntect 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db9fffcb25a761118df53811bd1cfcd54cf57fcbc51e1ea3167ae263477129ad" "checksum tempdir 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f73eebdb68c14bcb24aef74ea96079830e7fa7b31a6106e42ea7ee887c1e134e" +"checksum tendril 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9de21546595a0873061940d994bbbc5c35f024ae4fd61ec5c5b159115684f508" "checksum tera 0.11.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fc1a35d04c2444875b1319293fbc72c00215ae6220f8c70f9f14fefa5eaae0c6" "checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" "checksum term-painter 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "dcaa948f0e3e38470cd8dc8dcfe561a75c9e43f28075bb183845be2b9b3c08cf" @@ -1715,9 +1906,11 @@ dependencies = [ "checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum unidecode 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2adb95ee07cd579ed18131f2d9e7a17c25a4b76022935c7f2460d2bfae89fd2" +"checksum unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f30360d7979f5e9c6e6cea48af192ea8fab4afb3cf72597154b8f08935bc9c7f" "checksum url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f808aadd8cfec6ef90e4a14eb46f24511824d1ac596b9682703c87056c8678b7" +"checksum utf-8 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f1262dfab4c30d5cb7c07026be00ee343a6cf5027fdc0104a9160f354e5db75c" "checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" "checksum vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "887b5b631c2ad01628bbbaa7dd4c869f80d3186688f8d0b6f58774fbe324988c" "checksum version_check 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6b772017e347561807c1aa192438c5fd74242a670a6cffacc40f2defd1dc069d" diff --git a/Cargo.toml b/Cargo.toml index 4448eb26..232ac49e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,4 +52,5 @@ members = [ "components/taxonomies", "components/templates", "components/utils", + "components/search", ] diff --git a/components/config/Cargo.toml b/components/config/Cargo.toml index 298f8131..3515145a 100644 --- a/components/config/Cargo.toml +++ b/components/config/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "config" version = "0.1.0" -authors = ["Vincent Prouillet "] +authors = ["Vincent Prouillet "] [dependencies] toml = "0.4" diff --git a/components/content/Cargo.toml b/components/content/Cargo.toml index 07c21634..1aab5094 100644 --- a/components/content/Cargo.toml +++ b/components/content/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "content" version = "0.1.0" -authors = ["Vincent Prouillet "] +authors = ["Vincent Prouillet "] [dependencies] tera = "0.11" diff --git a/components/content/src/page.rs b/components/content/src/page.rs index c02680d2..26ebbf2b 100644 --- a/components/content/src/page.rs +++ b/components/content/src/page.rs @@ -75,7 +75,7 @@ impl Page { } pub fn is_draft(&self) -> bool { - self.meta.draft.unwrap_or(false) + self.meta.draft } /// Parse a page given the content of the .md file diff --git a/components/content/src/section.rs b/components/content/src/section.rs index 8ce6dfea..d8a9ae09 100644 --- a/components/content/src/section.rs +++ b/components/content/src/section.rs @@ -104,7 +104,7 @@ impl Section { config.highlight_theme.clone(), &self.permalink, permalinks, - self.meta.insert_anchor_links.unwrap() + self.meta.insert_anchor_links, ); let res = markdown_to_html(&self.raw_content, &context)?; self.content = res.0; diff --git a/components/content/src/sorting.rs b/components/content/src/sorting.rs index 89ef9c90..b96c6b1d 100644 --- a/components/content/src/sorting.rs +++ b/components/content/src/sorting.rs @@ -149,7 +149,7 @@ mod tests { fn create_draft_page_with_order(order: usize) -> Page { let mut front_matter = PageFrontMatter::default(); front_matter.order = Some(order); - front_matter.draft = Some(true); + front_matter.draft = true; Page::new("content/hello.md", front_matter) } diff --git a/components/errors/Cargo.toml b/components/errors/Cargo.toml index a43ee054..285bfeda 100644 --- a/components/errors/Cargo.toml +++ b/components/errors/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "errors" version = "0.1.0" -authors = ["Vincent Prouillet "] +authors = ["Vincent Prouillet "] [dependencies] error-chain = "0.11" diff --git a/components/front_matter/Cargo.toml b/components/front_matter/Cargo.toml index 564d08c7..a551ba31 100644 --- a/components/front_matter/Cargo.toml +++ b/components/front_matter/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "front_matter" version = "0.1.0" -authors = ["Vincent Prouillet "] +authors = ["Vincent Prouillet "] [dependencies] tera = "0.11" diff --git a/components/front_matter/src/page.rs b/components/front_matter/src/page.rs index ca2c95ac..86cde2a9 100644 --- a/components/front_matter/src/page.rs +++ b/components/front_matter/src/page.rs @@ -62,6 +62,7 @@ fn fix_toml_dates(table: Map) -> Value { /// The front matter of every page #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] pub struct PageFrontMatter { /// of the page pub title: Option<String>, @@ -71,7 +72,7 @@ pub struct PageFrontMatter { #[serde(default, deserialize_with = "from_toml_datetime")] pub date: Option<String>, /// Whether this page is a draft and should be ignored for pagination etc - pub draft: Option<bool>, + pub draft: bool, /// The page slug. Will be used instead of the filename if present /// Can't be an empty string if present pub slug: Option<String>, @@ -90,12 +91,15 @@ pub struct PageFrontMatter { /// All aliases for that page. Gutenberg will create HTML templates that will /// redirect to this #[serde(skip_serializing)] - pub aliases: Option<Vec<String>>, + pub aliases: Vec<String>, /// Specify a template different from `page.html` to use for that page #[serde(skip_serializing)] pub template: Option<String>, + /// Whether the page is included in the search index + /// Defaults to `true` but is only used if search if explicitly enabled in the config. + #[serde(skip_serializing)] + pub in_search_index: bool, /// Any extra parameter present in the front matter - #[serde(default)] pub extra: Map<String, Value>, } @@ -166,14 +170,15 @@ impl Default for PageFrontMatter { title: None, description: None, date: None, - draft: None, + draft: false, slug: None, path: None, tags: None, category: None, order: None, weight: None, - aliases: None, + aliases: Vec::new(), + in_search_index: true, template: None, extra: Map::new(), } diff --git a/components/front_matter/src/section.rs b/components/front_matter/src/section.rs index 8ceedc1b..64eb27dc 100644 --- a/components/front_matter/src/section.rs +++ b/components/front_matter/src/section.rs @@ -12,6 +12,7 @@ static DEFAULT_PAGINATE_PATH: &'static str = "page"; /// The front matter of every section #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] pub struct SectionFrontMatter { /// <title> of the page pub title: Option<String>, @@ -19,11 +20,11 @@ pub struct SectionFrontMatter { pub description: Option<String>, /// Whether to sort by "date", "order", "weight" or "none". Defaults to `none`. #[serde(skip_serializing)] - pub sort_by: Option<SortBy>, + pub sort_by: SortBy, /// Used by the parent section to order its subsections. - /// Higher values means it will be at the end. + /// Higher values means it will be at the end. Defaults to `0` #[serde(skip_serializing)] - pub weight: Option<usize>, + pub weight: usize, /// Optional template, if we want to specify which template to render for that section #[serde(skip_serializing)] pub template: Option<String>, @@ -32,59 +33,38 @@ pub struct SectionFrontMatter { pub paginate_by: Option<usize>, /// Path to be used by pagination: the page number will be appended after it. Defaults to `page`. #[serde(skip_serializing)] - pub paginate_path: Option<String>, + pub paginate_path: String, /// Whether to insert a link for each header like the ones you can see in this site if you hover one /// The default template can be overridden by creating a `anchor-link.html` in the `templates` directory - pub insert_anchor_links: Option<InsertAnchor>, + pub insert_anchor_links: InsertAnchor, /// Whether to render that section or not. Defaults to `true`. /// Useful when the section is only there to organize things but is not meant /// to be used directly, like a posts section in a personal site #[serde(skip_serializing)] - pub render: Option<bool>, + pub render: bool, /// Whether to redirect when landing on that section. Defaults to `None`. /// Useful for the same reason as `render` but when you don't want a 404 when /// landing on the root section page #[serde(skip_serializing)] pub redirect_to: Option<String>, + /// Whether the section content and its pages/subsections are included in the index. + /// Defaults to `true` but is only used if search if explicitly enabled in the config. + #[serde(skip_serializing)] + pub in_search_index: bool, /// Any extra parameter present in the front matter - pub extra: Option<HashMap<String, Value>>, + pub extra: HashMap<String, Value>, } impl SectionFrontMatter { pub fn parse(toml: &str) -> Result<SectionFrontMatter> { - let mut f: SectionFrontMatter = match toml::from_str(toml) { + let f: SectionFrontMatter = match toml::from_str(toml) { Ok(d) => d, Err(e) => bail!(e), }; - if f.paginate_path.is_none() { - f.paginate_path = Some(DEFAULT_PAGINATE_PATH.to_string()); - } - - if f.render.is_none() { - f.render = Some(true); - } - - if f.sort_by.is_none() { - f.sort_by = Some(SortBy::None); - } - - if f.insert_anchor_links.is_none() { - f.insert_anchor_links = Some(InsertAnchor::None); - } - - if f.weight.is_none() { - f.weight = Some(0); - } - Ok(f) } - /// Returns the current sorting method, defaults to `None` (== no sorting) - pub fn sort_by(&self) -> SortBy { - self.sort_by.unwrap() - } - /// Only applies to section, whether it is paginated or not. pub fn is_paginated(&self) -> bool { match self.paginate_by { @@ -92,10 +72,6 @@ impl SectionFrontMatter { None => false } } - - pub fn should_render(&self) -> bool { - self.render.unwrap() - } } impl Default for SectionFrontMatter { @@ -103,15 +79,16 @@ impl Default for SectionFrontMatter { SectionFrontMatter { title: None, description: None, - sort_by: Some(SortBy::None), - weight: Some(0), + sort_by: SortBy::None, + weight: 0, template: None, paginate_by: None, - paginate_path: Some(DEFAULT_PAGINATE_PATH.to_string()), - render: Some(true), + paginate_path: DEFAULT_PAGINATE_PATH.to_string(), + render: true, redirect_to: None, - insert_anchor_links: Some(InsertAnchor::None), - extra: None, + insert_anchor_links: InsertAnchor::None, + in_search_index: true, + extra: HashMap::new(), } } } diff --git a/components/highlighting/Cargo.toml b/components/highlighting/Cargo.toml index 0f45a8eb..b2fed89a 100644 --- a/components/highlighting/Cargo.toml +++ b/components/highlighting/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "highlighting" version = "0.1.0" -authors = ["Vincent Prouillet <vincent@wearewizards.io>"] +authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"] [dependencies] lazy_static = "1" diff --git a/components/pagination/Cargo.toml b/components/pagination/Cargo.toml index 313b85ba..0ee865f8 100644 --- a/components/pagination/Cargo.toml +++ b/components/pagination/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pagination" version = "0.1.0" -authors = ["Vincent Prouillet <vincent@wearewizards.io>"] +authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"] [dependencies] tera = "0.11" diff --git a/components/pagination/src/lib.rs b/components/pagination/src/lib.rs index 2d8b2fc5..45bc3abf 100644 --- a/components/pagination/src/lib.rs +++ b/components/pagination/src/lib.rs @@ -72,13 +72,9 @@ impl<'a> Paginator<'a> { /// It will always at least create one pager (the first) even if there are no pages to paginate pub fn new(all_pages: &'a [Page], section: &'a Section) -> Paginator<'a> { let paginate_by = section.meta.paginate_by.unwrap(); - let paginate_path = match section.meta.paginate_path { - Some(ref p) => p, - None => unreachable!(), - }; - let mut pages = vec![]; let mut current_page = vec![]; + for page in all_pages { current_page.push(page); @@ -99,7 +95,7 @@ impl<'a> Paginator<'a> { continue; } - let page_path = format!("{}/{}/", paginate_path, index + 1); + let page_path = format!("{}/{}/", section.meta.paginate_path, index + 1); let permalink = format!("{}{}", section.permalink, page_path); let pager_path = if section.is_index() { page_path @@ -189,7 +185,7 @@ mod tests { fn create_section(is_index: bool) -> Section { let mut f = SectionFrontMatter::default(); f.paginate_by = Some(2); - f.paginate_path = Some("page".to_string()); + f.paginate_path = "page".to_string(); let mut s = Section::new("content/_index.md", f); if !is_index { s.path = "posts/".to_string(); diff --git a/components/rebuild/Cargo.toml b/components/rebuild/Cargo.toml index 1e001d78..a6ae5b70 100644 --- a/components/rebuild/Cargo.toml +++ b/components/rebuild/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rebuild" version = "0.1.0" -authors = ["Vincent Prouillet <vincent@wearewizards.io>"] +authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"] [dependencies] errors = { path = "../errors" } diff --git a/components/rebuild/src/lib.rs b/components/rebuild/src/lib.rs index de1d3528..fac94802 100644 --- a/components/rebuild/src/lib.rs +++ b/components/rebuild/src/lib.rs @@ -60,7 +60,7 @@ fn find_section_front_matter_changes(current: &SectionFrontMatter, new: &Section // We want to hide the section // TODO: what to do on redirect_path change? - if current.should_render() && !new.should_render() { + if current.render && !new.render { changes_needed.push(SectionChangesNeeded::Delete); // Nothing else we can do return changes_needed; @@ -383,14 +383,14 @@ mod tests { #[test] fn can_find_sort_changes_in_section_frontmatter() { - let new = SectionFrontMatter { sort_by: Some(SortBy::Date), ..SectionFrontMatter::default() }; + let new = SectionFrontMatter { sort_by: SortBy::Date, ..SectionFrontMatter::default() }; let changes = find_section_front_matter_changes(&SectionFrontMatter::default(), &new); assert_eq!(changes, vec![SectionChangesNeeded::Sort, SectionChangesNeeded::Render]); } #[test] fn can_find_render_changes_in_section_frontmatter() { - let new = SectionFrontMatter { render: Some(false), ..SectionFrontMatter::default() }; + let new = SectionFrontMatter { render: false, ..SectionFrontMatter::default() }; let changes = find_section_front_matter_changes(&SectionFrontMatter::default(), &new); assert_eq!(changes, vec![SectionChangesNeeded::Delete]); } diff --git a/components/rendering/Cargo.toml b/components/rendering/Cargo.toml index 576016fc..6a19b2dd 100644 --- a/components/rendering/Cargo.toml +++ b/components/rendering/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rendering" version = "0.1.0" -authors = ["Vincent Prouillet <vincent@wearewizards.io>"] +authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"] [dependencies] tera = "0.11" diff --git a/components/search/Cargo.toml b/components/search/Cargo.toml new file mode 100644 index 00000000..97aca1aa --- /dev/null +++ b/components/search/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "search" +version = "0.1.0" +authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"] + +[dependencies] +elasticlunr-rs = "2" +ammonia = "1" +lazy_static = "1" + +errors = { path = "../errors" } +content = { path = "../content" } diff --git a/components/search/src/elasticlunr.min.js b/components/search/src/elasticlunr.min.js new file mode 100644 index 00000000..06cc9b32 --- /dev/null +++ b/components/search/src/elasticlunr.min.js @@ -0,0 +1,10 @@ +/** + * elasticlunr - http://weixsong.github.io + * Lightweight full-text search engine in Javascript for browser search and offline search. - 0.9.5 + * + * Copyright (C) 2017 Oliver Nightingale + * Copyright (C) 2017 Wei Song + * MIT Licensed + * @license + */ +!function(){function e(e){if(null===e||"object"!=typeof e)return e;var t=e.constructor();for(var n in e)e.hasOwnProperty(n)&&(t[n]=e[n]);return t}var t=function(e){var n=new t.Index;return n.pipeline.add(t.trimmer,t.stopWordFilter,t.stemmer),e&&e.call(n,n),n};t.version="0.9.5",lunr=t,t.utils={},t.utils.warn=function(e){return function(t){e.console&&console.warn&&console.warn(t)}}(this),t.utils.toString=function(e){return void 0===e||null===e?"":e.toString()},t.EventEmitter=function(){this.events={}},t.EventEmitter.prototype.addListener=function(){var e=Array.prototype.slice.call(arguments),t=e.pop(),n=e;if("function"!=typeof t)throw new TypeError("last argument must be a function");n.forEach(function(e){this.hasHandler(e)||(this.events[e]=[]),this.events[e].push(t)},this)},t.EventEmitter.prototype.removeListener=function(e,t){if(this.hasHandler(e)){var n=this.events[e].indexOf(t);-1!==n&&(this.events[e].splice(n,1),0==this.events[e].length&&delete this.events[e])}},t.EventEmitter.prototype.emit=function(e){if(this.hasHandler(e)){var t=Array.prototype.slice.call(arguments,1);this.events[e].forEach(function(e){e.apply(void 0,t)},this)}},t.EventEmitter.prototype.hasHandler=function(e){return e in this.events},t.tokenizer=function(e){if(!arguments.length||null===e||void 0===e)return[];if(Array.isArray(e)){var n=e.filter(function(e){return null===e||void 0===e?!1:!0});n=n.map(function(e){return t.utils.toString(e).toLowerCase()});var i=[];return n.forEach(function(e){var n=e.split(t.tokenizer.seperator);i=i.concat(n)},this),i}return e.toString().trim().toLowerCase().split(t.tokenizer.seperator)},t.tokenizer.defaultSeperator=/[\s\-]+/,t.tokenizer.seperator=t.tokenizer.defaultSeperator,t.tokenizer.setSeperator=function(e){null!==e&&void 0!==e&&"object"==typeof e&&(t.tokenizer.seperator=e)},t.tokenizer.resetSeperator=function(){t.tokenizer.seperator=t.tokenizer.defaultSeperator},t.tokenizer.getSeperator=function(){return t.tokenizer.seperator},t.Pipeline=function(){this._queue=[]},t.Pipeline.registeredFunctions={},t.Pipeline.registerFunction=function(e,n){n in t.Pipeline.registeredFunctions&&t.utils.warn("Overwriting existing registered function: "+n),e.label=n,t.Pipeline.registeredFunctions[n]=e},t.Pipeline.getRegisteredFunction=function(e){return e in t.Pipeline.registeredFunctions!=!0?null:t.Pipeline.registeredFunctions[e]},t.Pipeline.warnIfFunctionNotRegistered=function(e){var n=e.label&&e.label in this.registeredFunctions;n||t.utils.warn("Function is not registered with pipeline. This may cause problems when serialising the index.\n",e)},t.Pipeline.load=function(e){var n=new t.Pipeline;return e.forEach(function(e){var i=t.Pipeline.getRegisteredFunction(e);if(!i)throw new Error("Cannot load un-registered function: "+e);n.add(i)}),n},t.Pipeline.prototype.add=function(){var e=Array.prototype.slice.call(arguments);e.forEach(function(e){t.Pipeline.warnIfFunctionNotRegistered(e),this._queue.push(e)},this)},t.Pipeline.prototype.after=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._queue.indexOf(e);if(-1===i)throw new Error("Cannot find existingFn");this._queue.splice(i+1,0,n)},t.Pipeline.prototype.before=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._queue.indexOf(e);if(-1===i)throw new Error("Cannot find existingFn");this._queue.splice(i,0,n)},t.Pipeline.prototype.remove=function(e){var t=this._queue.indexOf(e);-1!==t&&this._queue.splice(t,1)},t.Pipeline.prototype.run=function(e){for(var t=[],n=e.length,i=this._queue.length,o=0;n>o;o++){for(var r=e[o],s=0;i>s&&(r=this._queue[s](r,o,e),void 0!==r&&null!==r);s++);void 0!==r&&null!==r&&t.push(r)}return t},t.Pipeline.prototype.reset=function(){this._queue=[]},t.Pipeline.prototype.get=function(){return this._queue},t.Pipeline.prototype.toJSON=function(){return this._queue.map(function(e){return t.Pipeline.warnIfFunctionNotRegistered(e),e.label})},t.Index=function(){this._fields=[],this._ref="id",this.pipeline=new t.Pipeline,this.documentStore=new t.DocumentStore,this.index={},this.eventEmitter=new t.EventEmitter,this._idfCache={},this.on("add","remove","update",function(){this._idfCache={}}.bind(this))},t.Index.prototype.on=function(){var e=Array.prototype.slice.call(arguments);return this.eventEmitter.addListener.apply(this.eventEmitter,e)},t.Index.prototype.off=function(e,t){return this.eventEmitter.removeListener(e,t)},t.Index.load=function(e){e.version!==t.version&&t.utils.warn("version mismatch: current "+t.version+" importing "+e.version);var n=new this;n._fields=e.fields,n._ref=e.ref,n.documentStore=t.DocumentStore.load(e.documentStore),n.pipeline=t.Pipeline.load(e.pipeline),n.index={};for(var i in e.index)n.index[i]=t.InvertedIndex.load(e.index[i]);return n},t.Index.prototype.addField=function(e){return this._fields.push(e),this.index[e]=new t.InvertedIndex,this},t.Index.prototype.setRef=function(e){return this._ref=e,this},t.Index.prototype.saveDocument=function(e){return this.documentStore=new t.DocumentStore(e),this},t.Index.prototype.addDoc=function(e,n){if(e){var n=void 0===n?!0:n,i=e[this._ref];this.documentStore.addDoc(i,e),this._fields.forEach(function(n){var o=this.pipeline.run(t.tokenizer(e[n]));this.documentStore.addFieldLength(i,n,o.length);var r={};o.forEach(function(e){e in r?r[e]+=1:r[e]=1},this);for(var s in r){var u=r[s];u=Math.sqrt(u),this.index[n].addToken(s,{ref:i,tf:u})}},this),n&&this.eventEmitter.emit("add",e,this)}},t.Index.prototype.removeDocByRef=function(e){if(e&&this.documentStore.isDocStored()!==!1&&this.documentStore.hasDoc(e)){var t=this.documentStore.getDoc(e);this.removeDoc(t,!1)}},t.Index.prototype.removeDoc=function(e,n){if(e){var n=void 0===n?!0:n,i=e[this._ref];this.documentStore.hasDoc(i)&&(this.documentStore.removeDoc(i),this._fields.forEach(function(n){var o=this.pipeline.run(t.tokenizer(e[n]));o.forEach(function(e){this.index[n].removeToken(e,i)},this)},this),n&&this.eventEmitter.emit("remove",e,this))}},t.Index.prototype.updateDoc=function(e,t){var t=void 0===t?!0:t;this.removeDocByRef(e[this._ref],!1),this.addDoc(e,!1),t&&this.eventEmitter.emit("update",e,this)},t.Index.prototype.idf=function(e,t){var n="@"+t+"/"+e;if(Object.prototype.hasOwnProperty.call(this._idfCache,n))return this._idfCache[n];var i=this.index[t].getDocFreq(e),o=1+Math.log(this.documentStore.length/(i+1));return this._idfCache[n]=o,o},t.Index.prototype.getFields=function(){return this._fields.slice()},t.Index.prototype.search=function(e,n){if(!e)return[];e="string"==typeof e?{any:e}:JSON.parse(JSON.stringify(e));var i=null;null!=n&&(i=JSON.stringify(n));for(var o=new t.Configuration(i,this.getFields()).get(),r={},s=Object.keys(e),u=0;u<s.length;u++){var a=s[u];r[a]=this.pipeline.run(t.tokenizer(e[a]))}var l={};for(var c in o){var d=r[c]||r.any;if(d){var f=this.fieldSearch(d,c,o),h=o[c].boost;for(var p in f)f[p]=f[p]*h;for(var p in f)p in l?l[p]+=f[p]:l[p]=f[p]}}var v,g=[];for(var p in l)v={ref:p,score:l[p]},this.documentStore.hasDoc(p)&&(v.doc=this.documentStore.getDoc(p)),g.push(v);return g.sort(function(e,t){return t.score-e.score}),g},t.Index.prototype.fieldSearch=function(e,t,n){var i=n[t].bool,o=n[t].expand,r=n[t].boost,s=null,u={};return 0!==r?(e.forEach(function(e){var n=[e];1==o&&(n=this.index[t].expandToken(e));var r={};n.forEach(function(n){var o=this.index[t].getDocs(n),a=this.idf(n,t);if(s&&"AND"==i){var l={};for(var c in s)c in o&&(l[c]=o[c]);o=l}n==e&&this.fieldSearchStats(u,n,o);for(var c in o){var d=this.index[t].getTermFrequency(n,c),f=this.documentStore.getFieldLength(c,t),h=1;0!=f&&(h=1/Math.sqrt(f));var p=1;n!=e&&(p=.15*(1-(n.length-e.length)/n.length));var v=d*a*h*p;c in r?r[c]+=v:r[c]=v}},this),s=this.mergeScores(s,r,i)},this),s=this.coordNorm(s,u,e.length)):void 0},t.Index.prototype.mergeScores=function(e,t,n){if(!e)return t;if("AND"==n){var i={};for(var o in t)o in e&&(i[o]=e[o]+t[o]);return i}for(var o in t)o in e?e[o]+=t[o]:e[o]=t[o];return e},t.Index.prototype.fieldSearchStats=function(e,t,n){for(var i in n)i in e?e[i].push(t):e[i]=[t]},t.Index.prototype.coordNorm=function(e,t,n){for(var i in e)if(i in t){var o=t[i].length;e[i]=e[i]*o/n}return e},t.Index.prototype.toJSON=function(){var e={};return this._fields.forEach(function(t){e[t]=this.index[t].toJSON()},this),{version:t.version,fields:this._fields,ref:this._ref,documentStore:this.documentStore.toJSON(),index:e,pipeline:this.pipeline.toJSON()}},t.Index.prototype.use=function(e){var t=Array.prototype.slice.call(arguments,1);t.unshift(this),e.apply(this,t)},t.DocumentStore=function(e){this._save=null===e||void 0===e?!0:e,this.docs={},this.docInfo={},this.length=0},t.DocumentStore.load=function(e){var t=new this;return t.length=e.length,t.docs=e.docs,t.docInfo=e.docInfo,t._save=e.save,t},t.DocumentStore.prototype.isDocStored=function(){return this._save},t.DocumentStore.prototype.addDoc=function(t,n){this.hasDoc(t)||this.length++,this.docs[t]=this._save===!0?e(n):null},t.DocumentStore.prototype.getDoc=function(e){return this.hasDoc(e)===!1?null:this.docs[e]},t.DocumentStore.prototype.hasDoc=function(e){return e in this.docs},t.DocumentStore.prototype.removeDoc=function(e){this.hasDoc(e)&&(delete this.docs[e],delete this.docInfo[e],this.length--)},t.DocumentStore.prototype.addFieldLength=function(e,t,n){null!==e&&void 0!==e&&0!=this.hasDoc(e)&&(this.docInfo[e]||(this.docInfo[e]={}),this.docInfo[e][t]=n)},t.DocumentStore.prototype.updateFieldLength=function(e,t,n){null!==e&&void 0!==e&&0!=this.hasDoc(e)&&this.addFieldLength(e,t,n)},t.DocumentStore.prototype.getFieldLength=function(e,t){return null===e||void 0===e?0:e in this.docs&&t in this.docInfo[e]?this.docInfo[e][t]:0},t.DocumentStore.prototype.toJSON=function(){return{docs:this.docs,docInfo:this.docInfo,length:this.length,save:this._save}},t.stemmer=function(){var e={ational:"ate",tional:"tion",enci:"ence",anci:"ance",izer:"ize",bli:"ble",alli:"al",entli:"ent",eli:"e",ousli:"ous",ization:"ize",ation:"ate",ator:"ate",alism:"al",iveness:"ive",fulness:"ful",ousness:"ous",aliti:"al",iviti:"ive",biliti:"ble",logi:"log"},t={icate:"ic",ative:"",alize:"al",iciti:"ic",ical:"ic",ful:"",ness:""},n="[^aeiou]",i="[aeiouy]",o=n+"[^aeiouy]*",r=i+"[aeiou]*",s="^("+o+")?"+r+o,u="^("+o+")?"+r+o+"("+r+")?$",a="^("+o+")?"+r+o+r+o,l="^("+o+")?"+i,c=new RegExp(s),d=new RegExp(a),f=new RegExp(u),h=new RegExp(l),p=/^(.+?)(ss|i)es$/,v=/^(.+?)([^s])s$/,g=/^(.+?)eed$/,m=/^(.+?)(ed|ing)$/,y=/.$/,S=/(at|bl|iz)$/,x=new RegExp("([^aeiouylsz])\\1$"),w=new RegExp("^"+o+i+"[^aeiouwxy]$"),I=/^(.+?[^aeiou])y$/,b=/^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/,E=/^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/,D=/^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/,F=/^(.+?)(s|t)(ion)$/,_=/^(.+?)e$/,P=/ll$/,k=new RegExp("^"+o+i+"[^aeiouwxy]$"),z=function(n){var i,o,r,s,u,a,l;if(n.length<3)return n;if(r=n.substr(0,1),"y"==r&&(n=r.toUpperCase()+n.substr(1)),s=p,u=v,s.test(n)?n=n.replace(s,"$1$2"):u.test(n)&&(n=n.replace(u,"$1$2")),s=g,u=m,s.test(n)){var z=s.exec(n);s=c,s.test(z[1])&&(s=y,n=n.replace(s,""))}else if(u.test(n)){var z=u.exec(n);i=z[1],u=h,u.test(i)&&(n=i,u=S,a=x,l=w,u.test(n)?n+="e":a.test(n)?(s=y,n=n.replace(s,"")):l.test(n)&&(n+="e"))}if(s=I,s.test(n)){var z=s.exec(n);i=z[1],n=i+"i"}if(s=b,s.test(n)){var z=s.exec(n);i=z[1],o=z[2],s=c,s.test(i)&&(n=i+e[o])}if(s=E,s.test(n)){var z=s.exec(n);i=z[1],o=z[2],s=c,s.test(i)&&(n=i+t[o])}if(s=D,u=F,s.test(n)){var z=s.exec(n);i=z[1],s=d,s.test(i)&&(n=i)}else if(u.test(n)){var z=u.exec(n);i=z[1]+z[2],u=d,u.test(i)&&(n=i)}if(s=_,s.test(n)){var z=s.exec(n);i=z[1],s=d,u=f,a=k,(s.test(i)||u.test(i)&&!a.test(i))&&(n=i)}return s=P,u=d,s.test(n)&&u.test(n)&&(s=y,n=n.replace(s,"")),"y"==r&&(n=r.toLowerCase()+n.substr(1)),n};return z}(),t.Pipeline.registerFunction(t.stemmer,"stemmer"),t.stopWordFilter=function(e){return e&&t.stopWordFilter.stopWords[e]!==!0?e:void 0},t.clearStopWords=function(){t.stopWordFilter.stopWords={}},t.addStopWords=function(e){null!=e&&Array.isArray(e)!==!1&&e.forEach(function(e){t.stopWordFilter.stopWords[e]=!0},this)},t.resetStopWords=function(){t.stopWordFilter.stopWords=t.defaultStopWords},t.defaultStopWords={"":!0,a:!0,able:!0,about:!0,across:!0,after:!0,all:!0,almost:!0,also:!0,am:!0,among:!0,an:!0,and:!0,any:!0,are:!0,as:!0,at:!0,be:!0,because:!0,been:!0,but:!0,by:!0,can:!0,cannot:!0,could:!0,dear:!0,did:!0,"do":!0,does:!0,either:!0,"else":!0,ever:!0,every:!0,"for":!0,from:!0,get:!0,got:!0,had:!0,has:!0,have:!0,he:!0,her:!0,hers:!0,him:!0,his:!0,how:!0,however:!0,i:!0,"if":!0,"in":!0,into:!0,is:!0,it:!0,its:!0,just:!0,least:!0,let:!0,like:!0,likely:!0,may:!0,me:!0,might:!0,most:!0,must:!0,my:!0,neither:!0,no:!0,nor:!0,not:!0,of:!0,off:!0,often:!0,on:!0,only:!0,or:!0,other:!0,our:!0,own:!0,rather:!0,said:!0,say:!0,says:!0,she:!0,should:!0,since:!0,so:!0,some:!0,than:!0,that:!0,the:!0,their:!0,them:!0,then:!0,there:!0,these:!0,they:!0,"this":!0,tis:!0,to:!0,too:!0,twas:!0,us:!0,wants:!0,was:!0,we:!0,were:!0,what:!0,when:!0,where:!0,which:!0,"while":!0,who:!0,whom:!0,why:!0,will:!0,"with":!0,would:!0,yet:!0,you:!0,your:!0},t.stopWordFilter.stopWords=t.defaultStopWords,t.Pipeline.registerFunction(t.stopWordFilter,"stopWordFilter"),t.trimmer=function(e){if(null===e||void 0===e)throw new Error("token should not be undefined");return e.replace(/^\W+/,"").replace(/\W+$/,"")},t.Pipeline.registerFunction(t.trimmer,"trimmer"),t.InvertedIndex=function(){this.root={docs:{},df:0}},t.InvertedIndex.load=function(e){var t=new this;return t.root=e.root,t},t.InvertedIndex.prototype.addToken=function(e,t,n){for(var n=n||this.root,i=0;i<=e.length-1;){var o=e[i];o in n||(n[o]={docs:{},df:0}),i+=1,n=n[o]}var r=t.ref;n.docs[r]?n.docs[r]={tf:t.tf}:(n.docs[r]={tf:t.tf},n.df+=1)},t.InvertedIndex.prototype.hasToken=function(e){if(!e)return!1;for(var t=this.root,n=0;n<e.length;n++){if(!t[e[n]])return!1;t=t[e[n]]}return!0},t.InvertedIndex.prototype.getNode=function(e){if(!e)return null;for(var t=this.root,n=0;n<e.length;n++){if(!t[e[n]])return null;t=t[e[n]]}return t},t.InvertedIndex.prototype.getDocs=function(e){var t=this.getNode(e);return null==t?{}:t.docs},t.InvertedIndex.prototype.getTermFrequency=function(e,t){var n=this.getNode(e);return null==n?0:t in n.docs?n.docs[t].tf:0},t.InvertedIndex.prototype.getDocFreq=function(e){var t=this.getNode(e);return null==t?0:t.df},t.InvertedIndex.prototype.removeToken=function(e,t){if(e){var n=this.getNode(e);null!=n&&t in n.docs&&(delete n.docs[t],n.df-=1)}},t.InvertedIndex.prototype.expandToken=function(e,t,n){if(null==e||""==e)return[];var t=t||[];if(void 0==n&&(n=this.getNode(e),null==n))return t;n.df>0&&t.push(e);for(var i in n)"docs"!==i&&"df"!==i&&this.expandToken(e+i,t,n[i]);return t},t.InvertedIndex.prototype.toJSON=function(){return{root:this.root}},t.Configuration=function(e,n){var e=e||"";if(void 0==n||null==n)throw new Error("fields should not be null");this.config={};var i;try{i=JSON.parse(e),this.buildUserConfig(i,n)}catch(o){t.utils.warn("user configuration parse failed, will use default configuration"),this.buildDefaultConfig(n)}},t.Configuration.prototype.buildDefaultConfig=function(e){this.reset(),e.forEach(function(e){this.config[e]={boost:1,bool:"OR",expand:!1}},this)},t.Configuration.prototype.buildUserConfig=function(e,n){var i="OR",o=!1;if(this.reset(),"bool"in e&&(i=e.bool||i),"expand"in e&&(o=e.expand||o),"fields"in e)for(var r in e.fields)if(n.indexOf(r)>-1){var s=e.fields[r],u=o;void 0!=s.expand&&(u=s.expand),this.config[r]={boost:s.boost||0===s.boost?s.boost:1,bool:s.bool||i,expand:u}}else t.utils.warn("field name in user configuration not found in index instance fields");else this.addAllFields2UserConfig(i,o,n)},t.Configuration.prototype.addAllFields2UserConfig=function(e,t,n){n.forEach(function(n){this.config[n]={boost:1,bool:e,expand:t}},this)},t.Configuration.prototype.get=function(){return this.config},t.Configuration.prototype.reset=function(){this.config={}},lunr.SortedSet=function(){this.length=0,this.elements=[]},lunr.SortedSet.load=function(e){var t=new this;return t.elements=e,t.length=e.length,t},lunr.SortedSet.prototype.add=function(){var e,t;for(e=0;e<arguments.length;e++)t=arguments[e],~this.indexOf(t)||this.elements.splice(this.locationFor(t),0,t);this.length=this.elements.length},lunr.SortedSet.prototype.toArray=function(){return this.elements.slice()},lunr.SortedSet.prototype.map=function(e,t){return this.elements.map(e,t)},lunr.SortedSet.prototype.forEach=function(e,t){return this.elements.forEach(e,t)},lunr.SortedSet.prototype.indexOf=function(e){for(var t=0,n=this.elements.length,i=n-t,o=t+Math.floor(i/2),r=this.elements[o];i>1;){if(r===e)return o;e>r&&(t=o),r>e&&(n=o),i=n-t,o=t+Math.floor(i/2),r=this.elements[o]}return r===e?o:-1},lunr.SortedSet.prototype.locationFor=function(e){for(var t=0,n=this.elements.length,i=n-t,o=t+Math.floor(i/2),r=this.elements[o];i>1;)e>r&&(t=o),r>e&&(n=o),i=n-t,o=t+Math.floor(i/2),r=this.elements[o];return r>e?o:e>r?o+1:void 0},lunr.SortedSet.prototype.intersect=function(e){for(var t=new lunr.SortedSet,n=0,i=0,o=this.length,r=e.length,s=this.elements,u=e.elements;;){if(n>o-1||i>r-1)break;s[n]!==u[i]?s[n]<u[i]?n++:s[n]>u[i]&&i++:(t.add(s[n]),n++,i++)}return t},lunr.SortedSet.prototype.clone=function(){var e=new lunr.SortedSet;return e.elements=this.toArray(),e.length=e.elements.length,e},lunr.SortedSet.prototype.union=function(e){var t,n,i;this.length>=e.length?(t=this,n=e):(t=e,n=this),i=t.clone();for(var o=0,r=n.toArray();o<r.length;o++)i.add(r[o]);return i},lunr.SortedSet.prototype.toJSON=function(){return this.toArray()},function(e,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():e.elasticlunr=t()}(this,function(){return t})}(); diff --git a/components/search/src/lib.rs b/components/search/src/lib.rs new file mode 100644 index 00000000..ea3bcb21 --- /dev/null +++ b/components/search/src/lib.rs @@ -0,0 +1,80 @@ +extern crate elasticlunr; +#[macro_use] +extern crate lazy_static; +extern crate ammonia; +#[macro_use] +extern crate errors; +extern crate content; + +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; + +use elasticlunr::{Index, Language}; + +use content::Section; +use errors::Result; + + +pub const ELASTICLUNR_JS: &'static str = include_str!("elasticlunr.min.js"); + +lazy_static! { + static ref AMMONIA: ammonia::Builder<'static> = { + let mut clean_content = HashSet::new(); + clean_content.insert("script"); + clean_content.insert("style"); + let mut builder = ammonia::Builder::new(); + builder + .tags(HashSet::new()) + .tag_attributes(HashMap::new()) + .generic_attributes(HashSet::new()) + .link_rel(None) + .allowed_classes(HashMap::new()) + .clean_content_tags(clean_content); + builder + }; +} + + +/// Returns the generated JSON index with all the documents of the site added using +/// the language given +/// Errors if the language given is not available in Elasticlunr +/// TODO: is making `in_search_index` apply to subsections of a `false` section useful? +pub fn build_index(sections: &HashMap<PathBuf, Section>, lang: &str) -> Result<String> { + let language = match Language::from_code(lang) { + Some(l) => l, + None => { bail!("Tried to build search index for language {} which is not supported", lang); } + }; + + let mut index = Index::with_language(language, &["title", "body"]); + + for section in sections.values() { + add_section_to_index(&mut index, section); + } + + Ok(index.to_json()) +} + +fn add_section_to_index(index: &mut Index, section: &Section) { + if !section.meta.in_search_index { + return; + } + + // Don't index redirecting sections + if section.meta.redirect_to.is_none() { + index.add_doc( + §ion.permalink, + &[§ion.meta.title.clone().unwrap_or(String::new()), &AMMONIA.clean(§ion.content).to_string()], + ); + } + + for page in §ion.pages { + if !page.meta.in_search_index || page.meta.draft { + continue; + } + + index.add_doc( + &page.permalink, + &[&page.meta.title.clone().unwrap_or(String::new()), &AMMONIA.clean(&page.content).to_string()], + ); + } +} diff --git a/components/site/Cargo.toml b/components/site/Cargo.toml index 99c1abbe..2299932f 100644 --- a/components/site/Cargo.toml +++ b/components/site/Cargo.toml @@ -1,12 +1,11 @@ [package] name = "site" version = "0.1.0" -authors = ["Vincent Prouillet <vincent@wearewizards.io>"] +authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"] [dependencies] tera = "0.11" glob = "0.2" -walkdir = "2" rayon = "1" serde = "1" serde_derive = "1" @@ -20,6 +19,7 @@ front_matter = { path = "../front_matter" } pagination = { path = "../pagination" } taxonomies = { path = "../taxonomies" } content = { path = "../content" } +search = { path = "../search" } [dev-dependencies] tempdir = "0.3" diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index 668bce70..210f8483 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -1,7 +1,6 @@ extern crate tera; extern crate rayon; extern crate glob; -extern crate walkdir; extern crate serde; #[macro_use] extern crate serde_derive; @@ -16,23 +15,23 @@ extern crate templates; extern crate pagination; extern crate taxonomies; extern crate content; +extern crate search; #[cfg(test)] extern crate tempdir; use std::collections::HashMap; -use std::fs::{remove_dir_all, copy, create_dir_all}; +use std::fs::{create_dir_all, remove_dir_all, copy}; use std::mem; use std::path::{Path, PathBuf}; use glob::glob; use tera::{Tera, Context}; -use walkdir::WalkDir; use sass_rs::{Options as SassOptions, OutputStyle, compile_file}; use errors::{Result, ResultExt}; use config::{Config, get_config}; -use utils::fs::{create_file, create_directory, ensure_directory_exists}; +use utils::fs::{create_file, copy_directory, create_directory, ensure_directory_exists}; use utils::templates::{render_template, rewrite_theme_paths}; use content::{Page, Section, populate_previous_and_next_pages, sort_pages}; use templates::{GUTENBERG_TERA, global_fns, render_redirect_template}; @@ -67,7 +66,7 @@ pub struct Site { pub sections: HashMap<PathBuf, Section>, pub tera: Tera, live_reload: bool, - output_path: PathBuf, + pub output_path: PathBuf, pub static_path: PathBuf, pub tags: Option<Taxonomy>, pub categories: Option<Taxonomy>, @@ -125,6 +124,11 @@ impl Site { Ok(site) } + /// The index section is ALWAYS at that path + pub fn index_section_path(&self) -> PathBuf { + self.base_path.join("content").join("_index.md") + } + /// What the function name says pub fn enable_live_reload(&mut self) { self.live_reload = true; @@ -198,7 +202,17 @@ impl Site { // Insert a default index section if necessary so we don't need to create // a _index.md to render the index page - let index_path = self.base_path.join("content").join("_index.md"); + let index_path = self.index_section_path(); + if let Some(ref index_section) = self.sections.get(&index_path) { + if self.config.build_search_index && !index_section.meta.in_search_index { + bail!( + "You have enabled search in the config but disabled it in the index section: \ + either turn off the search in the config or remote `in_search_index = true` from the \ + section front-matter." + ) + } + } + // Not in else because of borrow checker if !self.sections.contains_key(&index_path) { let mut index_section = Section::default(); index_section.permalink = self.config.make_permalink(""); @@ -308,7 +322,7 @@ impl Site { /// Defaults to `AnchorInsert::None` if no parent section found pub fn find_parent_section_insert_anchor(&self, parent_path: &PathBuf) -> InsertAnchor { match self.sections.get(&parent_path.join("_index.md")) { - Some(s) => s.meta.insert_anchor_links.unwrap(), + Some(s) => s.meta.insert_anchor_links, None => InsertAnchor::None } } @@ -350,7 +364,7 @@ impl Site { .map(|p| sections[p].clone()) .collect::<Vec<_>>(); section.subsections - .sort_by(|a, b| a.meta.weight.unwrap().cmp(&b.meta.weight.unwrap())); + .sort_by(|a, b| a.meta.weight.cmp(&b.meta.weight)); } } } @@ -365,7 +379,7 @@ impl Site { } } let pages = mem::replace(&mut section.pages, vec![]); - let (sorted_pages, cannot_be_sorted_pages) = sort_pages(pages, section.meta.sort_by()); + let (sorted_pages, cannot_be_sorted_pages) = sort_pages(pages, section.meta.sort_by); section.pages = populate_previous_and_next_pages(&sorted_pages); section.ignored_pages = cannot_be_sorted_pages; } @@ -409,45 +423,18 @@ impl Site { html } - /// Copy the file at the given path into the public folder - pub fn copy_static_file<P: AsRef<Path>>(&self, path: P, base_path: &PathBuf) -> Result<()> { - let relative_path = path.as_ref().strip_prefix(base_path).unwrap(); - let target_path = self.output_path.join(relative_path); - if let Some(parent_directory) = target_path.parent() { - create_dir_all(parent_directory)?; - } - copy(path.as_ref(), &target_path)?; - Ok(()) - } - - /// Copy the content of the given folder into the `public` folder - fn copy_static_directory(&self, path: &PathBuf) -> Result<()> { - for entry in WalkDir::new(path).into_iter().filter_map(|e| e.ok()) { - let relative_path = entry.path().strip_prefix(path).unwrap(); - let target_path = self.output_path.join(relative_path); - if entry.path().is_dir() { - if !target_path.exists() { - create_directory(&target_path)?; - } - } else { - let entry_fullpath = self.base_path.join(entry.path()); - self.copy_static_file(entry_fullpath, path)?; - } - } - Ok(()) - } - /// Copy the main `static` folder and the theme `static` folder if a theme is used pub fn copy_static_directories(&self) -> Result<()> { // The user files will overwrite the theme files if let Some(ref theme) = self.config.theme { - self.copy_static_directory( - &self.base_path.join("themes").join(theme).join("static") + copy_directory( + &self.base_path.join("themes").join(theme).join("static"), + &self.output_path )?; } // We're fine with missing static folders if self.static_path.exists() { - self.copy_static_directory(&self.static_path)?; + copy_directory(&self.static_path, &self.output_path)?; } Ok(()) @@ -522,7 +509,32 @@ impl Site { self.compile_sass(&self.base_path)?; } - self.copy_static_directories() + self.copy_static_directories()?; + + if self.config.build_search_index { + self.build_search_index()?; + } + + Ok(()) + } + + pub fn build_search_index(&self) -> Result<()> { + // index first + create_file( + &self.output_path.join(&format!("search_index.{}.js", self.config.default_language)), + &format!( + "window.searchIndex = {};", + search::build_index(&self.sections, &self.config.default_language)? + ), + )?; + + // then elasticlunr.min.js + create_file( + &self.output_path.join("elasticlunr.min.js"), + search::ELASTICLUNR_JS, + )?; + + Ok(()) } pub fn compile_sass(&self, base_path: &Path) -> Result<()> { @@ -585,18 +597,16 @@ impl Site { pub fn render_aliases(&self) -> Result<()> { for page in self.pages.values() { - if let Some(ref aliases) = page.meta.aliases { - for alias in aliases { - let mut output_path = self.output_path.to_path_buf(); - for component in alias.split('/') { - output_path.push(&component); + for alias in &page.meta.aliases { + let mut output_path = self.output_path.to_path_buf(); + for component in alias.split('/') { + output_path.push(&component); - if !output_path.exists() { - create_directory(&output_path)?; - } + if !output_path.exists() { + create_directory(&output_path)?; } - create_file(&output_path.join("index.html"), &render_redirect_template(&page.permalink, &self.tera)?)?; } + create_file(&output_path.join("index.html"), &render_redirect_template(&page.permalink, &self.tera)?)?; } } Ok(()) @@ -773,7 +783,7 @@ impl Site { .reduce(|| Ok(()), Result::and)?; } - if !section.meta.should_render() { + if !section.meta.render { return Ok(()); } @@ -827,13 +837,8 @@ impl Site { pub fn render_paginated(&self, output_path: &Path, section: &Section) -> Result<()> { ensure_directory_exists(&self.output_path)?; - let paginate_path = match section.meta.paginate_path { - Some(ref s) => s.clone(), - None => unreachable!() - }; - let paginator = Paginator::new(§ion.pages, section); - let folder_path = output_path.join(&paginate_path); + let folder_path = output_path.join(§ion.meta.paginate_path); create_directory(&folder_path)?; paginator diff --git a/components/site/tests/site.rs b/components/site/tests/site.rs index 598a4f4c..bb24c3c3 100644 --- a/components/site/tests/site.rs +++ b/components/site/tests/site.rs @@ -445,3 +445,21 @@ fn can_build_rss_feed() { // Next is posts/python.md assert!(file_contains!(public, "rss.xml", "Python in posts")); } + + +#[test] +fn can_build_search_index() { + let mut path = env::current_dir().unwrap().parent().unwrap().parent().unwrap().to_path_buf(); + path.push("test_site"); + let mut site = Site::new(&path, "config.toml").unwrap(); + site.load().unwrap(); + site.config.build_search_index = true; + let tmp_dir = TempDir::new("example").expect("create temp dir"); + let public = &tmp_dir.path().join("public"); + site.set_output_path(&public); + site.build().unwrap(); + + assert!(Path::new(&public).exists()); + assert!(file_exists!(public, "elasticlunr.min.js")); + assert!(file_exists!(public, "search_index.en.js")); +} diff --git a/components/taxonomies/Cargo.toml b/components/taxonomies/Cargo.toml index 1329965e..2298737d 100644 --- a/components/taxonomies/Cargo.toml +++ b/components/taxonomies/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "taxonomies" version = "0.1.0" -authors = ["Vincent Prouillet <vincent@wearewizards.io>"] +authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"] [dependencies] tera = "0.11" diff --git a/components/templates/Cargo.toml b/components/templates/Cargo.toml index f53c4183..0d6ebe8a 100644 --- a/components/templates/Cargo.toml +++ b/components/templates/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "templates" version = "0.1.0" -authors = ["Vincent Prouillet <vincent@wearewizards.io>"] +authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"] [dependencies] tera = "0.11" diff --git a/components/utils/Cargo.toml b/components/utils/Cargo.toml index 27ca4204..ee330ee1 100644 --- a/components/utils/Cargo.toml +++ b/components/utils/Cargo.toml @@ -1,11 +1,12 @@ [package] name = "utils" version = "0.1.0" -authors = ["Vincent Prouillet <vincent@wearewizards.io>"] +authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"] [dependencies] errors = { path = "../errors" } tera = "0.11" +walkdir = "2" [dev-dependencies] diff --git a/components/utils/src/fs.rs b/components/utils/src/fs.rs index 7b5381cc..06cd80ce 100644 --- a/components/utils/src/fs.rs +++ b/components/utils/src/fs.rs @@ -1,9 +1,12 @@ use std::io::prelude::*; -use std::fs::{File, create_dir_all, read_dir}; +use std::fs::{File, create_dir_all, read_dir, copy}; use std::path::{Path, PathBuf}; +use walkdir::WalkDir; + use errors::{Result, ResultExt}; + /// Create a file with the content given pub fn create_file(path: &Path, content: &str) -> Result<()> { let mut file = File::create(&path)?; @@ -60,6 +63,36 @@ pub fn find_related_assets(path: &Path) -> Vec<PathBuf> { assets } +/// Copy a file but takes into account where to start the copy as +/// there might be folders we need to create on the way +pub fn copy_file(src: &Path, dest: &PathBuf, base_path: &PathBuf) -> Result<()> { + let relative_path = src.strip_prefix(base_path).unwrap(); + let target_path = dest.join(relative_path); + + if let Some(parent_directory) = target_path.parent() { + create_dir_all(parent_directory)?; + } + + copy(src, target_path)?; + Ok(()) +} + +pub fn copy_directory(src: &PathBuf, dest: &PathBuf) -> Result<()> { + for entry in WalkDir::new(src).into_iter().filter_map(|e| e.ok()) { + let relative_path = entry.path().strip_prefix(src).unwrap(); + let target_path = dest.join(relative_path); + + if entry.path().is_dir() { + if !target_path.exists() { + create_directory(&target_path)?; + } + } else { + copy_file(entry.path(), dest, src)?; + } + } + Ok(()) +} + #[cfg(test)] mod tests { use std::fs::File; diff --git a/components/utils/src/lib.rs b/components/utils/src/lib.rs index 6b447f24..b529aa45 100644 --- a/components/utils/src/lib.rs +++ b/components/utils/src/lib.rs @@ -4,6 +4,7 @@ extern crate errors; #[cfg(test)] extern crate tempdir; extern crate tera; +extern crate walkdir; pub mod fs; pub mod site; diff --git a/docs/config.toml b/docs/config.toml index ea9e9bac..ce69764a 100644 --- a/docs/config.toml +++ b/docs/config.toml @@ -6,6 +6,7 @@ compile_sass = true highlight_code = true insert_anchor_links = true highlight_theme = "kronuz" +build_search_index = true [extra] author = "Vincent Prouillet" diff --git a/docs/content/documentation/content/page.md b/docs/content/documentation/content/page.md index 14131a66..1a8b7707 100644 --- a/docs/content/documentation/content/page.md +++ b/docs/content/documentation/content/page.md @@ -54,6 +54,11 @@ weight = 0 # current one. This takes an array of path, not URLs. aliases = [] +# Whether the page should be in the search index. This is only used if +# `build_search_index` is set to true in the config and the parent section +# hasn't set `in_search_index` to false in its front-matter +in_search_index = true + # Template to use to render this page template = "page.html" diff --git a/docs/content/documentation/content/search.md b/docs/content/documentation/content/search.md new file mode 100644 index 00000000..c3e79f55 --- /dev/null +++ b/docs/content/documentation/content/search.md @@ -0,0 +1,22 @@ ++++ +title = "Search" +weight = 100 ++++ + +Gutenberg can build a search index from the sections and pages content to +be used by a JavaScript library: [elasticlunr](http://elasticlunr.com/). + +To enable it, you only need to set `build_search_index = true` in your `config.toml` and Gutenberg will +generate an index for the `default_language` set for all pages not excluded from the search index. + +It is very important to set the `default_language` in your `config.toml` if you are writing a site not in +English: the index building pipelines are very different depending on the language. + +After `gutenberg build` or `gutenberg serve`, you should see two files in your static directory: + +- `search_index.${default_language}.js`: so `search_index.en.js` for a default setup +- `elasticlunr.min.js` + +As each site will be different, Gutenberg makes no assumptions about how your search and doesn't provide +the JavaScript/CSS code to do an actual search and display results. You can however look at how this very site +is implementing it to have an idea: [search.js](https://github.com/Keats/gutenberg/tree/master/docs/static/search.js). diff --git a/docs/content/documentation/content/section.md b/docs/content/documentation/content/section.md index 00739877..bb5cd28c 100644 --- a/docs/content/documentation/content/section.md +++ b/docs/content/documentation/content/section.md @@ -52,6 +52,10 @@ paginate_path = "page" # Options are "left", "right" and "none" insert_anchor_links = "none" +# Whether the section pages should be in the search index. This is only used if +# `build_search_index` is set to true in the config +in_search_index = true + # Whether to render that section homepage or not. # Useful when the section is only there to organize things but is not meant # to be used directly diff --git a/docs/content/documentation/getting-started/configuration.md b/docs/content/documentation/getting-started/configuration.md index 11ed9040..58df27ad 100644 --- a/docs/content/documentation/getting-started/configuration.md +++ b/docs/content/documentation/getting-started/configuration.md @@ -51,6 +51,10 @@ generate_categories_pages = false # Whether to compile the Sass files found in the `sass` directory compile_sass = false +# Whether to build a search index out of the pages and section +# content for the `default_language` +build_search_index = false + # A list of glob patterns specifying asset files to ignore when # processing the content directory. # Defaults to none, which means all asset files are copied over to the public folder. diff --git a/docs/sass/_search.scss b/docs/sass/_search.scss new file mode 100644 index 00000000..f1ffb70e --- /dev/null +++ b/docs/sass/_search.scss @@ -0,0 +1,47 @@ +.search-container { + display: inline-block; + position: relative; + width: 300px; + + input { + width: 100%; + padding: 0.5rem; + } +} + +.search-results { + display: none; + position: absolute; + background: white; + color: black; + padding: 1rem; + box-shadow: 2px 2px 2px 0 rgba(0, 0, 0, 0.5); + max-height: 500px; + overflow: auto; + width: 150%; + right: 0; + + &__items { + list-style: none; + } + + li { + margin-top: 1rem; + border-bottom: 1px solid #ccc; + font-size: 0.9rem; + + &:first-of-type { + margin-top: 0; + } + } + + &__item { + margin-bottom: 1rem; + + a { + font-size: 1.2rem; + display: inline-block; + margin-bottom: 0.5rem; + } + } +} diff --git a/docs/sass/site.scss b/docs/sass/site.scss index fadaa37b..a9e5129c 100644 --- a/docs/sass/site.scss +++ b/docs/sass/site.scss @@ -16,3 +16,4 @@ $link-color: #007CBC; @import "index"; @import "docs"; @import "themes"; +@import "search"; diff --git a/docs/static/search.js b/docs/static/search.js new file mode 100644 index 00000000..c24a5962 --- /dev/null +++ b/docs/static/search.js @@ -0,0 +1,180 @@ +function debounce(func, wait) { + var timeout; + + return function () { + var context = this; + var args = arguments; + clearTimeout(timeout); + + timeout = setTimeout(function () { + timeout = null; + func.apply(context, args); + }, wait); + }; +} + +// Taken from mdbook +// The strategy is as follows: +// First, assign a value to each word in the document: +// Words that correspond to search terms (stemmer aware): 40 +// Normal words: 2 +// First word in a sentence: 8 +// Then use a sliding window with a constant number of words and count the +// sum of the values of the words within the window. Then use the window that got the +// maximum sum. If there are multiple maximas, then get the last one. +// Enclose the terms in <b>. +function makeTeaser(body, terms) { + var TERM_WEIGHT = 40; + var NORMAL_WORD_WEIGHT = 2; + var FIRST_WORD_WEIGHT = 8; + var TEASER_MAX_WORDS = 30; + + var stemmedTerms = terms.map(function (w) { + return elasticlunr.stemmer(w.toLowerCase()); + }); + var termFound = false; + var index = 0; + var weighted = []; // contains elements of ["word", weight, index_in_document] + + // split in sentences, then words + var sentences = body.toLowerCase().split(". "); + + for (var i in sentences) { + var words = sentences[i].split(" "); + var value = FIRST_WORD_WEIGHT; + + for (var j in words) { + var word = words[j]; + + if (word.length > 0) { + for (var k in stemmedTerms) { + if (elasticlunr.stemmer(word).startsWith(stemmedTerms[k])) { + value = TERM_WEIGHT; + termFound = true; + } + } + weighted.push([word, value, index]); + value = NORMAL_WORD_WEIGHT; + } + + index += word.length; + index += 1; // ' ' or '.' if last word in sentence + } + + index += 1; // because we split at a two-char boundary '. ' + } + + if (weighted.length === 0) { + return body; + } + + var windowWeights = []; + var windowSize = Math.min(weighted.length, TEASER_MAX_WORDS); + // We add a window with all the weights first + var curSum = 0; + for (var i = 0; i < windowSize; i++) { + curSum += weighted[i][1]; + } + windowWeights.push(curSum); + + for (var i = 0; i < weighted.length - windowSize; i++) { + curSum -= weighted[i][1]; + curSum += weighted[i + windowSize][1]; + windowWeights.push(curSum); + } + + // If we didn't find the term, just pick the first window + var maxSumIndex = 0; + if (termFound) { + var maxFound = 0; + // backwards + for (var i = windowWeights.length - 1; i >= 0; i--) { + if (windowWeights[i] > maxFound) { + maxFound = windowWeights[i]; + maxSumIndex = i; + } + } + } + + var teaser = []; + var startIndex = weighted[maxSumIndex][2]; + for (var i = maxSumIndex; i < maxSumIndex + windowSize; i++) { + var word = weighted[i]; + if (startIndex < word[2]) { + // missing text from index to start of `word` + teaser.push(body.substring(startIndex, word[2])); + startIndex = word[2]; + } + + // add <em/> around search terms + if (word[1] === TERM_WEIGHT) { + teaser.push("<b>"); + } + startIndex = word[2] + word[0].length; + teaser.push(body.substring(word[2], startIndex)); + + if (word[1] === TERM_WEIGHT) { + teaser.push("</b>"); + } + } + teaser.push("…"); + return teaser.join(""); +} + +function formatSearchResultItem(item, terms) { + return '<div class="search-results__item">' + + `<a href="${item.ref}">${item.doc.title}</a>` + + `<div>${makeTeaser(item.doc.body, terms)}</div>` + + '</div>'; +} + +function initSearch() { + var $searchInput = document.getElementById("search"); + var $searchResults = document.querySelector(".search-results"); + var $searchResultsItems = document.querySelector(".search-results__items"); + var MAX_ITEMS = 10; + + var options = { + bool: "AND", + fields: { + title: {boost: 2}, + body: {boost: 1}, + } + }; + var currentTerm = ""; + var index = elasticlunr.Index.load(window.searchIndex); + + $searchInput.addEventListener("keyup", debounce(function() { + var term = $searchInput.value.trim(); + if (term === currentTerm || !index) { + return; + } + $searchResults.style.display = term === "" ? "none" : "block"; + $searchResultsItems.innerHTML = ""; + if (term === "") { + return; + } + + var results = index.search(term, options); + if (results.length === 0) { + $searchResults.style.display = "none"; + return; + } + + currentTerm = term; + for (var i = 0; i < Math.min(results.length, MAX_ITEMS); i++) { + var item = document.createElement("li"); + item.innerHTML = formatSearchResultItem(results[i], term.split(" ")); + $searchResultsItems.appendChild(item); + } + }, 150)); +} + + +if (document.readyState === "complete" || + (document.readyState !== "loading" && !document.documentElement.doScroll) +) { + initSearch(); +} else { + document.addEventListener("DOMContentLoaded", initSearch); +} diff --git a/docs/templates/index.html b/docs/templates/index.html index 5b3bac13..87d795de 100644 --- a/docs/templates/index.html +++ b/docs/templates/index.html @@ -18,6 +18,14 @@ <a class="white" href="{{ get_url(path="./documentation/_index.md") }}" class="nav-link">Docs</a> <a class="white" href="{{ get_url(path="./themes/_index.md") }}" class="nav-link">Themes</a> <a class="white" href="https://github.com/Keats/gutenberg" class="nav-link">GitHub</a> + + <div class="search-container"> + <input id="search" type="search" placeholder="🔎 Search the docs"> + + <div class="search-results"> + <div class="search-results__items"></div> + </div> + </div> </nav> </header> @@ -93,5 +101,9 @@ <footer> ©2017-2018 — <a class="white" href="https://vincent.is">Vincent Prouillet</a> and <a class="white" href="https://github.com/Keats/gutenberg/graphs/contributors">contributors</a> </footer> + + <script type="text/javascript" src="{{ get_url(path="elasticlunr.min.js", trailing_slash=false) }}"></script> + <script type="text/javascript" src="{{ get_url(path="search_index.en.js", trailing_slash=false) }}"></script> + <script type="text/javascript" src="{{ get_url(path="search.js", trailing_slash=false) }}"></script> </body> </html> diff --git a/src/cmd/init.rs b/src/cmd/init.rs index 8aae24a0..d7b57601 100644 --- a/src/cmd/init.rs +++ b/src/cmd/init.rs @@ -58,7 +58,6 @@ pub fn create_new_project(name: &str) -> Result<()> { if compile_sass { create_dir(path.join("sass"))?; } - // TODO: if search == true, copy a lunr js file embedded in gutenberg println!(); console::success(&format!("Done! Your site was created in {:?}", canonicalize(path).unwrap())); diff --git a/src/cmd/serve.rs b/src/cmd/serve.rs index edf6806f..67486d69 100644 --- a/src/cmd/serve.rs +++ b/src/cmd/serve.rs @@ -38,6 +38,7 @@ use ctrlc; use site::Site; use errors::{Result, ResultExt}; +use utils::fs::copy_file; use console; use rebuild; @@ -207,7 +208,7 @@ pub fn serve(interface: &str, port: &str, output_dir: &str, base_url: &str, conf (ChangeKind::StaticFiles, p) => { if path.is_file() { console::info(&format!("-> Static file changes detected {}", path.display())); - rebuild_done_handling(&broadcaster, site.copy_static_file(&path, &site.static_path), &p); + rebuild_done_handling(&broadcaster, copy_file(&path, &site.output_path, &site.static_path), &p); } }, (ChangeKind::Sass, p) => {