add Allow: / to default robots.txt template (#1258)

Also update the documentation about it.
2020-12-10 14:57:43 +01:00 · 2020-12-10 14:57:43 +01:00 · 679af5b8c8
parent 4754cb5792
commit 679af5b8c8
2 changed files with 3 additions and 0 deletions
--- a/components/templates/src/builtins/robots.txt
+++ b/components/templates/src/builtins/robots.txt
@ -1,2 +1,3 @@
 User-agent: *
+Allow: /
 Sitemap: {{ get_url(path="sitemap.xml") }}
--- a/docs/content/documentation/templates/robots.md
+++ b/docs/content/documentation/templates/robots.md
@ -11,4 +11,6 @@ and the default is what most sites want:

 ```jinja2
 User-agent: *
+Allow: /
+Sitemap: {{ get_url(path="sitemap.xml") }}
 ```