summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOmar Polo <op@omarpolo.com>2021-02-13 12:11:30 +0100
committerOmar Polo <op@omarpolo.com>2021-02-13 12:11:30 +0100
commit3864659458dc9f31fd6edcb921e5bd662bec7118 (patch)
tree5f3c8eb674a8cf6a5a94cb8ada7bc213096cdf20
parentfec02a6bc4085efa69cbf9c0ae2e2126326f1555 (diff)
downloadblog-3864659458dc9f31fd6edcb921e5bd662bec7118.tar.gz
blog-3864659458dc9f31fd6edcb921e5bd662bec7118.tar.bz2
generate robots.txt and change upload dir
-rw-r--r--src/blog/core.clj15
1 files changed, 14 insertions, 1 deletions
diff --git a/src/blog/core.clj b/src/blog/core.clj
index fa06202..f84d059 100644
--- a/src/blog/core.clj
+++ b/src/blog/core.clj
@@ -142,6 +142,18 @@
(spit (str "resources/out/http/rss.xml")
(rss/feed #(str "https://www.omarpolo.com/post/" % ".html") @posts)))
+(defn generate-robots-txt []
+ (spit "resources/out/gemini/robots.txt" "# block some bots from accessing the gempkg
+User-agent: archiver
+Disallow: /cgi/gempkg/
+
+User-agent: indexer
+Disallow: /cgi/gempkg/
+
+User-agent: researcher
+Disallow: /cgi/gempkg/
+"))
+
(defn copy-dir
"Copy the content of resources/`dir` to resources/out/`proto`/`dir`,
assuming these two directories exists."
@@ -177,6 +189,7 @@
(copy-assets)
(copy-cgi)
(render-rss)
+ (generate-robots-txt)
(doseq [[proto ffn ext homefn postfn tagsfn tagfn pagefn]
[[:http identity ".html" http/home-page http/post-page http/tags-page http/tag-page http/custom-page]
[:gemini gemini-post ".gmi" gemini/home-page gemini/post-page gemini/tags-page gemini/tag-page gemini/custom-page]]]
@@ -217,7 +230,7 @@
"Copy the files to the server"
[]
(sh "rsync" "-r" "--delete" "resources/out/http/" "op:sites/www.omarpolo.com/")
- (sh "rsync" "-r" "--delete" "resources/out/gemini/" "op:gemini"))
+ (sh "rsync" "-r" "--delete" "resources/out/gemini/" "op:gemini/gemini.omarpolo.com"))
(defn stop-jetty []
(.stop @j)