commit 6f373e6b2308d8202452970c53bb834d84b6d65d
from: Omar Polo <op@omarpolo.com>
date: Mon May 31 07:30:10 2021 UTC

tweaks and robots.txt

commit - c8fd567dead8bfd32e3a9d1b0c3728ad8671a62a
commit + 6f373e6b2308d8202452970c53bb834d84b6d65d
blob - f84d059d52152e04168dc83eab2ba6f070f226dd
blob + 5630f0bdc493c7e46e1b2839f4ef6fcb0049417b
--- src/blog/core.clj
+++ src/blog/core.clj
@@ -77,7 +77,7 @@
              "resources/out/http/img"]]
     (.. (File. d) mkdirs)))
 
-(defn gemini-post [{? :gemtext?}] ?)
+(def gemini-post :gemtext?)
 
 (defn post-pages [{:keys [proto]}]
   (let [tags (keys @per-tag)
@@ -143,15 +143,18 @@
         (rss/feed #(str "https://www.omarpolo.com/post/" % ".html") @posts)))
 
 (defn generate-robots-txt []
-  (spit "resources/out/gemini/robots.txt" "# block some bots from accessing the gempkg
+  (spit "resources/out/gemini/robots.txt" "# block some bots from accessing gempkg/man
 User-agent: archiver
 Disallow: /cgi/gempkg/
 
-User-agent: indexer
+User-agent: researcher
 Disallow: /cgi/gempkg/
 
+User-agent: archiver
+Disallow: /cgi/man/
+
 User-agent: researcher
-Disallow: /cgi/gempkg/
+Disallow: /cgi/man/
 "))
 
 (defn copy-dir