# robots.txt for http://www.paganlibrary.com User-agent: * Disallow: /admin/ Disallow: /cgi-bin/ Disallow: /*.bak$ Disallow: /bot-trap/ User-agent: * Crawl-delay: 10 # Directories # Disallow: /themes/ # Files # Disallow: /CHANGELOG.txt # Paths (clean URLs) # Disallow: /admin/ # Paths (no clean URLs) # Disallow: /?q=admin/ # Disallow: *.xlsx$ # Begin block Bad-Robots from robots.txt