# www.robotstxt.org/ # Allow crawling of all content User-agent: * Crawl-delay: 10 Disallow: /cgi-bin/ Disallow: /blog/wp-login.php Disallow: /blog/wp-register.php Disallow: /blog/xmlrpc.php Disallow: /blog/wp-admin/ Disallow: /blog/wp-includes/ Disallow: /blog/wp-content/plugins/ Disallow: /blog/wp-content/cache/ Disallow: /blog/wp-content/themes/