# /robots.txt file # # Robots read from top to bottom and stop when they reach something that applies to them. # Disallow all bots and crawlers. # Uncomment the User-agent and Crawl-delay lines to enable them - Google does not adhere to Crawl-Delay. # Microsoft's crawler # User-agent: MSNBot # Crawl-Delay: 5 # ShopWiki's crawler # User-agent: ShopWiki # Crawl-Delay: 5 # Ask.com's crawler # User-agent: Teoma # Crawl-Delay: 5 #Google's crawler # User-Agent: Googlebot # Allow: / # Yahoo's crawler # User-agent: Slurp # Allow: / # Crawl-Delay: 2 # Disallow everyone else User-agent: * Disallow: /