# Generally unwanted robots: User-agent: DBLBot # http://www.dontbuylists.com/ ??? User-agent: gonzo* # http://webadress.de/webadress/faq.html#c3 User-agent: ia_archiver # http://www.alexa.com/site/help/webmasters User-agent: SurveyBot # http://www.domaintools.com/webmasters/surveybot.php Disallow: / # No image (and video) spiders: User-agent: DittoSpyder # http://www.ditto.com/robots.aspx User-agent: Googlebot-Image # http://www.google.com/support/webmasters/bin/answer.py?answer=40364 User-agent: MSNBot-Media # http://help.live.com/help.aspx?project=wl_webmasters#faq12 User-agent: psbot # http://www.picsearch.com/menu.cgi?item=Psbot User-agent: vscooter # AltaVista image spider User-agent: Yahoo-MMAudVid # http://help.yahoo.com/help/us/ysearch/video/video-05.html User-agent: Yahoo-MMCrawler # http://help.yahoo.com/help/us/ysearch/image/image-08.html Disallow: / # Allow helpful tools: User-agent: W3C-checklink # http://validator.w3.org/checklink User-agent: WDG_SiteValidator # http://www.htmlhelp.org/tools/validator/ Disallow: