# robots.txt for search engine robots # examples # User-agent: * # Disallow: /folder/folder/ # This is an infinite virtual URL space # Disallow: /temp/ # these will soon disappear # Disallow: /page.html #User-Agent: msnbot #Disallow: /*.PDF$ #Disallow: /*.jpeg$ #Disallow: /*.exe$ #User-agent: Googlebot #Disallow: /*.PDF$ # Yahoo Slurp #User-agent: Slurp #Disallow: /*exe$ #Disallow: /*.PDF$ #Disallow: /*.swf$ User-agent: * Disallow: /activesearchengine/demo/ Disallow: /activesearchengine/demosp/ Disallow: /cgi-bin/ Disallow: /checkcodes/ Disallow: /download/custom/ Disallow: /Cart/ Disallow: /affiliates/