User-agent: * # All robots will spider the domain Disallow: /antioxidant/ Disallow: /antioxidants/ Disallow: /index3.htm Disallow: /index4.htm Disallow: /index5.htm