User-Agent: Googlebot-Image
Disallow: /

User-agent: dotbot 
Disallow: /

User-agent: psbot
Disallow: /

User-agent: IRLbot
Disallow: /

User-agent: twiceler
Disallow: /

User-agent: Slurp
Crawl-delay: 3
Disallow: /*.gif$
Disallow: /*.png$
Disallow: /*.jpg$

User-agent: *
Disallow: /admin/
Disallow: /cache/
Disallow: /cgi-bin/
Disallow: /common/
Disallow: /controllers/
Disallow: /css/
Disallow: /data/
Disallow: /images/
Disaalow: /img/
Disallow: /lib/
Disallow: /picture_library/
Disallow: /plesk_stat/
Disallow: /tpl/
Disallow: /test/

Disallow: /*.gif$
Disallow: /*.png$
Disallow: /*.jpg$

Sitemap: http://www.produkte24.de/kategorien