User-agent: *
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: twiceler
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Baidu
Disallow: /

User-agent: *alexa*
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: MSNBot
Disallow: /

User-agent: Slurp
Disallow: /

User-agent: Teoma
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: Scrubby
Disallow: /

User-agent: Robozilla
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: yahoo-mmcrawler
Disallow: /

User-agent: psbot
Disallow: /

User-agent: asterias
Disallow: /

User-agent: yahoo-blogs/v3.9
Disallow: /

User-agent: baiduspider
Disallow: /

User-agent: *
Allow: /Core/
Disallow: /Core/admin/
Disallow: /Core/blocks/
Disallow: /cgi-bin/
Disallow: /Core/cgi-bin/
Disallow: /Core/images/
Disallow: /Core/includes/
Disallow: /Core/language/
Disallow: /Core/modules/
Disallow: /Core/themes/
Disallow: /Core/admin.php
Disallow: /Core/config.php
Disallow: /Core/ips.php
Disallow: /Core/trap.php