# robots.txt for http://www.planetdata.net/

User-agent: *
Disallow: clicks.php
Disallow: redirect.php
Disallow: css/
Disallow: errors/
Disallow: images/
Disallow: includes/
Disallow: themes/
Disallow: users/

# Allow only specific directories

User-agent: Googlebot
Crawl-delay: 120

User-agent: Slurp
Crawl-delay: 120

User-agent: msnbot
Crawl-delay: 120

User-Agent: Charlotte
Disallow: /

User-agent: DotBot
Disallow: /

User-Agent: Exabot
Disallow: /

User-agent: Radian6
Disallow: /

User-agent: ScoutJet
Disallow: /

User-agent: Spinn3r
Disallow: /

User-Agent: Twiceler
Disallow: /

User-agent: XmarksFetch
Disallow: /