User-agent: psbot
Disallow: /
User-agent: Krugle
Disallow: /
User-agent: Indy Library
Disallow: /
User-agent: NASA Search
Disallow: /
User-agent: bot/1.0 (bot; http://; bot@bot.bot)
Disallow: /
User-agent: python-urllib
Disallow: /
User-agent: Jakarta Commons-HttpClient/3.0.1
Disallow: /
User-agent: www.marcuswalker.ws
Disallow: /

User-agent: *
# Data folders
Disallow: /charts/
Disallow: /css/
Disallow: /db/
Disallow: /flash/
Disallow: /img/
Disallow: /inc/
Disallow: /js/
Disallow: /pdf/
Disallow: /video/
Disallow: /errors/
Disallow: /login/
# Advisor pages
Disallow: /en_advisor*.*
Disallow: /fr_advisor*.*
# Stand-alone pages
Disallow: /tracking/
Disallow: /error*.*