################################
# FreestyleRoars.com Smart Robots v2.3
################################
# This is a smart robots.txt which logs the ip and user agent of every visitor.
# Due to the compatibility issues between different bots and whether they support
# wildcards (*), multiple user-agents and end-anchors ($), I am providing different
# blocks for some.
#
# Detected Spider/Bot: None
#
# Headers Sent:
# Content-Type: text/plain
# Expires: Fri, 20 Feb 2009 22:29:57 GMT (24 hour validity)
#
# Let Spiders Know FreestyleRoars.com has a Sitemap
Sitemap: http://freestyleroars.com/sitemap.xml

# Google - Most Important Bot
#	Unfortunately a robots.txt will only stop it crawling certain urls, and NOT adding any
#	urls which it comes across into its index. So we're relying on a meta noindex tag.
User-agent: Googlebot
#	Disallow Trap Folder!
Disallow: /trap/
#	Don't index mobile versions
Disallow: /forum/index.php?*;wap
Disallow: /forum/index.php?*;wap2
Disallow: /forum/index.php?*;imode
#	Don't allow these #
Disallow: /forum/index.php?*.msg
Disallow: /forum/index.php?topic=*.msg*0$
Disallow: /forum/index.php?topic=*.msg*5$
Disallow: /forum/index.php?*.new
#	Anything with a ; disallow
Disallow: /forum/index.php?*;*

# Yahoo - Too aggressive
#	So limit it as much as possible.
User-agent: Slurp
#	Disallow Trap Folder!
Disallow: /trap/
#	Disallow Everything
Disallow: /forum/
#	Now allow bits and then disallow bits
Allow: /forum/index.php$
Allow: /forum/index.php?topic=*.0$
Allow: /forum/index.php?topic=*.*0$
Allow: /forum/index.php?topic=*.*5$
Allow: /forum/index.php?board=*.0$
Allow: /forum/index.php?board=*.*0$
Allow: /forum/index.php?board=*.*5$
#	But don't allow these #
Disallow: /forum/index.php?*.msg
Disallow: /forum/index.php?topic=*.msg*0$
Disallow: /forum/index.php?topic=*.msg*5$
Disallow: /forum/index.php?*.new
#	Anything with a ; disallow
Disallow: /forum/index.php?*;*

# Bad bot - Often ignores robots.txt - Waste of bandwidth
#	Despite claiming on their website to be a search engine in development
#	I'm suspicious as to whether they are a harvester pretending to be SE
#User-agent: Twiceler
#Disallow: /

# New bot - Monitor Behavior
#	Hostname: netid.com
#	Real IP: 77.232.72.47
#User-agent: FollowSite Bot
#Disallow: /

#User-agent: W3C-checklink
#Disallow: /

# Stop following PHPSESSID's
User-agent: MJ12bot
Disallow: /forum/index.php?PHPSESSID

# Catch all (remainder)
#	Will be followed by any bots other than ones identified above
#	Uses BASIC robots.txt directives without wildcards, end-anchors etc
#	So Spiders should understand these (including MSNBOT)
User-agent: *
#	Disallow Trap Folder!
Disallow: /trap/
# Default SMF Folders
Disallow: /attachments/
Disallow: /avatars/
Disallow: /chat/
Disallow: /downloads/
Disallow: /gallery/
Disallow: /forum/Games/
Disallow: /images/
Disallow: /Packages/
Disallow: /profile_pictures/
Disallow: /Smileys/
Disallow: /Sources/
Disallow: /Themes/
# Default SMF Actions
Disallow: /forum/index.php?action=activate
Disallow: /forum/index.php?action=admin
Disallow: /forum/index.php?action=calendar
Disallow: /forum/index.php?action=calendar;year
Disallow: /forum/index.php?action=collapse
Disallow: /forum/index.php?action=emailuser
Disallow: /forum/index.php?action=findmember
Disallow: /forum/index.php?action=help
Disallow: /forum/index.php?action=helpadmin
Disallow: /forum/index.php?action=login
Disallow: /forum/index.php?action=logout
Disallow: /forum/index.php?action=markasread
Disallow: /forum/index.php?action=mlist
Disallow: /forum/index.php?action=modifykarma
Disallow: /forum/index.php?action=notify
Disallow: /forum/index.php?action=notifyboard
Disallow: /forum/index.php?action=pm
Disallow: /forum/index.php?action=post
Disallow: /forum/index.php?action=printpage
Disallow: /forum/index.php?action=profile
Disallow: /forum/index.php?action=profile;u=
Disallow: /forum/index.php?action=recent
Disallow: /forum/index.php?action=register
Disallow: /forum/index.php?action=reminder
Disallow: /forum/index.php?action=reporttm
Disallow: /forum/index.php?action=search
Disallow: /forum/index.php?action=search;advanced
Disallow: /forum/index.php?action=sendtopic
Disallow: /forum/index.php?action=stats
Disallow: /forum/index.php?action=stats;collapse
Disallow: /forum/index.php?action=stats;expand
Disallow: /forum/index.php?action=theme
Disallow: /forum/index.php?action=unread
Disallow: /forum/index.php?action=unreadreplies
Disallow: /forum/index.php?action=verificationcode
Disallow: /forum/index.php?action=who
Disallow: /forum/index.php?theme
# SMF Mod Related
Disallow: /forum/index.php?action=arcade
Disallow: /forum/archive.php
Disallow: /forum/index.php?action=blog
Disallow: /forum/index.php?action=chat
Disallow: /forum/index.php?action=chess
Disallow: /forum/index.php?action=comment
Disallow: /forum/index.php?action=downloads
Disallow: /forum/index.php?action=downloads;sa=
Disallow: /forum/index.php?action=gallery
Disallow: /forum/index.php?action=gallery;sa=
Disallow: /forum/index.php?action=links
Disallow: /forum/index.php?action=mm
Disallow: /forum/index.php?action=recenttopics
Disallow: /forum/index.php?action=reporttm
Disallow: /forum/index.php?action=shop
Disallow: /forum/index.php?action=sitemap
Disallow: /forum/index.php?action=staff
Disallow: /forum/index.php?action=tags
Disallow: /forum/index.php?action=thankyou
Disallow: /forum/index.php?action=topposters
Disallow: /forum/index.php?action=usercp
Disallow: /forum/index.php?action=viewblog
Disallow: /forum/index.php?action=viewkarma
Disallow: /forum/index.php?action=viewers
Disallow: /forum/index.php?f=
Disallow: /forum/index.php?filter
Disallow: /forum/index.php?referredby
Disallow: /forum/index.php?action=arcade;category
Disallow: /forum/index.php?action=arcade;favorites
Disallow: /forum/index.php?action=arcade;sa=highscore
Disallow: /forum/index.php?action=arcade;sa=play;game
Disallow: /forum/index.php?action=arcade;sa=play;random
Disallow: /forum/index.php?action=arcade;sort
Disallow: /forum/index.php?action=arcade;sa=stats