User-agent: Googlebot
#Google allows the allow command eg:
#Disallow: /folder1/
#Allow: /folder1/myfile.html
#Ignored by Google:
#  Crawl-delay: 5
# Disallow all directories and files within
Disallow: /cgi-bin/
Disallow: /wordpress/wp-admin/
Disallow: /wordpress/wp-includes/
Disallow: /wordpress/wp-content/plugins/
Disallow: /tag/
# Miva
Disallow: /mm5/
# Offgid
Disallow: /adverts/
Disallow: /manager/
Disallow: /awstats/
Disallow: /newsletter/
Disallow: /common/
# Disallow all files ending with these extensions
Disallow: /*.php$
Disallow: /*.js$
Disallow: /*.inc$
Disallow: /*.css$
# Disallow parsing individual post feeds, categories and trackbacks..
#Changed next two lines 2/6/08 as they may be causing Google Alerts to stop working
#Disallow: */trackback/
#Disallow: */feed/
Disallow: /category/

User-agent: *
Crawl-delay: 5
# Disallow all directories and files within
Disallow: /cgi-bin/
Disallow: /wordpress/wp-admin/
Disallow: /wordpress/wp-includes/
Disallow: /wordpress/wp-content/plugins/
Disallow: /tag/
# Miva
Disallow: /mm5/
# Offgid
Disallow: /adverts/
Disallow: /manager/
Disallow: /awstats/
Disallow: /newsletter/
Disallow: /common/
# Disallow all files ending with these extensions
Disallow: /*.php$
Disallow: /*.js$
Disallow: /*.inc$
Disallow: /*.css$
# Disallow parsing individual post feeds, categories and trackbacks..
# Next two lines gave errors in Alexa
#  Disallow: */trackback/
#  Disallow: */feed/
Disallow: /category/

User-agent: feedburner
Crawl-delay: 5
# Disallow all directories and files within
Disallow: /cgi-bin/
Disallow: /wordpress/wp-admin/
Disallow: /wordpress/wp-includes/
Disallow: /wordpress/wp-content/plugins/
Disallow: /tag/
# Miva
Disallow: /mm5/
# Offgid
Disallow: /adverts/
Disallow: /manager/
Disallow: /awstats/
Disallow: /newsletter/
Disallow: /common/
# Disallow all files ending with these extensions
Disallow: /*.php$
Disallow: /*.js$
Disallow: /*.inc$
Disallow: /*.css$

User-agent: ConveraCrawler
Disallow: /

# Yahoo
User-agent: Slurp
Crawl-delay: 5

# MSN
User-agent: MSNBot
Crawl-delay: 5

Sitemap: http://www.off-grid.net/sitemap.xml.gz