Sitemap: http://www.mashget.com/sitemap.xml

User-agent: *
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /page/
Disallow: /toolbar/
Disallow: /archives/
Disallow: */trackback/
Disallow: */comment-page*
Disallow: /*.js
Disallow: /*.css
Allow: /

User-agent: twiceler
Disallow: /

User-agent: baiduspider
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: MLBot
Disallow: / 

User-agent: Googlebot
Disallow: /2009/
Disallow: /2008/
Disallow: /topic/
Disallow: /tag/
Disallow: /category/
Disallow: /related-stories/
Allow: /