Sitemap: http://cnx.org/sitemap_index.xml
User-agent: ScoutJet
Crawl-delay: 10
Disallow: legacy.cnx.org
Disallow: /lenses
Disallow: /browse_content
Disallow: /content/
Disallow: /content$
Disallow: /*/pdf$
Disallow: /*/epub$
Disallow: /*/complete$
Disallow: /*/offline$
Disallow: /*?format=*$
Disallow: /*/multimedia$
Disallow: /*/lens_add?*$
Disallow: /lens_add
Disallow: /*/lens_view/*$
Disallow: /content/*view_mode=statistics$

User-agent: *
Disallow: legacy.cnx.org
Disallow: /lenses
Disallow: /browse_content
Disallow: /content/
Disallow: /content$
Disallow: /*/pdf$
Disallow: /*/epub$
Disallow: /*/complete$
Disallow: /*/offline$
Disallow: /*?format=*$
Disallow: /*/multimedia$
Disallow: /*/lens_add?*$
Disallow: /lens_add
Disallow: /*/lens_view/*$
Disallow: /content/*view_mode=statistics$

User-agent: Baiduspider
Crawl-delay: 10
Disallow: legacy.cnx.org
Disallow: /lenses
Disallow: /browse_content
Disallow: /content/
Disallow: /content$
Disallow: /*/pdf$
Disallow: /*/epub$
Disallow: /*/complete$
Disallow: /*/offline$
Disallow: /*?format=*$
Disallow: /*/multimedia$
Disallow: /*/lens_add?*$
Disallow: /lens_add
Disallow: /*/lens_view/*$
Disallow: /content/*view_mode=statistics$

User-agent: BecomeBot
Crawl-delay: 20
Disallow: legacy.cnx.org
Disallow: /lenses
Disallow: /browse_content
Disallow: /content/
Disallow: /content$
Disallow: /*/pdf$
Disallow: /*/epub$
Disallow: /*/complete$
Disallow: /*/offline$
Disallow: /*?format=*$
Disallow: /*/multimedia$
Disallow: /*/lens_add?*$
Disallow: /lens_add
Disallow: /*/lens_view/*$
Disallow: /content/*view_mode=statistics$

User-agent: Slurp
Crawl-delay: 10
Disallow: legacy.cnx.org
Disallow: /lenses
Disallow: /browse_content
Disallow: /content/
Disallow: /content$
Disallow: /*/pdf$
Disallow: /*/epub$
Disallow: /*/complete$
Disallow: /*/offline$
Disallow: /*?format=*$
Disallow: /*/multimedia$
Disallow: /*/lens_add?*$
Disallow: /lens_add
Disallow: /*/lens_view/*$
Disallow: /content/*view_mode=statistics$

