# Robots.txt tells search engine crawlers which URLs the crawler can access on your site. This is used mainly to avoid overloading your site with requests

User-agent: *
Disallow: /error/
Disallow: /training/Player/
Disallow: /training/Courses/Transcript/
Disallow: /training/courses/transcript/
Disallow: /training/courses/Transcript/
Disallow: /training/Courses/Transcript/
Disallow: /training/Courses/ExerciseFiles/
Disallow: /training/Courses/Assessment/
Disallow: /training/Login/
Disallow: /training/Subscribe/Step1
Disallow: /Courses/Transcript/
Disallow: /courses/transcript/
Disallow: /courses/Transcript/
Disallow: /Courses/Transcript/
Disallow: /Courses/exercise-files/
Disallow: /courses/Exercise-Files/
Disallow: /Courses/Exercise-Files/
Disallow: /Courses/exercise-files/
Disallow: /Courses/Assessment/
Disallow: /Courses/assessment/
Disallow: /courses/assessment/
Disallow: /courses/Assessment/
Disallow: /content/dist/fonts/
Disallow: /account/
Disallow: /training/Error/
Disallow: /a/subscribe
Disallow: /customer/
Disallow: /search
Disallow: /newsroom/in-the-news/
Disallow: /buy
Disallow: /redeemlink
Disallow: /content/dam
Disallow: /_jcr_content/
Disallow: /content/pluralsight/en/jcr:content/

User-agent: Twitterbot
Allow: /content/dam

User-agent: facebookexternalhit
Allow: /content/dam

User-agent: LinkedInBot/1.0
Allow: /content/dam

User-agent: Yandex
Disallow: /
Crawl-delay: 300

User-Agent: rogerbot
Disallow: /
Crawl-delay: 300

user-agent: AhrefsBot
disallow: /

user-agent: MJ12bot
disallow: /

user-agent: Mozilla/4.0+(compatible;+MSIE+4.01;+Windows+NT;+MS+Search+6.0+Robot)
disallow: /

user-agent: 008
disallow: /

user-agent: sistrix
disallow: /

User-agent: grapeshot
Disallow: /

User-agent: WordPress*
Crawl-delay: 10

User-agent: proximic
Disallow: /

User-agent: MaxPoint Bot
Disallow: /

User-agent: MaxPointCrawler
Disallow: /

User-agent: Sistrix Crawler
Disallow: /

User-agent: HTTrack
Disallow: /
Disallow: /training

User-agent: RavenCrawler
Disallow: /
Disallow: /training

User-agent: ZumBot*
Disallow: /
Disallow: /training

User-agent: SputnikBot
Disallow: /

User-agent: Sputnik
Disallow: /

User-agent: SputnikBot/2.3
Disallow: /

User-agent: spbot
Disallow: /

User-agent: spbot/4.0.9
Disallow: /

User-agent: ia_archiver
Crawl-delay: 5

User-agent: BLEXBot
Disallow: /

User-agent: Spinn3r
Disallow: /

User-agent: Zing-BottaBot/2.0 
Crawl-delay:90 
Disallow: /

User-agent: crawler4j
Disallow: /

User-agent: Diffbot/0.1
Disallow: /
Crawl-delay: 300

User-agent: 360Spider
Disallow: /

User-agent: Caliperbot/1.0
Crawl-delay: 5

User-agent: Sogou+web+spider/4.0
Disallow: /

User-agent: bingbot
Disallow: /content/
Crawl-delay: 1

Sitemap: https://www.pluralsight.com/sitemap.xml