# robots.txt file for Newspapers.com
# For more site information, contact webmaster@newspapers.com
# Some bots are known to be trouble, particularly those designed to copy
# entire sites. Please obey robots.txt.
Sitemap: https://www.newspapers.com/sitemap/pages
Sitemap: https://www.newspapers.com/sitemap/clippings
Sitemap: https://www.newspapers.com/sitemap/titles
User-Agent: *
Disallow: /busy.html
Disallow: /error.html
Disallow: /error.php
Disallow: /download/
Disallow: /clippings/download/
# Slow Bots see https://ahrefs.com/robot for more info
User-Agent: AhrefsBot
Disallow: /busy.html
Disallow: /error.html
Disallow: /error.php
Crawl-Delay: 5
# Blocked Bots
User-agent: sitecheck.internetseer.com
User-agent: Zealbot
User-agent: SiteSnagger
User-agent: WebStripper
User-agent: WebCopier
User-agent: Fetch
User-agent: Offline Explorer
User-agent: Teleport
User-agent: TeleportPro
User-agent: WebZIP
User-agent: linko
User-agent: HTTrack
User-agent: Xenu
User-agent: larbin
User-agent: libwww
User-agent: ZyBORG
User-agent: Download Ninja
User-agent: MyFamilyBot
Disallow: /