########################## ## tfs robots.txt: 22/01/16 ########################## User-agent: archive.org_bot User-agent: bingbot User-agent: Googlebot User-agent: Googlebot-Image User-agent: msnbot User-agent: msnbot-media User-agent: Slurp Allow: / User-agent: 007ac9 Crawler Disallow: / User-agent: Abonti Disallow: / User-agent: AdnormCrawler Disallow: / User-agent: AdvBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: BaiduSpider Disallow: / User-agent: BaiduSpider-image Disallow: / User-agent: betaBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: BPImageWalker Disallow: / User-agent: ca-crawler Disallow: / User-agent: cmscrawler Disallow: / User-agent: CMS crawler Disallow: / User-agent: coccoc Disallow: / User-agent: datagnionbot Disallow: / User-agent: deadlinkchecker Disallow: / User-agent: DeuSu Disallow: / User-agent: DomainAppender Disallow: / User-agent: DotBot Disallow: / User-agent: EasouSpider Disallow: / User-agent: electricmonk Disallow: / User-agent: Exabot Disallow: / User-agent: Findxbot Disallow: / User-agent: gimme60bot Disallow: / User-agent: gimmeUSAbot Disallow: / User-agent: GrapeshotCrawler Disallow: / User-agent: GWPImages Disallow: / User-agent: HaosouSpider Disallow: / User-agent: IErachnid Disallow: / User-agent: imagescoccoc Disallow: / User-agent: ips-agent Disallow: / User-agent: Linkdexbot Disallow: / User-agent: LinkParser Disallow: / User-agent: LinkSaver Disallow: / User-agent: Lipperhey Disallow: / User-agent: LSSRocketCrawler Disallow: / User-agent: Mail.RU_Bot Disallow: / User-agent: meanpathbot Disallow: / User-agent: memoryBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: MojeekBot Disallow: / User-agent: netEstate NE Crawler Disallow: / User-agent: NetSeer crawler Disallow: / User-agent: nbot Disallow: / User-agent: OrangeBot Disallow: / User-agent: PagesInventory Disallow: / User-agent: PageAnalyser Disallow: / User-agent: panscient Disallow: / User-agent: Pinterest Disallow: / User-agent: plukkie Disallow: / User-agent: publiclibraryarchive.org/1.0 Disallow: / User-agent: Quantify Disallow: / User-agent: Riddler Disallow: / User-agent: rogerbot Disallow: / User-agent: ScoutJet Disallow: / User-agent: Screaming Frog Disallow: / User-agent: SearchmetricsBot Disallow: / User-agent: SeznamBot Disallow: / User-agent: SISTRIX Crawler Disallow: / User-agent: SiteExplorer Disallow: / User-agent: SkimBot Disallow: / User-agent: SMTBot Disallow: / User-agent: spbot Disallow: / User-agent: ssearch_bot Disallow: / User-agent: TestCrawler Disallow: / User-agent: TurnitinBot Disallow: / User-agent: TweetmemeBot Disallow: / User-agent: uMBot Disallow: / User-agent: Vagabondo Disallow: / User-agent: VoilaBot Disallow: / User-agent: WBSearchBot Disallow: / User-agent: webcollage Disallow: / User-agent: webcrawler Disallow: / User-agent: WeSEE Disallow: / User-agent: woobot Disallow: / User-agent: worldwebheritage Disallow: / User-agent: Wotbox Disallow: / User-agent: Yacybot Disallow: / User-agent: YandexBot Disallow: / User-agent: YandexImages Disallow: / User-agent: YisouSpider Disallow: / User-agent: yoozBot Disallow: / ##################### ## tfs robots.txt END