# robots.txt for http://www.globerecords.com/ #4-3-18 changed some to 10 sec- seems over 10 sec gets ignored ## 7-8-07 upped slurp to 15, added msnbot linit to 10, upped voila to 15, added google delay of 6 # last out of memory error was june 18, 2007 #Additional Symbols #Additional symbols allowed in the robots.txt directives include: # #'*' - matches a sequence of characters #'$' - anchors at the end of the URL string # #Using Wildcard Match: '*' #A '*' in robots directives is used to wildcard match a sequence of characters in your URL. #You can use this symbol in any part of the URL string that you provide in the robots directive. # #Example of '*': # User-agent: Slurp # Allow: /public*/ # Disallow: /*_print*.html # Disallow: /*?sessionid User-agent: * #Disallow: /cgi-bin/db/search.cgi/ # changed from /search.cgi*/ 10/2/08 #commented 9-9-10 #Disallow: /cgi-bin/ch/searchCH.cgi/ # changed from /searchCH.cgi*/ 10/2/08 #commented 9-9-10 #Disallow: /*&cart_id= #added 2/9/09 #removed 1/14/11 Disallow: /test/ # test area Disallow: /demo/ # client demos Disallow: /covers/ # added 6-4-08 10:52 pdt Disallow: /images/ # added 6-4-08 10:52 pdt Disallow: /includes/ Disallow: /tda/ # Disallow: /scripts/ # Disallow: /plugins/ # Disallow: /_logs/ # Disallow: /*view_cart= #added 4/25/24 Disallow: /*add_to_cart= #added 1/14/11 (and others like this) Disallow: /*update_cart= #added 1/14/11 Disallow: /*compute_order= #added 1/14/11 Disallow: /*continue_compute= #added 1/14/11 Disallow: /*checkout= #added 1/14/11 #Disallow: /cgi-bin/ #removed 1/14/11 User-agent: SemrushBot #added 10-25-18 Crawl-delay: 10 User-agent: SemrushBot-SA #added 10-25-18 Crawl-delay: 10 User-agent: BLEXBot # 12-80-18 changed from Crawl-Delay to Crawl-delay http://webmeup-crawler.com Crawl-delay: 10 User-agent: 008 #http://www.80legs.com/webcrawler.html Gecko/2008032620 Crawl-delay: 50 #added 3-18-11, upped from 40 to 50 4-6-11 User-agent: bingbot Crawl-delay: 10 #added 4-6-11 User-agent: discobot Crawl-delay: 50 #added 4-6-11 User-agent: Amazonbot # Amazon's user agent added 4-25-24 Crawl-delay: 20 #Disallow: /do-not-crawl/ #maybe need to use link-level rel=nofollow directive. Include these in your HTML like this to keep Amazonbot #for following and not crawling a particular link from your website. #Sign in User-agent: anthropic-ai #added 4-25-24 13:38 big log probs- ClaudeBot many accesses #Disallow: /cgi-bin/db/search.cgi/ Crawl-delay: 20 # Also, if it stops obeying robots.txt. Add this to the .htaccess file. # Code: # BrowserMatchNoCase "anthropic-ai" bad_bot # Order Deny,Allow # Deny from env=bad_bot User-agent: Slurp #yahoo #Disallow: /cgi-bin/db/search.cgi/ # changed from /search.cgi*/ 10/2/08 #removed 1/14/11 #Disallow: /cgi-bin/ch/searchCH.cgi/ # changed from /searchCH.cgi*/ 10/2/08 #removed 1/14/11 #Disallow: /*&cart_id= #added 2/9/09 #removed 1/14/11 Disallow: /test/ # test area Disallow: /demo/ # client demos Disallow: /covers/ # added 6-4-08 10:52 pdt Disallow: /images/ # added 6-4-08 10:52 pdt Disallow: /includes/ Disallow: /scripts/ # Disallow: /plugins/ # Disallow: /_logs/ # Disallow: /*add_to_cart= Disallow: /*update_cart= Disallow: /*compute_order= Disallow: /*continue_compute= Disallow: /*checkout= Disallow: /cgi-bin/store/pers2.cgi Crawl-delay: 60 User-Agent: The Knowledge AI #added 7-29-18 big error log probs- this bot 1/3 of accesses july 29 Disallow: / User-agent: Fatbot #thefind Crawl-delay: 40 Disallow: /*&cart_id= #added 2/9/09 Disallow: /cgi-bin/db/search.cgi/ # Disallow: /cgi-bin/ch/searchCH.cgi/ # added (itemno only) 7-3-08 User-agent: msnbot #Disallow: /*&cart_id= #added 2/9/09# removed 1-14-11 #Disallow: /cgi-bin/db/search.cgi/ #removed 1-14-11 #Disallow: /cgi-bin/ch/searchCH.cgi/ #removed 1-14-11 Disallow: /*add_to_cart= Disallow: /*update_cart= Disallow: /*compute_order= Disallow: /*continue_compute= Disallow: /*checkout= Disallow: /cgi-bin/store/pers2.cgi Crawl-delay: 40 User-agent: MJ12bot #added 8-21-17 Crawl-delay: 20 User-Agent: Twiceler Crawl-delay: 40 User-Agent: Googlebot #Disallow: /*&cart_id= #added 2/9/09 #removed 1-14-11 #Disallow: /cgi-bin/db/search.cgi/ #commented 9-9-10 #Disallow: /cgi-bin/ch/searchCH.cgi/ #commented 9-9-10 #Disallow: /*add_to_cart= Disallow: /*update_cart= Disallow: /*compute_order= Disallow: /*continue_compute= Disallow: /*checkout= Disallow: /cgi-bin/store/pers2.cgi Crawl-delay: 30 User-Agent: Charlotte #searchme trying Crawl-delay instead of disallow, starting 6-19-08 Crawl-delay: 50 #Disallow: / User-agent: ShopWiki Crawl-delay: 50 #Disallow: /cgi-bin/db/search.cgi/ #commented 9-9-10 #Disallow: /cgi-bin/ch/searchCH.cgi/ #commented 9-9-10 Disallow: /*add_to_cart= Disallow: /*update_cart= Disallow: /*compute_order= Disallow: /*continue_compute= Disallow: /*checkout= Disallow: /cgi-bin/store/pers2.cgi User-agent: ia_archiver #Disallow: /cgi-bin Disallow: /test/ # test area Disallow: /demo/ # client demos Disallow: /includes/ Disallow: /tda/ # Disallow: /scripts/ # User-agent: Teoma #Disallow: /cgi-bin Disallow: /test/ # test area Disallow: /demo/ # client demos Disallow: /includes/ Disallow: /tda/ # Disallow: /scripts/ # User-agent: NuSearch Spider Disallow: /cgi-bin Disallow: /test/ # test area Disallow: /demo/ # client demos Disallow: /includes/ Disallow: /tda/ # Disallow: /scripts/ # User-agent: VoilaBot #opened up 1/14/11 Crawl-delay: 30 #Disallow: /*&cart_id= #added 2/9/09 #Disallow: /cgi-bin #Disallow: /cgi-bin/db/search.cgi/ #Disallow: /cgi-bin/ch/searchCH.cgi/ Disallow: /test/ # test area Disallow: /demo/ # client demos Disallow: /includes/ Disallow: /tda/ # Disallow: /scripts/ # Disallow: /*add_to_cart= Disallow: /*update_cart= Disallow: /*compute_order= Disallow: /*continue_compute= Disallow: /*checkout= Disallow: /cgi-bin/store/pers2.cgi User-agent: Yandex Crawl-delay: 10 # sets a 2 second time-out User-agent: Gigabot Crawl-delay: 30 #Disallow: /cgi-bin maybe have to deactivate this #Disallow: /*&cart_id= #added 2/9/09 #Disallow: /cgi-bin/db/search.cgi/ #Disallow: /cgi-bin/ch/searchCH.cgi/ Disallow: /test/ # test area Disallow: /demo/ # client demos Disallow: /includes/ Disallow: /tda/ # Disallow: /scripts/ # Disallow: /*add_to_cart= Disallow: /*update_cart= Disallow: /*compute_order= Disallow: /*continue_compute= Disallow: /*checkout= Disallow: /cgi-bin/store/pers2.cgi