# robots.txt for http://www.globerecords.com/
## 7-8-07 upped slurp to 15, added msnbot linit to 10,  upped voila to 15, added google delay of 6
# last out of memory error was june 18, 2007

#Additional Symbols
#Additional symbols allowed in the robots.txt directives include:
#
#'*' - matches a sequence of characters
#'$' - anchors at the end of the URL string
#
#Using Wildcard Match: '*'
#A '*' in robots directives is used to wildcard match a sequence of characters in your URL. 
#You can use this symbol in any part of the URL string that you provide in the robots directive.
#
#Example of '*':
#  User-agent: Slurp
#  Allow: /public*/
#  Disallow: /*_print*.html
#  Disallow: /*?sessionid
  
User-agent: *
Disallow: /cgi-bin/db/search.cgi/ # changed from /search.cgi*/ 10/2/08
Disallow: /cgi-bin/ch/searchCH.cgi/ # changed from /searchCH.cgi*/ 10/2/08  
Disallow: /*&cart_id=  #added 2/9/09
Disallow: /test/ # test area
Disallow: /demo/ # client demos
Disallow: /covers/ # added 6-4-08 10:52 pdt
Disallow: /images/ # added 6-4-08 10:52 pdt
Disallow: /includes/ 
Disallow: /tda/ # 
Disallow: /scripts/ # 
Disallow: /plugins/ # 
Disallow: /_logs/ # 

User-agent: Slurp #yahoo
Disallow: /cgi-bin/db/search.cgi/ # changed from /search.cgi*/ 10/2/08
Disallow: /cgi-bin/ch/searchCH.cgi/ # changed from /searchCH.cgi*/ 10/2/08  
Disallow: /*&cart_id=  #added 2/9/09
Disallow: /test/ # test area
Disallow: /demo/ # client demos
Disallow: /covers/ # added 6-4-08 10:52 pdt
Disallow: /images/ # added 6-4-08 10:52 pdt
Disallow: /includes/ 
Disallow: /scripts/ # 
Disallow: /plugins/ # 
Disallow: /_logs/ # 
Crawl-delay: 60

User-agent: Fatbot #thefind
Crawl-delay: 40
Disallow: /*&cart_id=  #added 2/9/09
Disallow: /cgi-bin/db/search.cgi/ # 
Disallow: /cgi-bin/ch/searchCH.cgi/ # added (itemno only) 7-3-08

User-agent: msnbot
Disallow: /*&cart_id=  #added 2/9/09
Disallow: /cgi-bin/db/search.cgi/ 
Disallow: /cgi-bin/ch/searchCH.cgi/ 
Crawl-delay: 40

User-Agent: Twiceler
Crawl-delay: 40

User-Agent: Googlebot
Disallow: /*&cart_id=  #added 2/9/09
Disallow: /cgi-bin/db/search.cgi/ 
Disallow: /cgi-bin/ch/searchCH.cgi/ 
Crawl-delay: 20

User-Agent: Charlotte  #searchme trying crawl-delay instead of disallow, starting 6-19-08
Crawl-delay: 50
#Disallow: /

User-agent: ShopWiki
Crawl-Delay: 50
Disallow: /cgi-bin/db/search.cgi/ 
Disallow: /cgi-bin/ch/searchCH.cgi/ 

User-agent: ia_archiver
#Disallow: /cgi-bin
Disallow: /test/ # test area
Disallow: /demo/ # client demos
Disallow: /includes/ 
Disallow: /tda/ # 
Disallow: /scripts/ # 

User-agent: Teoma
#Disallow: /cgi-bin
Disallow: /test/ # test area
Disallow: /demo/ # client demos
Disallow: /includes/ 
Disallow: /tda/ # 
Disallow: /scripts/ # 

User-agent: NuSearch Spider
Disallow: /cgi-bin
Disallow: /test/ # test area
Disallow: /demo/ # client demos
Disallow: /includes/ 
Disallow: /tda/ # 
Disallow: /scripts/ # 

User-agent: VoilaBot
Crawl-Delay: 30
Disallow: /*&cart_id=  #added 2/9/09
Disallow: /cgi-bin
Disallow: /cgi-bin/db/search.cgi/ 
Disallow: /cgi-bin/ch/searchCH.cgi/ 
Disallow: /test/ # test area
Disallow: /demo/ # client demos
Disallow: /includes/ 
Disallow: /tda/ # 
Disallow: /scripts/ # 

User-agent: Gigabot
Crawl-Delay: 30
#Disallow: /cgi-bin  maybe have to deactivate this
Disallow: /*&cart_id=  #added 2/9/09
Disallow: /cgi-bin/db/search.cgi/ 
Disallow: /cgi-bin/ch/searchCH.cgi/ 
Disallow: /test/ # test area
Disallow: /demo/ # client demos
Disallow: /includes/ 
Disallow: /tda/ # 
Disallow: /scripts/ #