#robots.txt file for domain http://www.diamondgoldjewellery.com User-agent:* Disallow: /cgi-bin/ Disallow: /logstats/ Disallow: /mailphp/ Disallow: /images/ Disallow: /scripts/ Disallow: /styles/ Disallow: /java.js Disallow: /cart.js Disallow: /validate.js Disallow: /emailCheck.js Disallow: /ccheck.js Disallow: /map.js Disallow: /setup.js Disallow: /navbars.js Disallow: /layer.js Disallow: /mail.php Disallow: /order.php Disallow: /search.inc Disallow: /layers.inc Disallow: /links.inc ############################################################################################################### # # # Following text taken from www.robotstxt.org - Modified by H.Yeend of FreeTimers.com # # Note that you need a separate "Disallow" line for every URL prefix you want to exclude # -- you cannot say "Disallow: /cgi-bin/ /tmp/". # # Note also that only 1 robots.txt may exist on a server, and it must be in the 'root' directory # (eg http://www.thisdomain.com/robots.txt) # # Also, you may not have blank lines in a record, as they are used to delimit multiple records. # # Note also that regular expression are not supported in either the User-agent or Disallow lines. # The '*' in the User-agent field is a special value meaning "any robot". # E.G: You cannot have lines like "Disallow: /tmp/*" or "Disallow: *.gif". # # Everything not explicitly disallowed is considered fair game to retrieve. Here follow some examples: # # To exclude all robots from the entire server # User-agent: * # Disallow: / # # To allow all robots complete access (Or just don't have a robots.txt file.) # User-agent: * # Disallow: # # To exclude all robots from part of the server # User-agent: * # Disallow: /cgi-bin/ # Disallow: /tmp/ # Disallow: /private/ # # To exclude a single robot # User-agent: BadBot # Disallow: / # # To allow only a single robot # User-agent: WebCrawler # Disallow: # # User-agent: * # Disallow: / # # To exclude all files except one # This is currently a bit awkward, as there is no "Allow" field. # The easy way is to put all files to be disallowed into a separate directory, say "/~joe/docs/", # and leave the one file in the level above this directory: # User-agent: * # Disallow: /~joe/docs/ # # Alternatively you can explicitly disallow all disallowed pages: # User-agent: * # Disallow: /~joe/private.html # Disallow: /~joe/foo.html # Disallow: /~joe/bar.html # # ###############################################################################################################