#------------------------------------------------------
#	To tell a crawler where your sitemap is
#------------------------------------------------------
#		Sitemap: http://www.example.tld/sitemap.xml
#
#------------------------------------------------------
#	To exclude all robots from the entire server
#------------------------------------------------------
#		User-agent: *
#		Disallow: /
#
#------------------------------------------------------
#	To allow all robots complete access
#------------------------------------------------------
#		User-agent: *
#		Disallow:
# 
#------------------------------------------------------
#	To exclude all robots from part of the server
#------------------------------------------------------
#		User-agent: *
#		Disallow: /cgi-bin/
#		Disallow: /tmp/
#		Disallow: /junk/
#
#------------------------------------------------------
#	To exclude all robots from the entire server
#------------------------------------------------------
#		User-agent: *
#		Disallow: /
#
#------------------------------------------------------
#	To exclude a single robot
#------------------------------------------------------
#		User-agent: BadBot
#		Disallow: /
#
#------------------------------------------------------
#	To allow a single robot
#------------------------------------------------------
#		User-agent: Google
#		Disallow:
#		User-agent: *
#		Disallow: /
#
#------------------------------------------------------
#	To exclude individual pages
#------------------------------------------------------
#		User-agent: *
#		Disallow: /admin/login.php
#		Disallow: /main.html
#
#------------------------------------------------------
#	To disallow any JPEG image files
#------------------------------------------------------
#		User-agent: *
#		Disallow: /*.jpg$
#
#	There are no wildcards (also known as pattern matching) in the official robots.txt specs, but various search engines have added extensions to support this. For example, Google, MSN Search, and Yahoo! allow an asterisk (*) to match any sequence of characters, and a dollar sign ($) to match the end of the URL.
#------------------------------------------------------