User-agent: *  # Applies to all robots

# Disallow indexing of downloadable files
Disallow: /appexchange/servlet/  # Disallow access to download servlet
Disallow: /appexchange/uploadedcontent*  # Disallow access to vf page for downloading files
Disallow: /appexchange/apex/uploadedcontent*  # Disallow access to vf page for downloading files
Disallow: /servlet/  # Disallow direct access to download servlet without site directory

# Disallow direct access to visualforce pages without site directory. Only Google may respect the regex statements. Google tried crawling these at one point, probably as an experiment.
Disallow: /listingDetail*
Disallow: /apex/listingDetail*
Disallow: /gettingStarted
Disallow: /apex/gettingStarted
Disallow: /browse*
Disallow: /apex/browse*
Disallow: /results*
Disallow: /apex/results*
Disallow: /savedListings
Disallow: /apex/savedListings
Disallow: /userAgreement
Disallow: /apex/userAgreement