User-agent: * #Disallow: / #disallow access to entire website Disallow: /knowledge-centre/knowledge-centre/documents/* #not sure about this one Disallow: /knowledge-centre/knowledge-centre/news-and-events/news-and-events/*? Disallow: /knowledge-centre/knowledge-centre/forums/*? Disallow: /error-pages/*? Disallow: /media/ Disallow: /new-registration.aspx Disallow: /buyers/property-search-results.aspx Disallow: /buyers/individual-property-page.aspx Disallow: /buyers/Print.aspx #Allowing specific pages Allow: /default.aspx Allow: /Terms-Conditions.aspx Allow: /ContactUs.aspx Allow: /AboutUs.aspx Allow: /SiteMap.aspx Allow: /dreamerscontent.aspx Allow: /Partners.aspx Allow: /FAQ.aspx Allow: /xmlsitemap.aspx Allow: /sitemap.xml Allow: /services-directory.aspx Allow: /sellers/sellers-landing.aspx Allow: /buyers/for-buyers.aspx Allow: /knowledge-centre/knowledge-centre.aspx Allow: /knowledge-centre/knowledge-centre/ Allow: /terms-conditions/ #Allow: /*?$ #Allowing where url ends in a question mark #Disallow: /*? #Blocking urls with a question mark #Disallow: /*.xls$ #specific file type