# disallow archiving site User-agent: ia_archiver Disallow: / # disable duggmirror User-agent: duggmirror Disallow: / #disallow all files ending with these extensions User-agent: Googlebot Disallow: /*.php$ Disallow: /*.js$ Disallow: /*.inc$ Disallow: /*.css$ Disallow: /*.gz$ Disallow: /*.wmv$ Disallow: /*.cgi$ Disallow: /*.xhtml$ Disallow: /*.phtml$ # disallow all files with ? in url #Disallow: /*?* Disallow: /share?* Disallow: /print?* Disallow: /info?* Disallow: /piwik/ User-agent: Googlebot-Image #Disallow: Allow: /*