# allow all User-agent: * Disallow: # disallow all files in these directories Disallow: /cgi-bin/ Disallow: /flash/ Disallow: /images/ Disallow: /mp3/ Disallow: /news/ Disallow: /phpThumb/ Disallow: /scripts/ # disallow all files in /wp- directories Disallow: /wp-*/ # disallow all files ending with these extensions Disallow: /*.js$ Disallow: /*.inc$ Disallow: /*.css$ Disallow: /*.txt$