User-agent: * 
Allow: /
Allow: .htm$
Allow: .html$
Allow: .asp$
Disallow: /admin
Disallow: /database
Disallow: /include
Disallow: /script
Disallow: /scripts