# Website Sitemap
Sitemap: http://standupsurfshop.com.au/sitemap.xml

# Crawlers Setup
User-agent: *
Crawl-delay: 10

# Directories
Disallow: /app/
Disallow: /lib/

# Allowable Index
Allow: /*?p=
Allow: /catalog/seo_sitemap/category/
Allow: /skin/
Allow: /media/
Allow: /js/

# Paths (clean URLs)
Disallow: /index.php/
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /customer/
Disallow: /customize/
Disallow: /newsletter/
Disallow: /poll/
Disallow: /review/
Disallow: /sendfriend/
Disallow: /tag/
Disallow: /wishlist/
Disallow: /directory/currency/switch/currency/
Disallow: /sales/order/

# Paths
Disallow: /*.php$
Disallow: /*?SID=

# Attributes
Disallow: /*?dir=*
Disallow: /*?order=*
Disallow: /*?price=*
Disallow: /*/filter/*

# Allow CSS / JS
Allow: /*.js*$
Allow: /*.css*$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.jpg$


User-agent: Googlebot-Image
Disallow: /
Allow: /media/catalog/product/


# Block other crawlers

User-agent: 008
Disallow: /

User-agent: sistrix
Disallow: /

User-agent: Fatbot
Disallow: /

User-agent: superfish
Disallow: /

User-agent: dtSearchSpider
Disallow: /

User-agent: EtaoSpider
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: Mail.RU_Bot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: TwengaBot-2.0
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: Baiduspider
Disallow: /