User-Agent: MJ12bot
Crawl-Delay: 20
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: AhrefsBot
Disallow: / 

User-agent: *
Disallow: /shop/index.php/
Disallow: /shop/wishlist/
Disallow: /shop/shopadmin/
Disallow: /shop/review/product/
Disallow: /shop/sendfriend/
Disallow: /shop/enable-cookies/
Disallow: /shop/LICENSE.txt
Disallow: /shop/LICENSE.html
Disallow: /shop/directory/

#Stop crawling user account and checkout pages by search engine robot:
Disallow: /shop/checkout/
Disallow: /shop/onestepcheckout/
Disallow: /shop/customer/account/
Disallow: /shop/customer/account/login/

#Blocking native catalog and search pages:
Disallow: /shop/catalogsearch/
Disallow: /shop/catalog/product_compare/

#Sometimes Webmasters block pages with filters..
#Disallow: /shop/*?dir*
#Disallow: /shop/*?dir=desc
#Disallow: /shop/*?dir=asc
#Disallow: /shop/*?limit=all
#Disallow: /shop/*?mode*

#More reasonable to use canonical tag on these pages.
#Blocking CMS directories.
Disallow: /shop/app/
Disallow: /shop/bin/
Disallow: /shop/dev/
Disallow: /shop/lib/
Disallow: /shop/phpserver/


#Blocking duplicate content:
Disallow: /shop/tag/
Disallow: /shop/review/

#Don’t forget about domain and sitemap pointing:
Host: www.originlive.com
Sitemap: https://www.originlive.com/shop/sitemap.xml
Sitemap: https://www.originlive.com/sitemap_index.xml