User-agent: *

Disallow: /core/

#On empeche la facette
Disallow: /*p=*
Disallow: /*l=*
Disallow: /*h=*
Disallow: /*y=*
Disallow: /*idm=*
Disallow: /*idmo=*
Disallow: /*ida=*
Allow: /*tUrl=*


# On empeche l'indexation des fichiers sensibles
Disallow: /*.php
Disallow: /*.inc$
Disallow: /*.gz$
Disallow: /*.swf$
Disallow: /*.wmv$
Disallow: /*.cgi$
Disallow: /*.xhtml$
Disallow: /*.pdf$

User-agent: SemrushBot
Disallow: /

User-Agent: MJ12bot
Disallow: /

User-agent: MauiBot 
Disallow: /

User-agent: MauiBot (crawler.feedback+wc@gmail.com)
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

# https://megaindex.com/crawler
User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: megaindex.com
Disallow: /