User-agent: * # CSS, JS, Images Allow: /core/*.css$ Allow: /core/*.css? Allow: /core/*.js$ Allow: /core/*.js? Allow: /core/*.gif Allow: /core/*.jpg Allow: /core/*.jpeg Allow: /core/*.png Allow: /core/*.svg Allow: /profiles/*.css$ Allow: /profiles/*.css? Allow: /profiles/*.js$ Allow: /profiles/*.js? Allow: /profiles/*.gif Allow: /profiles/*.jpg Allow: /profiles/*.jpeg Allow: /profiles/*.png Allow: /profiles/*.svg # Directories Disallow: /core/ Disallow: /profiles/ Disallow: /en-adm Disallow: /old-no Disallow: /old-nl Disallow: /old-cz Disallow: /old-fi Disallow: /old-dk # Files Disallow: /README.txt Disallow: /web.config # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /filter/tips Disallow: /node/add/ Disallow: /s/ Disallow: /user/register/ Disallow: /user/password/ Disallow: /user/login/ Disallow: /user/logout/ # Paths (no clean URLs) Disallow: /index.php/admin/ Disallow: /index.php/comment/reply/ Disallow: /index.php/filter/tips Disallow: /index.php/node/add/ Disallow: /index.php/search/ Disallow: /index.php/user/password/ Disallow: /index.php/user/register/ Disallow: /index.php/user/login/ Disallow: /index.php/user/logout/ Disallow: /*.gpx Disallow: /*/rest/V1/ Disallow: /*/node/ Disallow: /*/taxonomy/ Disallow: /*/gender/ Disallow: /00/ Disallow: /*?CMPID Disallow: /*?cmpid Disallow: /*?sfmcid Disallow: /*?color_filter_front Disallow: /*?dir Disallow: /*?is_scroll Disallow: /*?mode Disallow: /*?order Disallow: /*?is_ajax Disallow: /*?limit Disallow: /*?ctc Disallow: /*?typeProduct Disallow: /*?page Disallow: /*?u= Disallow: /*?sex Disallow: /*?id= Disallow: /*?ref= Disallow: /*catalogsearch/result* Disallow: /*/jobs/ Disallow: /*/get-user-country Disallow: /*/external/index/getInformation* Disallow: /*/customer/ Disallow: /*/external/ Disallow: /*/range/ Disallow: /*/interface/ Disallow: /ajax/salable/ Disallow: /*/get-user-country Disallow: /*?ajax_form Disallow: /*?main_page Disallow: /front_res/ Disallow: /hk/ Disallow: /others/ Disallow: /resizeImage.aspx Disallow: /skin/frontend/ Disallow: /tw/ Disallow: /*?gclid Disallow: /*?article Disallow: /*?fee Disallow: /*gclsrc Disallow: /*/cart/ Disallow: /*?cata_manage Disallow: /*?is_gtx Disallow: /*?email Disallow: /*?limit Disallow: /*?mindbox Disallow: /*query= Disallow: /*?_pxhc= Disallow: /*?bvstate= Disallow: /*?srsltid= Disallow: /*?pid= Disallow: /*?is_scroll= Disallow: /*?language= Disallow: /*?___store= Disallow: /*?c= Disallow: /*?lpoid= Disallow: /*?bvstate= Disallow: /*?redirect= Disallow: /*/catalog/product/view/ # Add rules in robots.txt to avoid crawl trap (SLN-10744) Disallow: /*?cat=* Disallow: /*html=$ # Block some bots User-agent: Yandex Disallow: / User-agent: YandexBot Disallow: / User-agent: UbiCrawler Disallow: / User-agent: DOC Disallow: / User-agent: Zao Disallow: / User-agent: sitecheck.internetseer.com Disallow: / User-agent: Zealbot Disallow: / User-agent: MSIECrawler Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebCopier Disallow: / User-agent: Fetch Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: / User-agent: linko Disallow: / User-agent: HTTrack Disallow: / User-agent: Microsoft.URL.Control Disallow: / User-agent: Xenu Disallow: / User-agent: larbin Disallow: / User-agent: libwww Disallow: / User-agent: ZyBORG Disallow: / User-agent: Download Ninja Disallow: / User-agent: rogerbot Disallow: / User-agent: spbot Disallow: / User-agent: Sosospider Disallow: / User-agent: BacklinkCrawler Disallow: / User-agent: ia_archiver Disallow: / User-agent: Ezooms Disallow: / User-agent: Gigabot Disallow: / User-agent: findlinks Disallow: / User-agent: SurveyBot Disallow: / User-agent: SEOENGBot Disallow: / User-agent: BPImageWalker Disallow: / User-agent: bdbrandprotect Disallow: / User-agent: LinkWalker Disallow: / User-agent: Updownerbot Disallow: / User-agent: HTTrack Disallow: / User-agent: AppEngine-Google Disallow: /