User-agent: 008 Disallow: / User-agent: SiteAuditBot Crawl-delay: 1 Allow: / User-agent: Semrushbot-SI Allow: / User-agent: Yahoo Pipes 2.0 Disallow: / User-agent: Facebot Disallow: /archive/graphs.php User-agent: Bingbot Disallow: /archive/graphs.php User-agent: BingPreview Disallow: /archive/graphs.php User-agent: Twitterbot Allow: /blog/* Allow: /ranking-factors/* User-agent: * Disallow: /blog/*?img=* Disallow: /archive/graphs.php Disallow: /forrester-marketing-silos/* Disallow: /partner/ Disallow: /blog/search/* Disallow: /analytics/seomagic/ Disallow: /my_reports/reports/* Disallow: /archive/ Disallow: /admin/ Disallow: /users/ Disallow: /clients/ Disallow: /custom_report/ Disallow: /limit_hits/ Disallow: /payment/ Disallow: /no_cookies/ Disallow: /support/ Disallow: /double_activation/ Disallow: /projects/?* Disallow: /?*/ Disallow: /activation_successful* Disallow: /api.html?* Disallow: /partners/ Disallow: /partner/ Disallow: */dvd-mode/ Disallow: /unsubscribe/ Disallow: /unsubscribe/* Disallow: /demonstration/ Disallow: /demonstration/* Allow: /projects/ Disallow: */custom-report/ Disallow: */request-demo/ Allow: */info/kdt/ Disallow: */pleasereg/ Disallow: */custom-report/ Disallow: */custom-report Allow: */sensor/* Disallow: /swa/api/* Disallow: /redirect?url=* Disallow: /sso/ Disallow: /_/report_blank/ Disallow: /seo-ab-testing/* Disallow: /seo-old/* Disallow: /projects2/* Allow: /news/*?page=* # Community rules Allow: /blog/* Disallow: /blog/search/* Disallow: /blog/*utm_source=* Disallow: /blog-manager/* Allow: /user/* # Features new pages Disallow: /features/index2/ Disallow: /features/for-* # Disallow technical signup pages Allow: /login/$ Allow: /signup/$ Disallow: /login/* Disallow: /signup/* # Solutions Disallow: /solutions/$ #landing Disallow: /landing/ Disallow: /lp/ Allow: /lp/affiliate-program/* Allow: /lp/for-agencies/en/ Allow: /lp/black-friday/* Allow: /lp/black-friday-aff/* Allow: /lp/cyber-week/* Allow: /lp/growth-quadrant/ Allow: /lp/momandpop/* Allow: /seo/ Allow: /content-marketing/ Allow: /ppc/ Allow: /competitive-research/ Allow: /analytics/traffic/overview/ Disallow: /listings-management/superpower-landing2/ #academy Allow: /academy/* Disallow: /academy/*utm_source=* Disallow: /academy/*/results #websites Allow: /website/* Disallow: /website/search/* Disallow: /*/?* Crawl-delay: 20 # Sitemap files #Blog Sitemap: https://www.semrush.com/sitemap.xml Sitemap: https://pt.semrush.com/blog/sitemap/ Sitemap: https://es.semrush.com/blog/sitemap/ Sitemap: https://it.semrush.com/blog/sitemap/ Sitemap: https://fr.semrush.com/blog/sitemap/ Sitemap: https://ja.semrush.com/blog/sitemap/ #Academy Sitemap: https://www.semrush.com/academy/sitemap.xml Sitemap: https://es.semrush.com/academy/sitemap.xml Sitemap: https://pt.semrush.com/academy/sitemap.xml Sitemap: https://it.semrush.com/academy/sitemap.xml Sitemap: https://fr.semrush.com/academy/sitemap.xml Sitemap: https://de.semrush.com/academy/sitemap.xml