# --------------------------------------------------------------------------- # Crawl control. # # This file is served for every tenant (server.js). The per-site Sitemap line is # appended dynamically by the route, so do not add a Sitemap: directive here. # # IMPORTANT: this is an Angular single page app. Googlebot must be able to fetch # /static/*, /:lang-COUNTRY/partials-* and the /api/* backend in order to render # a page at all. Never disallow those - doing so does not save crawl budget, it # stops the site being indexed. # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # SEO / backlink crawlers. High request volume, no referral traffic. # --------------------------------------------------------------------------- User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: Barkrowler Disallow: / User-agent: BLEXBot Disallow: / User-agent: SeekportBot Disallow: / User-agent: serpstatbot Disallow: / User-agent: PetalBot Disallow: / User-agent: Bytespider Disallow: / # --------------------------------------------------------------------------- # Baidu. # # A real search engine, unlike the block above - blocking it forgoes Baidu # indexing. That is the trade accepted here: the audience for Finnish local # events does not search on Baidu, so the crawl is cost without return. # # Note Baiduspider is still listed in SEARCH_CRAWLER_PATTERNS # (app/modules/fn-crawler-page.js), so it lands on the reduced page if it # crawls anyway. Left in place deliberately - robots.txt is a request, not # enforcement. # --------------------------------------------------------------------------- User-agent: Baiduspider Disallow: / # --------------------------------------------------------------------------- # AI training / answer-engine crawlers. # # These consume real bandwidth and send little or no referral traffic, so they # are blocked here for cost. This is a BUSINESS decision, not a technical one: # delete this block if you would rather the events be visible inside AI # assistants. Blocking Google-Extended affects Gemini/Vertex training ONLY - it # has no effect on Google Search indexing or ranking. # --------------------------------------------------------------------------- User-agent: GPTBot Disallow: / User-agent: OAI-SearchBot Disallow: / User-agent: ChatGPT-User Disallow: / User-agent: CCBot Disallow: / User-agent: ClaudeBot Disallow: / User-agent: anthropic-ai Disallow: / User-agent: Claude-Web Disallow: / User-agent: Google-Extended Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: meta-externalagent Disallow: / User-agent: Amazonbot Disallow: / User-agent: Diffbot Disallow: / User-agent: omgili Disallow: / User-agent: ImagesiftBot Disallow: / # --------------------------------------------------------------------------- # Everyone else, including Googlebot and Bingbot. # # Only non-indexable surfaces are disallowed. Page and topic URLs stay fully # crawlable, including their query strings - the sitemap emits them with # ?count=/?sort=/?reverse= and ?mode=, so a blanket query-string rule would # deindex the site's own canonical URLs. # --------------------------------------------------------------------------- User-agent: * crawl-delay: 10 # Search results. Faceted over q/start/end/categories/area/age/sort, so the URL # space is effectively unbounded. Not present in any sitemap. Disallow: /*/search # Static informational pages Disallow: /*/about Disallow: /*/privacy Disallow: /*/cookie-policy Disallow: /*/terms # Authentication and account flows Disallow: /*/login Disallow: /*/signup Disallow: /*/signup-confirmation/* Disallow: /*/forgot-password Disallow: /*/reset-password Disallow: /*/reset-signup-confirmation/* Disallow: /*/change-email-confirmation/* Disallow: /*/invited # Map view - renders the whole result set client side Disallow: /*/map/* # Personal profile surfaces Disallow: /*/profile/*/updates Disallow: /*/profile/*/news-feed Disallow: /*/profile/*/summary Disallow: /*/profile/*/questions Disallow: /*/profile/*/answers Disallow: /*/profile/*/messages Disallow: /*/profile/*/notifications Disallow: /*/profile/*/bookmarks Disallow: /*/profile/*/my-pages Disallow: /*/my-log # Content creation Disallow: /*/create-page # Site management and moderation consoles. Authenticated, but an unauthenticated # crawl still costs a full SPA render plus the API round trips behind it. Disallow: /*/management # /site/:site_id only. NOT "/*/site", which would also match //sitemap - a public page. Disallow: /*/site/ Disallow: /*/sites Disallow: /*/sub-sites Disallow: /*/users-by-state Disallow: /*/users-email-not-confirmed Disallow: /*/pages-by-state Disallow: /*/pages-external Disallow: /*/updates-by-state Disallow: /*/updates-reported Disallow: /*/widgets-by-state Disallow: /*/widget-ads-by-state Disallow: /*/publications-by-status Disallow: /*/prints Disallow: /*/letters Disallow: /*/exports Disallow: /*/cronlogs Disallow: /*/policies Disallow: /*/subscriptions Disallow: /*/categories # Business / pro consoles Disallow: /*/pro-desktop Disallow: /*/pro-contact-requests Disallow: /*/pro-quote-requests Disallow: /*/impact-of-events Disallow: /*/boost Disallow: /*/boost-select Disallow: /*/boost-success Disallow: /*/boosts # Embeddable widgets - the host page is what should be indexed, not the embed. # The widget routes have no locale segment (/widget/..., /widget-loader/...), so # the root forms are the ones that match. The locale forms are kept as well. Disallow: /widget/ Disallow: /widget-loader/ Disallow: /widget-preview/ Disallow: /*/widget/ Disallow: /*/widget-loader/ Disallow: /*/widget-preview/ # Widget-only bundles. Only the widget loader links these, never the app pages, # so this is safe for rendering. /* also covers the edge's /assets/... redirect # target. Never disallow /static as a whole - see the note at the top. Disallow: /*/widget.min. # Tracking / duplicate-URL parameters. These never appear in a sitemap. Disallow: /*?source_widget_id= Disallow: /*&source_widget_id= Disallow: /*?currentLocation= Disallow: /*¤tLocation= Sitemap: https://www.mikkelinyt.fi/sitemap.xml