# robots.txt - Nordic Custom Made
# Alle må crawle, undtagen de bots der er blokeret nedenfor.
 
User-agent: *
Allow: /
 
# --- SEO-værktøjer og tunge crawlere ---
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: SemrushBot-BA
User-agent: SemrushBot-SI
User-agent: SemrushBot-SWA
User-agent: SemrushBot-CT
User-agent: SemrushBot-BM
User-agent: SemrushBot-OCOB
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: Barkrowler
User-agent: serpstatbot
User-agent: Awario
User-agent: Superfeedr
User-agent: YandexBot
User-agent: YandexAdditional
User-agent: YandexAdditionalBot
User-agent: PetalBot
User-agent: Bytespider
User-agent: TikTokSpider
User-agent: netEstate Imprint Crawler
User-agent: Datenbank Crawler
User-agent: ISSCyberRiskCrawler
User-agent: Factset_spyderbot
User-agent: Sidetrade indexer bot
User-agent: Scrapy
Disallow: /
 
# --- AI-crawlere og scrapere (træning, datasæt, dataindsamling) ---
User-agent: AddSearchBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: aiHitBot
User-agent: Andibot
User-agent: Anomura
User-agent: atlassian-bot
User-agent: bigsur.ai
User-agent: Brightbot
User-agent: BuddyBot
User-agent: CCBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Cotoyogi
User-agent: Crawlspace
User-agent: DeepSeekBot
User-agent: Devin
User-agent: Diffbot
User-agent: Echobot Bot
User-agent: EchoboxBot
User-agent: FirecrawlAgent
User-agent: FriendlyCrawler
User-agent: iaskspider
User-agent: IbouBot
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: Kangaroo Bot
User-agent: KlaviyoAIBot
User-agent: LinerBot
User-agent: Linguee Bot
User-agent: MyCentralAIScraperBot
User-agent: omgili
User-agent: omgilibot
User-agent: PanguBot
User-agent: Panscient
User-agent: panscient.com
User-agent: PhindBot
User-agent: Poseidon Research Crawler
User-agent: QualifiedBot
User-agent: QuillBot
User-agent: quillbot.com
User-agent: SBIntuitionsBot
User-agent: ShapBot
User-agent: TerraCotta
User-agent: Thinkbot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: WARDBot
User-agent: Webzio-Extended
User-agent: wpbot
User-agent: YaK
User-agent: YouBot
Disallow: /

# Added by ITO 270524
User-agent: *
Disallow : *?wordfence_lh*
Disallow : */feed/*
Disallow : */vare/*

User-Agent: *
Crawl-Delay: 10
Allow: /wp-content/uploads/
Disallow: /wp-content/plugins/
Disallow: /wp-admin/
Disallow: /wp-includes/
Allow: /wp-admin/admin-ajax.php
Disallow: /cart/
Disallow: /kurv/
Disallow: /checkout/
Disallow: /kasse/
Disallow: /my-account/
Disallow: /min-konto/
Disallow: /*add-to-cart=*

Sitemap: https://bedrebegravelse.dk/sitemap_index.xml