HTTPS surface reachable (robots ✓, sitemap ✓, title ✓)
Why it matters: Public files — robots.txt, sitemap.xml, head meta — are what attackers see first during reconnaissance. Misadvertised paths, stale sitemaps, and verbose generators leak more than intended (ISO 27001 A.8.9).
robots.txt
present
User-agent: *
Crawl-delay: 5
Allow: /corporate/microsites/tysabri/ms-assessment-tool.html
Allow: /corporate/microsites/tysabri/ms-assessment-tool-page1.html
Allow: /corporate/microsites/tysabri/ms-assessment-tool-page2.html
Allow: /corporate/microsites/tysabri/ms-assessment-tool-page3.html
Allow: /corporate/microsites/tysabri/ms-assessment-tool-page4.html
Allow: /corporate/microsites/tysabri/ms-assessment-tool-page5.html
Allow: /corporate/microsites/tysabri/ms-assessment-tool-page6.html
Allow: /corporate/microsites/tysabri/ms-assessment-tool-page7.html
Allow: /corporate/microsites/tysabri/ms-assessment-tool-pageresults.html
Allow: /ads.txt
Disallow: /bmajax/
Disallow: /v2/
Disallow: /vp/
Disallow: /corporate/
Disallow: /health/mirgraine-headaches
Disallow: /health/wp-*
Disallow: /nutrition/wp-*
Disallow: /program/wp-*
Disallow: /health-news/wp-*
Disallow: /health/*/wp-*
Disallow: /nutrition/*/wp-*
Disallow: /health-news/*/wp-*
Disallow: /cdn.jwplayer.com/previews/*
# Internal Frontend Test Pages - Not for Public Indexing
Disallow: /health/sponsored-article-test-do-not-edit-this-ever
Disallow: /program/hms-test-program-page
Disallow: /health/QA-Tabbed-ManyTabs-21
Disallow: /health/post-block-widget-sampler
Disallow: /health/test-reference-article-for-front-end-tests
Disallow: /health/wp-2485828
Disallow: /health/this-is-an-infinite-scroll-test-page-please-do-not-modify-this-page-without-checking-with-frontend-site-teams
Disallow: /health/hm-test-17-proven-tips-to-sleep-better-at-night
Disallow: /healthy/hm-test-page-2-17-proven-tips-to-sleep-better-at-night
Disallow: /healthy/hm-test-page-3-17-proven-tips-to-sleep-better-at-night
Disallow: /healthy/hm-test-page-4-diabetes-and-sweets
Disallow: /healthy/hm-test-page-5-diabetes-and-sweets
Disallow: /health/drugs/hm-test-all-about-humira
Disallow: /health/hm-test-feature-template-for-sponsorship
Disallow: /this-is-a-test-page
# Invalid URLs
Disallow: */null$
Disallow: */inline$
# Static Test Articles
Disallow: /test/
# Media Revenue Test Pages
Disallow: /*/hm-test
# API Routes
Disallow: /api/
Allow: /api/auth/signup
Disallow: /hlcmsresource/*.txt$
Disallow: /hlcmsresource/*.xml$
User-agent: Mediapartners-Google
Disallow:
User-agent: Adsbot-Google
Disallow:
# --- Amazon / AWS crawlers (begin) ---
# AmazonAdbot is intentionally allowed (ad targeting — see HLMR-6504)
User-agent: AmazonAdbot
Disallow:
User-agent: Amazonbot
Disallow: /
User-agent: Amzn-SearchBot
Disallow: /
User-agent: Amzn-User
Disallow: /
User-agent: amazon-kendra
Disallow: /
User-agent: amazon-QBusiness
Disallow: /
User-agent: bedrockbot
Disallow: /
User-agent: aws-quick-on-behalf-of-
Disallow: /
User-agent: AmazonProductDiscoverybot
Disallow: /
User-agent: AmazonBuyForMe
Disallow: /
User-agent: AmazonSellerInitiatedListing
Disallow: /
User-agent: NovaAct
Disallow: /
# --- Amazon / AWS crawlers (end) ---
User-agent: Twitterbot
Disallow:
User-agent: NimbleCrawler
Disallow: /
User-agent: grapeshot
Disallow:
User-agent: BotRightHere
Disallow: /
User-agent: WebZip
Disallow: /
User-agent: larbin
Disallow: /
User-agent: b2w/0.1
Disallow: /
User-agent: Copernic
Disallow: /
User-agent: psbot
Disallow: /
User-agent: Python-urllib
Disallow: /
User-agent: URL_Spider_Pro
Disallow: /
User-agent: CherryPicker
Disallow: /
User-agent: EmailCollector
Disallow: /
User-agent: EmailSiphon
Disallow: /
User-agent: WebBandit
Disallow: /
User-agent: EmailWolf
Disallow: /
User-agent: ExtractorPro
Disallow: /
User-agent: CopyRightCheck
Disallow: /
User-agent: Crescent
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: ProWebWalker
Disallow: /
User-agent: CheeseBot
Disallow: /
User-agent: LNSpiderguy
Disallow: /
User-agent: Alexibot
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: MIIxpc
Disallow: /
User-agent: Telesoft
Disallow: /
User-agent: Website Quester
Disallow: /
User-agent: WebZip
Disallow: /
User-agent: moget/2.1
Disallow: /
User-agent: WebZip/4.0
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebS
sitemap.xml
present — 5 url(s)
head
- title
- Healthline: Medical information and health advice you can trust.
- description
- We're committed to being your source for expert health guidance. Come to us in your pursuit of wellness.
social
- og:title
- Healthline: Medical information and health advice you can trust.
- og:description
- We're committed to being your source for expert health guidance. Come to us in your pursuit of wellness.
- og:type
- article
- og:url
- https://www.healthline.com
- og:site_name
- Healthline
- og:image
- https://assets.healthline.com/content/hl_sharing_v2.jpg
- twitter:title
- Healthline: Medical information and health advice you can trust.
- twitter:description
- We're committed to being your source for expert health guidance. Come to us in your pursuit of wellness.
- twitter:card
- summary_large_image
- twitter:site
- @healthline
- twitter:creator
- @healthline
- twitter:image
- https://assets.healthline.com/content/hl_sharing_v2.jpg