# robots.txt for https://shorted.com.au
# Official ASIC short position data for ASX stocks — we WANT this content
# discoverable and usable by search engines and AI systems (see /ai.txt).

User-Agent: *
Allow: /
Allow: /api/mcp/
Disallow: /admin/
Disallow: /api/
Disallow: /embed/
Disallow: /subscribe/
Disallow: /alerts/
Disallow: /portfolio/
Disallow: /chat/
Disallow: /metrics/
Disallow: /dashboards/
Disallow: /shorts.v1alpha1.
Disallow: /marketdata.v1.
Disallow: /chat.v1.
Disallow: /register.v1.

# Content Signals (https://contentsignals.org/) — consistent with /ai.txt
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# AI crawlers — explicitly welcome
User-Agent: GPTBot
User-Agent: ChatGPT-User
User-Agent: Google-Extended
User-Agent: PerplexityBot
User-Agent: ClaudeBot
User-Agent: anthropic-ai
User-Agent: CCBot
Allow: /
Allow: /shorts/
Allow: /top
Allow: /battlegrounds
Allow: /statistics
Allow: /scans
Allow: /scans/
Allow: /industry/
Allow: /reports/
Allow: /market/
Allow: /market
Allow: /screener
Allow: /seasonality
Allow: /compare/
Allow: /directory/
Allow: /directory
Allow: /housing/
Allow: /housing
Allow: /features/
Allow: /glossary/
Allow: /learn/
Allow: /faq
Allow: /blog/
Allow: /docs/
Allow: /openapi.json
Allow: /openapi.yaml
Allow: /.well-known/
Allow: /news/
Allow: /news
Allow: /insider-trading/
Allow: /insider-trading
Allow: /data
Allow: /search
Allow: /authors/
Allow: /authors
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai.txt
Allow: /api/mcp/
Disallow: /admin/
Disallow: /api/
Disallow: /embed/
Disallow: /subscribe/
Disallow: /alerts/
Disallow: /portfolio/
Disallow: /chat/
Disallow: /metrics/
Disallow: /dashboards/
Disallow: /shorts.v1alpha1.
Disallow: /marketdata.v1.
Disallow: /chat.v1.
Disallow: /register.v1.

Host: https://shorted.com.au
# /sitemap.xml is a sitemapindex; the children are listed too so a crawler that
# does not follow index files still finds every section.
Sitemap: https://shorted.com.au/sitemap.xml
Sitemap: https://shorted.com.au/sitemap-core.xml
Sitemap: https://shorted.com.au/sitemap-shorts.xml
Sitemap: https://shorted.com.au/sitemap-reports.xml
Sitemap: https://shorted.com.au/sitemap-housing.xml
Sitemap: https://shorted.com.au/sitemap-politicians.xml