# ====================================================
# ROBOTS.TXT - SWARAJ PROPERTIES
# Domain: https://swarajproperties.in
# Generated: 2026-03-11
# ====================================================

# ----------------------
# ALL SEARCH ENGINES
# ----------------------
User-agent: *
Allow: /
Allow: /*.php$
Allow: /*.css$
Allow: /*.js$
Allow: /images/
Allow: /uploads/properties/
Allow: /partner-portal/

# Disallow sensitive directories
Disallow: /admin/
Disallow: /includes/
Disallow: /database/
Disallow: /config/
Disallow: /temp/
Disallow: /cache/
Disallow: /logs/
Disallow: /private/
Disallow: /vendor/
Disallow: /uploads/profiles/  # Private user data
Disallow: /uploads/temp/

# Disallow URLs with parameters
Disallow: /*?session=
Disallow: /*?logout=
Disallow: /*?action=delete
Disallow: /*?action=edit
Disallow: /*?debug=
Disallow: /*?token=
Disallow: /*?code=
Disallow: /*?redirect=*

# Crawl delay
Crawl-delay: 5

# ----------------------
# GOOGLE BOT
# ----------------------
User-agent: Googlebot
Allow: /
Allow: /*.php$
Allow: /uploads/properties/
Disallow: /admin/
Disallow: /includes/
Disallow: /uploads/profiles/

# Google Image Bot
User-agent: Googlebot-Image
Allow: /uploads/properties/
Allow: /images/
Disallow: /uploads/profiles/

# Google Ads Bot
User-agent: AdsBot-Google
Allow: /
Disallow: /admin/

# ----------------------
# BING BOT
# ----------------------
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /includes/
Crawl-delay: 2

# ----------------------
# YAHOO BOT
# ----------------------
User-agent: Yahoo! Slurp
Allow: /
Disallow: /admin/
Crawl-delay: 2

# ----------------------
# YANDEX BOT (Russia)
# ----------------------
User-agent: Yandex
Allow: /
Disallow: /admin/
Crawl-delay: 5

# ----------------------
# BAIDU BOT (China)
# ----------------------
User-agent: Baiduspider
Allow: /
Disallow: /admin/
Crawl-delay: 10

# ----------------------
# DUCKDUCKGO BOT
# ----------------------
User-agent: DuckDuckBot
Allow: /
Disallow: /admin/
Crawl-delay: 3

# ----------------------
# SPECIAL BOTS
# ----------------------
User-agent: ia_archiver  # Alexa/Archive.org
Disallow: /

User-agent: MJ12bot      # Majestic SEO
Crawl-delay: 10
Allow: /

User-agent: AhrefsBot    # Ahrefs
Crawl-delay: 10
Allow: /

User-agent: SemrushBot   # Semrush
Crawl-delay: 10
Allow: /

User-agent: DotBot       # DotBot
Disallow: /

# ----------------------
# SITEMAP LOCATIONS
# ----------------------
Sitemap: https://swarajproperties.in/sitemap.xml

# ----------------------
# HOST
# ----------------------
Host: https://swarajproperties.in

# ----------------------
# CLEAN URLS
# ----------------------
User-agent: *
Allow: /*.php$
Allow: /*.html$
Allow: /*.htm$

# ----------------------
# END OF ROBOTS.TXT
# ----------------------