# Allow all search engines
User-agent: *
Allow: /

# Disallow admin/backend areas
Disallow: /thewall/
Disallow: /admin/
Disallow: /backend/
Disallow: /Handler/
Disallow: /webservices/
Disallow: /App_Data/
Disallow: /bin/
Disallow: /obj/
Disallow: /packages/
Disallow: /uploadImages/temp/
Disallow: /StyleGuide.aspx
Disallow: /WebForm1.aspx
Disallow: /OldLinks.aspx
Disallow: /controls/


# Disallow query parameters that create duplicate content
Disallow: /*?*utm_source=*
Disallow: /*?*utm_medium=*
Disallow: /*?*utm_campaign=*
Disallow: /*?*LoadIndex=*
Disallow: /*?*preview=*

# Allow important paths
Allow: /news/
Allow: /article/
Allow: /category/
Allow: /images/
Allow: /Style/
Allow: /JS/
Allow: /YoutubePlayer/
Allow: /ads.txt


# Sitemap location (will be created next)
Sitemap: https://www.lebanon24.com/sitemap.xml
Sitemap: https://www.lebanon24.com/newssitemap.xml
Sitemap: https://www.lebanon24.com/imagessitemap.xml
Sitemap: https://www.lebanon24.com/videossitemap.xml

Sitemap: https://www.lebanon24.com/sitemaps/sitemap-495001-500000.xml
Sitemap: https://www.lebanon24.com/sitemaps/newssitemap-495001-500000.xml

Sitemap: https://www.lebanon24.com/sitemaps/sitemap-490001-495000.xml
Sitemap: https://www.lebanon24.com/sitemaps/newssitemap-490001-495000.xml

Sitemap: https://www.lebanon24.com/sitemaps/sitemap-485001-490000.xml
Sitemap: https://www.lebanon24.com/sitemaps/newssitemap-485001-490000.xml

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-News
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

User-agent: facebookexternalhit
Allow: /
Crawl-delay: 0
   
User-agent: Facebot
Allow: /
Crawl-delay: 0

# Twitter crawler (for Twitter Card image scraping)
User-agent: Twitterbot
Allow: /
Crawl-delay: 0

# Block CDN domain from indexing
User-agent: *
Disallow: https://cdn.lebanon24.com/