User-agent: Googlebot
Allow: /
# A share link is unlisted, not published. Users send one to a few people
# and reasonably expect it not to become a search result — and a crawler
# fetching one pulls the snapshot and every layer file behind it, unmetered.
Disallow: /s/
Disallow: /api/shared/

User-agent: Bingbot
Allow: /
Disallow: /s/
Disallow: /api/shared/

User-agent: Twitterbot
Allow: /
Disallow: /s/
Disallow: /api/shared/

User-agent: facebookexternalhit
Allow: /
Disallow: /s/
Disallow: /api/shared/

User-agent: *
Allow: /
Disallow: /s/
Disallow: /api/shared/