evict basic-browser from crawlers

This commit is contained in:
ed 2023-03-09 21:35:07 +00:00
parent b39ff92f34
commit 9270c2df19

View file

@ -3249,7 +3249,7 @@ class HttpCli(object):
raise Pebkac(403)
self.html_head = vn.flags.get("html_head", "")
if vn.flags.get("norobots"):
if vn.flags.get("norobots") or "b" in self.uparam:
self.out_headers["X-Robots-Tag"] = "noindex, nofollow"
else:
self.out_headers.pop("X-Robots-Tag", None)