evict basic-browser from crawlers
This commit is contained in:
parent
b39ff92f34
commit
9270c2df19
@ -3249,7 +3249,7 @@ class HttpCli(object):
|
|||||||
raise Pebkac(403)
|
raise Pebkac(403)
|
||||||
|
|
||||||
self.html_head = vn.flags.get("html_head", "")
|
self.html_head = vn.flags.get("html_head", "")
|
||||||
if vn.flags.get("norobots"):
|
if vn.flags.get("norobots") or "b" in self.uparam:
|
||||||
self.out_headers["X-Robots-Tag"] = "noindex, nofollow"
|
self.out_headers["X-Robots-Tag"] = "noindex, nofollow"
|
||||||
else:
|
else:
|
||||||
self.out_headers.pop("X-Robots-Tag", None)
|
self.out_headers.pop("X-Robots-Tag", None)
|
||||||
|
Loading…
Reference in New Issue
Block a user