add option to request noindex from crawlers

This commit is contained in:
ed 2022-03-20 03:23:42 +01:00
parent 67ff57f3a3
commit 954c7e7e50
9 changed files with 15 additions and 0 deletions

View file

@ -472,6 +472,7 @@ def run_argparse(argv, formatter):
ap2.add_argument("--no-readme", action="store_true", help="disable rendering readme.md into directory listings")
ap2.add_argument("--vague-403", action="store_true", help="send 404 instead of 403 (security through ambiguity, very enterprise)")
ap2.add_argument("--force-js", action="store_true", help="don't send HTML folder listings, force clients to use the embedded json instead")
ap2.add_argument("--no-robots", action="store_true", help="adds http and html headers asking search engines to not index anything")
ap2 = ap.add_argument_group('yolo options')
ap2.add_argument("--ign-ebind", action="store_true", help="continue running even if it's impossible to listen on some of the requested endpoints")
@ -540,6 +541,7 @@ def run_argparse(argv, formatter):
ap2 = ap.add_argument_group('ui options')
ap2.add_argument("--js-browser", metavar="L", type=u, help="URL to additional JS to include")
ap2.add_argument("--css-browser", metavar="L", type=u, help="URL to additional CSS to include")
ap2.add_argument("--html-head", metavar="TXT", type=u, default="", help="text to append to the <head> of all HTML pages")
ap2.add_argument("--textfiles", metavar="CSV", type=u, default="txt,nfo,diz,cue,readme", help="file extensions to present as plaintext")
ap2.add_argument("--doctitle", metavar="TXT", type=u, default="copyparty", help="title / service-name to show in html documents")

View file

@ -65,6 +65,8 @@ class HttpCli(object):
"Access-Control-Allow-Origin": "*",
"Cache-Control": "no-store; max-age=0",
}
if self.args.no_robots:
self.out_headers["X-Robots-Tag"] = "noindex, nofollow"
def log(self, msg, c=0):
ptn = self.asrv.re_pwd
@ -93,6 +95,7 @@ class HttpCli(object):
if ka:
ka["ts"] = self.conn.hsrv.cachebuster()
ka["svcname"] = self.args.doctitle
ka["html_head"] = self.args.html_head
return tpl.render(**ka)
return tpl

View file

@ -70,6 +70,10 @@ class SvcHub(object):
self.log("root", m, c=3)
if args.no_robots:
h = ['<meta name="robots" content="noindex, nofollow">', args.html_head]
args.html_head = "\n\t".join([x for x in h if x])
# initiate all services to manage
self.asrv = AuthSrv(self.args, self.log)
if args.ls:

View file

@ -6,6 +6,7 @@
<title>⇆🎉 {{ title }}</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=0.8">
{{ html_head }}
<link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
<link rel="stylesheet" media="screen" href="/.cpr/browser.css?_={{ ts }}">
{%- if css %}

View file

@ -6,6 +6,7 @@
<title>{{ title }}</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=0.8">
{{ html_head }}
<style>
html{font-family:sans-serif}
td{border:1px solid #999;border-width:1px 1px 0 0;padding:0 5px}

View file

@ -3,6 +3,7 @@
<title>📝🎉 {{ title }}</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=0.7">
{{ html_head }}
<link rel="stylesheet" href="/.cpr/ui.css?_={{ ts }}">
<link rel="stylesheet" href="/.cpr/md.css?_={{ ts }}">
{%- if edit %}

View file

@ -3,6 +3,7 @@
<title>📝🎉 {{ title }}</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=0.7">
{{ html_head }}
<link rel="stylesheet" href="/.cpr/ui.css?_={{ ts }}">
<link rel="stylesheet" href="/.cpr/mde.css?_={{ ts }}">
<link rel="stylesheet" href="/.cpr/deps/mini-fa.css?_={{ ts }}">

View file

@ -6,6 +6,7 @@
<title>{{ svcname }}</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=0.8">
{{ html_head }}
<link rel="stylesheet" media="screen" href="/.cpr/msg.css?_={{ ts }}">
</head>

View file

@ -6,6 +6,7 @@
<title>{{ svcname }}</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=0.8">
{{ html_head }}
<link rel="stylesheet" media="screen" href="/.cpr/splash.css?_={{ ts }}">
<link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
</head>