Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
ff8313d0fb |
40
.github/ISSUE_TEMPLATE/bug_report.md
vendored
40
.github/ISSUE_TEMPLATE/bug_report.md
vendored
@@ -1,40 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: bug
|
||||
assignees: '9001'
|
||||
|
||||
---
|
||||
|
||||
NOTE:
|
||||
all of the below are optional, consider them as inspiration, delete and rewrite at will, thx md
|
||||
|
||||
|
||||
**Describe the bug**
|
||||
a description of what the bug is
|
||||
|
||||
**To Reproduce**
|
||||
List of steps to reproduce the issue, or, if it's hard to reproduce, then at least a detailed explanation of what you did to run into it
|
||||
|
||||
**Expected behavior**
|
||||
a description of what you expected to happen
|
||||
|
||||
**Screenshots**
|
||||
if applicable, add screenshots to help explain your problem, such as the kickass crashpage :^)
|
||||
|
||||
**Server details**
|
||||
if the issue is possibly on the server-side, then mention some of the following:
|
||||
* server OS / version:
|
||||
* python version:
|
||||
* copyparty arguments:
|
||||
* filesystem (`lsblk -f` on linux):
|
||||
|
||||
**Client details**
|
||||
if the issue is possibly on the client-side, then mention some of the following:
|
||||
* the device type and model:
|
||||
* OS version:
|
||||
* browser version:
|
||||
|
||||
**Additional context**
|
||||
any other context about the problem here
|
||||
22
.github/ISSUE_TEMPLATE/feature_request.md
vendored
22
.github/ISSUE_TEMPLATE/feature_request.md
vendored
@@ -1,22 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: enhancement
|
||||
assignees: '9001'
|
||||
|
||||
---
|
||||
|
||||
all of the below are optional, consider them as inspiration, delete and rewrite at will
|
||||
|
||||
**is your feature request related to a problem? Please describe.**
|
||||
a description of what the problem is, for example, `I'm always frustrated when [...]` or `Why is it not possible to [...]`
|
||||
|
||||
**Describe the idea / solution you'd like**
|
||||
a description of what you want to happen
|
||||
|
||||
**Describe any alternatives you've considered**
|
||||
a description of any alternative solutions or features you've considered
|
||||
|
||||
**Additional context**
|
||||
add any other context or screenshots about the feature request here
|
||||
10
.github/ISSUE_TEMPLATE/something-else.md
vendored
10
.github/ISSUE_TEMPLATE/something-else.md
vendored
@@ -1,10 +0,0 @@
|
||||
---
|
||||
name: Something else
|
||||
about: "┐(ﾟ∀ﾟ)┌"
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
|
||||
7
.github/branch-rename.md
vendored
7
.github/branch-rename.md
vendored
@@ -1,7 +0,0 @@
|
||||
modernize your local checkout of the repo like so,
|
||||
```sh
|
||||
git branch -m master hovudstraum
|
||||
git fetch origin
|
||||
git branch -u origin/hovudstraum hovudstraum
|
||||
git remote set-head origin -a
|
||||
```
|
||||
16
.gitignore
vendored
16
.gitignore
vendored
@@ -5,16 +5,12 @@ __pycache__/
|
||||
MANIFEST.in
|
||||
MANIFEST
|
||||
copyparty.egg-info/
|
||||
buildenv/
|
||||
build/
|
||||
dist/
|
||||
sfx/
|
||||
.venv/
|
||||
|
||||
/buildenv/
|
||||
/build/
|
||||
/dist/
|
||||
/py2/
|
||||
/sfx*
|
||||
/unt/
|
||||
/log/
|
||||
|
||||
# ide
|
||||
*.sublime-workspace
|
||||
|
||||
@@ -24,7 +20,3 @@ copyparty.egg-info/
|
||||
# derived
|
||||
copyparty/web/deps/
|
||||
srv/
|
||||
|
||||
# state/logs
|
||||
up.*.txt
|
||||
.hist/
|
||||
2
.vscode/launch.json
vendored
2
.vscode/launch.json
vendored
@@ -17,7 +17,7 @@
|
||||
"-mtp",
|
||||
".bpm=f,bin/mtag/audio-bpm.py",
|
||||
"-aed:wark",
|
||||
"-vsrv::r:rw,ed:c,dupe",
|
||||
"-vsrv::r:aed:cnodupe",
|
||||
"-vdist:dist:r"
|
||||
]
|
||||
},
|
||||
|
||||
26
.vscode/settings.json
vendored
26
.vscode/settings.json
vendored
@@ -23,6 +23,7 @@
|
||||
"terminal.ansiBrightWhite": "#ffffff",
|
||||
},
|
||||
"python.testing.pytestEnabled": false,
|
||||
"python.testing.nosetestsEnabled": false,
|
||||
"python.testing.unittestEnabled": true,
|
||||
"python.testing.unittestArgs": [
|
||||
"-v",
|
||||
@@ -34,40 +35,18 @@
|
||||
"python.linting.pylintEnabled": true,
|
||||
"python.linting.flake8Enabled": true,
|
||||
"python.linting.banditEnabled": true,
|
||||
"python.linting.mypyEnabled": true,
|
||||
"python.linting.mypyArgs": [
|
||||
"--ignore-missing-imports",
|
||||
"--follow-imports=silent",
|
||||
"--show-column-numbers",
|
||||
"--strict"
|
||||
],
|
||||
"python.linting.flake8Args": [
|
||||
"--max-line-length=120",
|
||||
"--ignore=E722,F405,E203,W503,W293,E402,E501,E128",
|
||||
"--ignore=E722,F405,E203,W503,W293,E402",
|
||||
],
|
||||
"python.linting.banditArgs": [
|
||||
"--ignore=B104"
|
||||
],
|
||||
"python.linting.pylintArgs": [
|
||||
"--disable=missing-module-docstring",
|
||||
"--disable=missing-class-docstring",
|
||||
"--disable=missing-function-docstring",
|
||||
"--disable=wrong-import-position",
|
||||
"--disable=raise-missing-from",
|
||||
"--disable=bare-except",
|
||||
"--disable=invalid-name",
|
||||
"--disable=line-too-long",
|
||||
"--disable=consider-using-f-string"
|
||||
],
|
||||
// python3 -m isort --py=27 --profile=black copyparty/
|
||||
"python.formatting.provider": "black",
|
||||
"editor.formatOnSave": true,
|
||||
"[html]": {
|
||||
"editor.formatOnSave": false,
|
||||
},
|
||||
"[css]": {
|
||||
"editor.formatOnSave": false,
|
||||
},
|
||||
"files.associations": {
|
||||
"*.makefile": "makefile"
|
||||
},
|
||||
@@ -76,5 +55,4 @@
|
||||
"py27"
|
||||
],
|
||||
"python.linting.enabled": true,
|
||||
"python.pythonPath": "/usr/bin/python3"
|
||||
}
|
||||
5
.vscode/tasks.json
vendored
5
.vscode/tasks.json
vendored
@@ -9,10 +9,7 @@
|
||||
{
|
||||
"label": "no_dbg",
|
||||
"type": "shell",
|
||||
"command": "${config:python.pythonPath}",
|
||||
"args": [
|
||||
".vscode/launch.py"
|
||||
]
|
||||
"command": "${config:python.pythonPath} .vscode/launch.py"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
in the words of Abraham Lincoln:
|
||||
|
||||
> Be excellent to each other... and... PARTY ON, DUDES!
|
||||
|
||||
more specifically I'll paraphrase some examples from a german automotive corporation as they cover all the bases without being too wordy
|
||||
|
||||
## Examples of unacceptable behavior
|
||||
* intimidation, harassment, trolling
|
||||
* insulting, derogatory, harmful or prejudicial comments
|
||||
* posting private information without permission
|
||||
* political or personal attacks
|
||||
|
||||
## Examples of expected behavior
|
||||
* being nice, friendly, welcoming, inclusive, mindful and empathetic
|
||||
* acting considerate, modest, respectful
|
||||
* using polite and inclusive language
|
||||
* criticize constructively and accept constructive criticism
|
||||
* respect different points of view
|
||||
|
||||
## finally and even more specifically,
|
||||
* parse opinions and feedback objectively without prejudice
|
||||
* it's the message that matters, not who said it
|
||||
|
||||
aaand that's how you say `be nice` in a way that fills half a floppy w
|
||||
@@ -1,3 +0,0 @@
|
||||
* do something cool
|
||||
|
||||
really tho, send a PR or an issue or whatever, all appreciated, anything goes, just behave aight
|
||||
@@ -1,16 +1,3 @@
|
||||
# [`up2k.py`](up2k.py)
|
||||
* command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
|
||||
* file uploads, file-search, autoresume of aborted/broken uploads
|
||||
* faster than browsers
|
||||
* if something breaks just restart it
|
||||
|
||||
|
||||
# [`partyjournal.py`](partyjournal.py)
|
||||
produces a chronological list of all uploads by collecting info from up2k databases and the filesystem
|
||||
* outputs a standalone html file
|
||||
* optional mapping from IP-addresses to nicknames
|
||||
|
||||
|
||||
# [`copyparty-fuse.py`](copyparty-fuse.py)
|
||||
* mount a copyparty server as a local filesystem (read-only)
|
||||
* **supports Windows!** -- expect `194 MiB/s` sequential read
|
||||
@@ -60,7 +47,6 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
|
||||
* copyparty can Popen programs like these during file indexing to collect additional metadata
|
||||
|
||||
|
||||
|
||||
# [`dbtool.py`](dbtool.py)
|
||||
upgrade utility which can show db info and help transfer data between databases, for example when a new version of copyparty is incompatible with the old DB and automatically rebuilds the DB from scratch, but you have some really expensive `-mtp` parsers and want to copy over the tags from the old db
|
||||
|
||||
@@ -75,9 +61,3 @@ cd /mnt/nas/music/.hist
|
||||
~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -rm-mtp-flag -copy key
|
||||
~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -rm-mtp-flag -copy .bpm -vac
|
||||
```
|
||||
|
||||
|
||||
|
||||
# [`prisonparty.sh`](prisonparty.sh)
|
||||
* run copyparty in a chroot, preventing any accidental file access
|
||||
* creates bindmounts for /bin, /lib, and so on, see `sysdirs=`
|
||||
|
||||
@@ -42,7 +42,6 @@ import threading
|
||||
import traceback
|
||||
import http.client # py2: httplib
|
||||
import urllib.parse
|
||||
import calendar
|
||||
from datetime import datetime
|
||||
from urllib.parse import quote_from_bytes as quote
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
@@ -346,7 +345,7 @@ class Gateway(object):
|
||||
except:
|
||||
pass
|
||||
|
||||
def sendreq(self, meth, path, headers, **kwargs):
|
||||
def sendreq(self, *args, headers={}, **kwargs):
|
||||
if self.password:
|
||||
headers["Cookie"] = "=".join(["cppwd", self.password])
|
||||
|
||||
@@ -355,21 +354,21 @@ class Gateway(object):
|
||||
if c.rx_path:
|
||||
raise Exception()
|
||||
|
||||
c.request(meth, path, headers=headers, **kwargs)
|
||||
c.request(*list(args), headers=headers, **kwargs)
|
||||
c.rx = c.getresponse()
|
||||
return c
|
||||
except:
|
||||
tid = threading.current_thread().ident
|
||||
dbg(
|
||||
"\033[1;37;44mbad conn {:x}\n {} {}\n {}\033[0m".format(
|
||||
tid, meth, path, c.rx_path if c else "(null)"
|
||||
"\033[1;37;44mbad conn {:x}\n {}\n {}\033[0m".format(
|
||||
tid, " ".join(str(x) for x in args), c.rx_path if c else "(null)"
|
||||
)
|
||||
)
|
||||
|
||||
self.closeconn(c)
|
||||
c = self.getconn()
|
||||
try:
|
||||
c.request(meth, path, headers=headers, **kwargs)
|
||||
c.request(*list(args), headers=headers, **kwargs)
|
||||
c.rx = c.getresponse()
|
||||
return c
|
||||
except:
|
||||
@@ -387,7 +386,7 @@ class Gateway(object):
|
||||
path = dewin(path)
|
||||
|
||||
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
|
||||
c = self.sendreq("GET", web_path, {})
|
||||
c = self.sendreq("GET", web_path)
|
||||
if c.rx.status != 200:
|
||||
self.closeconn(c)
|
||||
log(
|
||||
@@ -441,7 +440,7 @@ class Gateway(object):
|
||||
)
|
||||
)
|
||||
|
||||
c = self.sendreq("GET", web_path, {"Range": hdr_range})
|
||||
c = self.sendreq("GET", web_path, headers={"Range": hdr_range})
|
||||
if c.rx.status != http.client.PARTIAL_CONTENT:
|
||||
self.closeconn(c)
|
||||
raise Exception(
|
||||
@@ -496,7 +495,7 @@ class Gateway(object):
|
||||
ts = 60 * 60 * 24 * 2
|
||||
try:
|
||||
sz = int(fsize)
|
||||
ts = calendar.timegm(time.strptime(fdate, "%Y-%m-%d %H:%M:%S"))
|
||||
ts = datetime.strptime(fdate, "%Y-%m-%d %H:%M:%S").timestamp()
|
||||
except:
|
||||
info("bad HTML or OS [{}] [{}]".format(fdate, fsize))
|
||||
# python cannot strptime(1959-01-01) on windows
|
||||
|
||||
@@ -22,7 +22,7 @@ dependencies:
|
||||
|
||||
note:
|
||||
you probably want to run this on windows clients:
|
||||
https://github.com/9001/copyparty/blob/hovudstraum/contrib/explorer-nothumbs-nofoldertypes.reg
|
||||
https://github.com/9001/copyparty/blob/master/contrib/explorer-nothumbs-nofoldertypes.reg
|
||||
|
||||
get server cert:
|
||||
awk '/-BEGIN CERTIFICATE-/ {a=1} a; /-END CERTIFICATE-/{exit}' <(openssl s_client -connect 127.0.0.1:3923 </dev/null 2>/dev/null) >cert.pem
|
||||
@@ -45,7 +45,6 @@ import threading
|
||||
import traceback
|
||||
import http.client # py2: httplib
|
||||
import urllib.parse
|
||||
import calendar
|
||||
from datetime import datetime
|
||||
from urllib.parse import quote_from_bytes as quote
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
@@ -55,13 +54,10 @@ MACOS = platform.system() == "Darwin"
|
||||
info = log = dbg = None
|
||||
|
||||
|
||||
print(
|
||||
"{} v{} @ {}".format(
|
||||
print("{} v{} @ {}".format(
|
||||
platform.python_implementation(),
|
||||
".".join([str(x) for x in sys.version_info]),
|
||||
sys.executable,
|
||||
)
|
||||
)
|
||||
sys.executable))
|
||||
|
||||
|
||||
try:
|
||||
@@ -72,7 +68,7 @@ except:
|
||||
elif MACOS:
|
||||
libfuse = "install https://osxfuse.github.io/"
|
||||
else:
|
||||
libfuse = "apt install libfuse3-3\n modprobe fuse"
|
||||
libfuse = "apt install libfuse\n modprobe fuse"
|
||||
|
||||
print(
|
||||
"\n could not import fuse; these may help:"
|
||||
@@ -303,14 +299,14 @@ class Gateway(object):
|
||||
except:
|
||||
pass
|
||||
|
||||
def sendreq(self, meth, path, headers, **kwargs):
|
||||
def sendreq(self, *args, headers={}, **kwargs):
|
||||
tid = get_tid()
|
||||
if self.password:
|
||||
headers["Cookie"] = "=".join(["cppwd", self.password])
|
||||
|
||||
try:
|
||||
c = self.getconn(tid)
|
||||
c.request(meth, path, headers=headers, **kwargs)
|
||||
c.request(*list(args), headers=headers, **kwargs)
|
||||
return c.getresponse()
|
||||
except:
|
||||
dbg("bad conn")
|
||||
@@ -318,7 +314,7 @@ class Gateway(object):
|
||||
self.closeconn(tid)
|
||||
try:
|
||||
c = self.getconn(tid)
|
||||
c.request(meth, path, headers=headers, **kwargs)
|
||||
c.request(*list(args), headers=headers, **kwargs)
|
||||
return c.getresponse()
|
||||
except:
|
||||
info("http connection failed:\n" + traceback.format_exc())
|
||||
@@ -335,7 +331,7 @@ class Gateway(object):
|
||||
path = dewin(path)
|
||||
|
||||
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
|
||||
r = self.sendreq("GET", web_path, {})
|
||||
r = self.sendreq("GET", web_path)
|
||||
if r.status != 200:
|
||||
self.closeconn()
|
||||
log(
|
||||
@@ -372,7 +368,7 @@ class Gateway(object):
|
||||
)
|
||||
)
|
||||
|
||||
r = self.sendreq("GET", web_path, {"Range": hdr_range})
|
||||
r = self.sendreq("GET", web_path, headers={"Range": hdr_range})
|
||||
if r.status != http.client.PARTIAL_CONTENT:
|
||||
self.closeconn()
|
||||
raise Exception(
|
||||
@@ -394,16 +390,15 @@ class Gateway(object):
|
||||
|
||||
rsp = json.loads(rsp.decode("utf-8"))
|
||||
ret = []
|
||||
for statfun, nodes in [
|
||||
[self.stat_dir, rsp["dirs"]],
|
||||
[self.stat_file, rsp["files"]],
|
||||
]:
|
||||
for is_dir, nodes in [[True, rsp["dirs"]], [False, rsp["files"]]]:
|
||||
for n in nodes:
|
||||
fname = unquote(n["href"].split("?")[0]).rstrip(b"/").decode("wtf-8")
|
||||
fname = unquote(n["href"]).rstrip(b"/")
|
||||
fname = fname.decode("wtf-8")
|
||||
if bad_good:
|
||||
fname = enwin(fname)
|
||||
|
||||
ret.append([fname, statfun(n["ts"], n["sz"]), 0])
|
||||
fun = self.stat_dir if is_dir else self.stat_file
|
||||
ret.append([fname, fun(n["ts"], n["sz"]), 0])
|
||||
|
||||
return ret
|
||||
|
||||
@@ -444,7 +439,7 @@ class Gateway(object):
|
||||
ts = 60 * 60 * 24 * 2
|
||||
try:
|
||||
sz = int(fsize)
|
||||
ts = calendar.timegm(time.strptime(fdate, "%Y-%m-%d %H:%M:%S"))
|
||||
ts = datetime.strptime(fdate, "%Y-%m-%d %H:%M:%S").timestamp()
|
||||
except:
|
||||
info("bad HTML or OS [{}] [{}]".format(fdate, fsize))
|
||||
# python cannot strptime(1959-01-01) on windows
|
||||
|
||||
@@ -11,18 +11,14 @@ import re
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import stat
|
||||
import errno
|
||||
import struct
|
||||
import codecs
|
||||
import platform
|
||||
import threading
|
||||
import http.client # py2: httplib
|
||||
import urllib.parse
|
||||
from datetime import datetime
|
||||
from urllib.parse import quote_from_bytes as quote
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
|
||||
try:
|
||||
import fuse
|
||||
@@ -42,7 +38,7 @@ except:
|
||||
mount a copyparty server (local or remote) as a filesystem
|
||||
|
||||
usage:
|
||||
python ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas
|
||||
python ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,url=http://192.168.1.69:3923 /mnt/nas
|
||||
|
||||
dependencies:
|
||||
sudo apk add fuse-dev python3-dev
|
||||
@@ -54,10 +50,6 @@ fork of copyparty-fuse.py based on fuse-python which
|
||||
"""
|
||||
|
||||
|
||||
WINDOWS = sys.platform == "win32"
|
||||
MACOS = platform.system() == "Darwin"
|
||||
|
||||
|
||||
def threadless_log(msg):
|
||||
print(msg + "\n", end="")
|
||||
|
||||
@@ -101,41 +93,6 @@ def html_dec(txt):
|
||||
)
|
||||
|
||||
|
||||
def register_wtf8():
|
||||
def wtf8_enc(text):
|
||||
return str(text).encode("utf-8", "surrogateescape"), len(text)
|
||||
|
||||
def wtf8_dec(binary):
|
||||
return bytes(binary).decode("utf-8", "surrogateescape"), len(binary)
|
||||
|
||||
def wtf8_search(encoding_name):
|
||||
return codecs.CodecInfo(wtf8_enc, wtf8_dec, name="wtf-8")
|
||||
|
||||
codecs.register(wtf8_search)
|
||||
|
||||
|
||||
bad_good = {}
|
||||
good_bad = {}
|
||||
|
||||
|
||||
def enwin(txt):
|
||||
return "".join([bad_good.get(x, x) for x in txt])
|
||||
|
||||
for bad, good in bad_good.items():
|
||||
txt = txt.replace(bad, good)
|
||||
|
||||
return txt
|
||||
|
||||
|
||||
def dewin(txt):
|
||||
return "".join([good_bad.get(x, x) for x in txt])
|
||||
|
||||
for bad, good in bad_good.items():
|
||||
txt = txt.replace(good, bad)
|
||||
|
||||
return txt
|
||||
|
||||
|
||||
class CacheNode(object):
|
||||
def __init__(self, tag, data):
|
||||
self.tag = tag
|
||||
@@ -158,9 +115,8 @@ class Stat(fuse.Stat):
|
||||
|
||||
|
||||
class Gateway(object):
|
||||
def __init__(self, base_url, pw):
|
||||
def __init__(self, base_url):
|
||||
self.base_url = base_url
|
||||
self.pw = pw
|
||||
|
||||
ui = urllib.parse.urlparse(base_url)
|
||||
self.web_root = ui.path.strip("/")
|
||||
@@ -179,7 +135,8 @@ class Gateway(object):
|
||||
self.conns = {}
|
||||
|
||||
def quotep(self, path):
|
||||
path = path.encode("wtf-8")
|
||||
# TODO: mojibake support
|
||||
path = path.encode("utf-8", "ignore")
|
||||
return quote(path, safe="/")
|
||||
|
||||
def getconn(self, tid=None):
|
||||
@@ -202,29 +159,20 @@ class Gateway(object):
|
||||
except:
|
||||
pass
|
||||
|
||||
def sendreq(self, *args, **ka):
|
||||
def sendreq(self, *args, **kwargs):
|
||||
tid = get_tid()
|
||||
if self.pw:
|
||||
ck = "cppwd=" + self.pw
|
||||
try:
|
||||
ka["headers"]["Cookie"] = ck
|
||||
except:
|
||||
ka["headers"] = {"Cookie": ck}
|
||||
try:
|
||||
c = self.getconn(tid)
|
||||
c.request(*list(args), **ka)
|
||||
c.request(*list(args), **kwargs)
|
||||
return c.getresponse()
|
||||
except:
|
||||
self.closeconn(tid)
|
||||
c = self.getconn(tid)
|
||||
c.request(*list(args), **ka)
|
||||
c.request(*list(args), **kwargs)
|
||||
return c.getresponse()
|
||||
|
||||
def listdir(self, path):
|
||||
if bad_good:
|
||||
path = dewin(path)
|
||||
|
||||
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
|
||||
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
|
||||
r = self.sendreq("GET", web_path)
|
||||
if r.status != 200:
|
||||
self.closeconn()
|
||||
@@ -234,12 +182,9 @@ class Gateway(object):
|
||||
)
|
||||
)
|
||||
|
||||
return self.parse_jls(r)
|
||||
return self.parse_html(r)
|
||||
|
||||
def download_file_range(self, path, ofs1, ofs2):
|
||||
if bad_good:
|
||||
path = dewin(path)
|
||||
|
||||
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?raw"
|
||||
hdr_range = "bytes={}-{}".format(ofs1, ofs2 - 1)
|
||||
log("downloading {}".format(hdr_range))
|
||||
@@ -255,27 +200,40 @@ class Gateway(object):
|
||||
|
||||
return r.read()
|
||||
|
||||
def parse_jls(self, datasrc):
|
||||
rsp = b""
|
||||
def parse_html(self, datasrc):
|
||||
ret = []
|
||||
remainder = b""
|
||||
ptn = re.compile(
|
||||
r"^<tr><td>(-|DIR)</td><td><a [^>]+>([^<]+)</a></td><td>([^<]+)</td><td>([^<]+)</td></tr>$"
|
||||
)
|
||||
|
||||
while True:
|
||||
buf = datasrc.read(1024 * 32)
|
||||
buf = remainder + datasrc.read(4096)
|
||||
# print('[{}]'.format(buf.decode('utf-8')))
|
||||
if not buf:
|
||||
break
|
||||
|
||||
rsp += buf
|
||||
remainder = b""
|
||||
endpos = buf.rfind(b"\n")
|
||||
if endpos >= 0:
|
||||
remainder = buf[endpos + 1 :]
|
||||
buf = buf[:endpos]
|
||||
|
||||
rsp = json.loads(rsp.decode("utf-8"))
|
||||
ret = []
|
||||
for statfun, nodes in [
|
||||
[self.stat_dir, rsp["dirs"]],
|
||||
[self.stat_file, rsp["files"]],
|
||||
]:
|
||||
for n in nodes:
|
||||
fname = unquote(n["href"].split("?")[0]).rstrip(b"/").decode("wtf-8")
|
||||
if bad_good:
|
||||
fname = enwin(fname)
|
||||
lines = buf.decode("utf-8").split("\n")
|
||||
for line in lines:
|
||||
m = ptn.match(line)
|
||||
if not m:
|
||||
# print(line)
|
||||
continue
|
||||
|
||||
ret.append([fname, statfun(n["ts"], n["sz"]), 0])
|
||||
ftype, fname, fsize, fdate = m.groups()
|
||||
fname = html_dec(fname)
|
||||
ts = datetime.strptime(fdate, "%Y-%m-%d %H:%M:%S").timestamp()
|
||||
sz = int(fsize)
|
||||
if ftype == "-":
|
||||
ret.append([fname, self.stat_file(ts, sz), 0])
|
||||
else:
|
||||
ret.append([fname, self.stat_dir(ts, sz), 0])
|
||||
|
||||
return ret
|
||||
|
||||
@@ -304,7 +262,6 @@ class CPPF(Fuse):
|
||||
Fuse.__init__(self, *args, **kwargs)
|
||||
|
||||
self.url = None
|
||||
self.pw = None
|
||||
|
||||
self.dircache = []
|
||||
self.dircache_mtx = threading.Lock()
|
||||
@@ -314,7 +271,7 @@ class CPPF(Fuse):
|
||||
|
||||
def init2(self):
|
||||
# TODO figure out how python-fuse wanted this to go
|
||||
self.gw = Gateway(self.url, self.pw) # .decode('utf-8'))
|
||||
self.gw = Gateway(self.url) # .decode('utf-8'))
|
||||
info("up")
|
||||
|
||||
def clean_dircache(self):
|
||||
@@ -579,8 +536,6 @@ class CPPF(Fuse):
|
||||
|
||||
def getattr(self, path):
|
||||
log("getattr [{}]".format(path))
|
||||
if WINDOWS:
|
||||
path = enwin(path) # windows occasionally decodes f0xx to xx
|
||||
|
||||
path = path.strip("/")
|
||||
try:
|
||||
@@ -613,25 +568,9 @@ class CPPF(Fuse):
|
||||
|
||||
def main():
|
||||
time.strptime("19970815", "%Y%m%d") # python#7980
|
||||
register_wtf8()
|
||||
if WINDOWS:
|
||||
os.system("rem")
|
||||
|
||||
for ch in '<>:"\\|?*':
|
||||
# microsoft maps illegal characters to f0xx
|
||||
# (e000 to f8ff is basic-plane private-use)
|
||||
bad_good[ch] = chr(ord(ch) + 0xF000)
|
||||
|
||||
for n in range(0, 0x100):
|
||||
# map surrogateescape to another private-use area
|
||||
bad_good[chr(n + 0xDC00)] = chr(n + 0xF100)
|
||||
|
||||
for k, v in bad_good.items():
|
||||
good_bad[v] = k
|
||||
|
||||
server = CPPF()
|
||||
server.parser.add_option(mountopt="url", metavar="BASE_URL", default=None)
|
||||
server.parser.add_option(mountopt="pw", metavar="PASSWORD", default=None)
|
||||
server.parse(values=server, errex=1)
|
||||
if not server.url or not str(server.url).startswith("http"):
|
||||
print("\nerror:")
|
||||
@@ -639,7 +578,7 @@ def main():
|
||||
print(" need argument: mount-path")
|
||||
print("example:")
|
||||
print(
|
||||
" ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas"
|
||||
" ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,url=http://192.168.1.69:3923 /mnt/nas"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
101
bin/dbtool.py
101
bin/dbtool.py
@@ -8,10 +8,7 @@ import sqlite3
|
||||
import argparse
|
||||
|
||||
DB_VER1 = 3
|
||||
DB_VER2 = 5
|
||||
|
||||
BY_PATH = None
|
||||
NC = None
|
||||
DB_VER2 = 4
|
||||
|
||||
|
||||
def die(msg):
|
||||
@@ -60,13 +57,8 @@ def compare(n1, d1, n2, d2, verbose):
|
||||
if rd.split("/", 1)[0] == ".hist":
|
||||
continue
|
||||
|
||||
if BY_PATH:
|
||||
q = "select w from up where rd = ? and fn = ?"
|
||||
hit = d2.execute(q, (rd, fn)).fetchone()
|
||||
else:
|
||||
q = "select w from up where substr(w,1,16) = ? and +w = ?"
|
||||
hit = d2.execute(q, (w1[:16], w1)).fetchone()
|
||||
|
||||
if not hit:
|
||||
miss += 1
|
||||
if verbose:
|
||||
@@ -78,32 +70,27 @@ def compare(n1, d1, n2, d2, verbose):
|
||||
n = 0
|
||||
miss = {}
|
||||
nmiss = 0
|
||||
for w1s, k, v in d1.execute("select * from mt"):
|
||||
for w1, k, v in d1.execute("select * from mt"):
|
||||
|
||||
n += 1
|
||||
if n % 100_000 == 0:
|
||||
m = f"\033[36mchecked {n:,} of {nt:,} tags in {n1} against {n2}, so far {nmiss} missing tags\033[0m"
|
||||
print(m)
|
||||
|
||||
q = "select w, rd, fn from up where substr(w,1,16) = ?"
|
||||
w1, rd, fn = d1.execute(q, (w1s,)).fetchone()
|
||||
q = "select rd, fn from up where substr(w,1,16) = ?"
|
||||
rd, fn = d1.execute(q, (w1,)).fetchone()
|
||||
if rd.split("/", 1)[0] == ".hist":
|
||||
continue
|
||||
|
||||
if BY_PATH:
|
||||
q = "select w from up where rd = ? and fn = ?"
|
||||
q = "select substr(w,1,16) from up where rd = ? and fn = ?"
|
||||
w2 = d2.execute(q, (rd, fn)).fetchone()
|
||||
else:
|
||||
q = "select w from up where substr(w,1,16) = ? and +w = ?"
|
||||
w2 = d2.execute(q, (w1s, w1)).fetchone()
|
||||
|
||||
if w2:
|
||||
w2 = w2[0]
|
||||
|
||||
v2 = None
|
||||
if w2:
|
||||
v2 = d2.execute(
|
||||
"select v from mt where w = ? and +k = ?", (w2[:16], k)
|
||||
"select v from mt where w = ? and +k = ?", (w2, k)
|
||||
).fetchone()
|
||||
if v2:
|
||||
v2 = v2[0]
|
||||
@@ -137,7 +124,7 @@ def compare(n1, d1, n2, d2, verbose):
|
||||
|
||||
for k, v in sorted(miss.items()):
|
||||
if v:
|
||||
print(f"{n1} has {v:7} more {k:<7} tags than {n2}")
|
||||
print(f"{n1} has {v:6} more {k:<6} tags than {n2}")
|
||||
|
||||
print(f"in total, {nmiss} missing tags in {n2}\n")
|
||||
|
||||
@@ -145,75 +132,47 @@ def compare(n1, d1, n2, d2, verbose):
|
||||
def copy_mtp(d1, d2, tag, rm):
|
||||
nt = next(d1.execute("select count(w) from mt where k = ?", (tag,)))[0]
|
||||
n = 0
|
||||
ncopy = 0
|
||||
nskip = 0
|
||||
for w1s, k, v in d1.execute("select * from mt where k = ?", (tag,)):
|
||||
ndone = 0
|
||||
for w1, k, v in d1.execute("select * from mt where k = ?", (tag,)):
|
||||
n += 1
|
||||
if n % 25_000 == 0:
|
||||
m = f"\033[36m{n:,} of {nt:,} tags checked, so far {ncopy} copied, {nskip} skipped\033[0m"
|
||||
m = f"\033[36m{n:,} of {nt:,} tags checked, so far {ndone} copied\033[0m"
|
||||
print(m)
|
||||
|
||||
q = "select w, rd, fn from up where substr(w,1,16) = ?"
|
||||
w1, rd, fn = d1.execute(q, (w1s,)).fetchone()
|
||||
q = "select rd, fn from up where substr(w,1,16) = ?"
|
||||
rd, fn = d1.execute(q, (w1,)).fetchone()
|
||||
if rd.split("/", 1)[0] == ".hist":
|
||||
continue
|
||||
|
||||
if BY_PATH:
|
||||
q = "select w from up where rd = ? and fn = ?"
|
||||
q = "select substr(w,1,16) from up where rd = ? and fn = ?"
|
||||
w2 = d2.execute(q, (rd, fn)).fetchone()
|
||||
else:
|
||||
q = "select w from up where substr(w,1,16) = ? and +w = ?"
|
||||
w2 = d2.execute(q, (w1s, w1)).fetchone()
|
||||
|
||||
if not w2:
|
||||
continue
|
||||
|
||||
w2s = w2[0][:16]
|
||||
hit = d2.execute("select v from mt where w = ? and +k = ?", (w2s, k)).fetchone()
|
||||
w2 = w2[0]
|
||||
hit = d2.execute("select v from mt where w = ? and +k = ?", (w2, k)).fetchone()
|
||||
if hit:
|
||||
hit = hit[0]
|
||||
|
||||
if hit != v:
|
||||
if NC and hit is not None:
|
||||
nskip += 1
|
||||
continue
|
||||
|
||||
ncopy += 1
|
||||
ndone += 1
|
||||
if hit is not None:
|
||||
d2.execute("delete from mt where w = ? and +k = ?", (w2s, k))
|
||||
d2.execute("delete from mt where w = ? and +k = ?", (w2, k))
|
||||
|
||||
d2.execute("insert into mt values (?,?,?)", (w2s, k, v))
|
||||
d2.execute("insert into mt values (?,?,?)", (w2, k, v))
|
||||
if rm:
|
||||
d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w2s,))
|
||||
d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w2,))
|
||||
|
||||
d2.commit()
|
||||
print(f"copied {ncopy} {tag} tags over, skipped {nskip}")
|
||||
|
||||
|
||||
def examples():
|
||||
print(
|
||||
"""
|
||||
# clearing the journal
|
||||
./dbtool.py up2k.db
|
||||
|
||||
# copy tags ".bpm" and "key" from old.db to up2k.db, and remove the mtp flag from matching files (so copyparty won't run any mtps on it)
|
||||
./dbtool.py -ls up2k.db
|
||||
./dbtool.py -src old.db up2k.db -cmp
|
||||
./dbtool.py -src old.v3 up2k.db -rm-mtp-flag -copy key
|
||||
./dbtool.py -src old.v3 up2k.db -rm-mtp-flag -copy .bpm -vac
|
||||
|
||||
"""
|
||||
)
|
||||
print(f"copied {ndone} {tag} tags over")
|
||||
|
||||
|
||||
def main():
|
||||
global NC, BY_PATH
|
||||
os.system("")
|
||||
print()
|
||||
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("db", help="database to work on")
|
||||
ap.add_argument("-h2", action="store_true", help="show examples")
|
||||
ap.add_argument("-src", metavar="DB", type=str, help="database to copy from")
|
||||
|
||||
ap2 = ap.add_argument_group("informational / read-only stuff")
|
||||
@@ -226,29 +185,11 @@ def main():
|
||||
ap2.add_argument(
|
||||
"-rm-mtp-flag",
|
||||
action="store_true",
|
||||
help="when an mtp tag is copied over, also mark that file as done, so copyparty won't run any mtps on those files",
|
||||
help="when an mtp tag is copied over, also mark that as done, so copyparty won't run mtp on it",
|
||||
)
|
||||
ap2.add_argument("-vac", action="store_true", help="optimize DB")
|
||||
|
||||
ap2 = ap.add_argument_group("behavior modifiers")
|
||||
ap2.add_argument(
|
||||
"-nc",
|
||||
action="store_true",
|
||||
help="no-clobber; don't replace/overwrite existing tags",
|
||||
)
|
||||
ap2.add_argument(
|
||||
"-by-path",
|
||||
action="store_true",
|
||||
help="match files based on location rather than warks (content-hash), use this if the databases have different wark salts",
|
||||
)
|
||||
|
||||
ar = ap.parse_args()
|
||||
if ar.h2:
|
||||
examples()
|
||||
return
|
||||
|
||||
NC = ar.nc
|
||||
BY_PATH = ar.by_path
|
||||
|
||||
for v in [ar.db, ar.src]:
|
||||
if v and not os.path.exists(v):
|
||||
|
||||
@@ -1,23 +1,10 @@
|
||||
standalone programs which take an audio file as argument
|
||||
|
||||
**NOTE:** these all require `-e2ts` to be functional, meaning you need to do at least one of these: `apt install ffmpeg` or `pip3 install mutagen`
|
||||
|
||||
some of these rely on libraries which are not MIT-compatible
|
||||
|
||||
* [audio-bpm.py](./audio-bpm.py) detects the BPM of music using the BeatRoot Vamp Plugin; imports GPL2
|
||||
* [audio-key.py](./audio-key.py) detects the melodic key of music using the Mixxx fork of keyfinder; imports GPL3
|
||||
|
||||
these invoke standalone programs which are GPL or similar, so is legally fine for most purposes:
|
||||
|
||||
* [media-hash.py](./media-hash.py) generates checksums for audio and video streams; uses FFmpeg (LGPL or GPL)
|
||||
* [image-noexif.py](./image-noexif.py) removes exif tags from images; uses exiftool (GPLv1 or artistic-license)
|
||||
|
||||
these do not have any problematic dependencies at all:
|
||||
|
||||
* [cksum.py](./cksum.py) computes various checksums
|
||||
* [exe.py](./exe.py) grabs metadata from .exe and .dll files (example for retrieving multiple tags with one parser)
|
||||
* [wget.py](./wget.py) lets you download files by POSTing URLs to copyparty
|
||||
|
||||
|
||||
# dependencies
|
||||
|
||||
@@ -31,10 +18,7 @@ run [`install-deps.sh`](install-deps.sh) to build/install most dependencies requ
|
||||
|
||||
# usage from copyparty
|
||||
|
||||
`copyparty -e2dsa -e2ts` followed by any combination of these:
|
||||
* `-mtp key=f,audio-key.py`
|
||||
* `-mtp .bpm=f,audio-bpm.py`
|
||||
* `-mtp ahash,vhash=f,media-hash.py`
|
||||
`copyparty -e2dsa -e2ts -mtp key=f,audio-key.py -mtp .bpm=f,audio-bpm.py`
|
||||
|
||||
* `f,` makes the detected value replace any existing values
|
||||
* the `.` in `.bpm` indicates numeric value
|
||||
@@ -42,12 +26,9 @@ run [`install-deps.sh`](install-deps.sh) to build/install most dependencies requ
|
||||
* `mtp` modules will not run if a file has existing tags in the db, so clear out the tags with `-e2tsr` the first time you launch with new `mtp` options
|
||||
|
||||
|
||||
## usage with volflags
|
||||
## usage with volume-flags
|
||||
|
||||
instead of affecting all volumes, you can set the options for just one volume like so:
|
||||
|
||||
`copyparty -v /mnt/nas/music:/music:r:c,e2dsa:c,e2ts` immediately followed by any combination of these:
|
||||
|
||||
* `:c,mtp=key=f,audio-key.py`
|
||||
* `:c,mtp=.bpm=f,audio-bpm.py`
|
||||
* `:c,mtp=ahash,vhash=f,media-hash.py`
|
||||
```
|
||||
copyparty -v /mnt/nas/music:/music:r:cmtp=key=f,audio-key.py:cmtp=.bpm=f,audio-bpm.py:ce2dsa:ce2ts
|
||||
```
|
||||
|
||||
@@ -19,18 +19,17 @@ dep: ffmpeg
|
||||
def det(tf):
|
||||
# fmt: off
|
||||
sp.check_call([
|
||||
b"ffmpeg",
|
||||
b"-nostdin",
|
||||
b"-hide_banner",
|
||||
b"-v", b"fatal",
|
||||
b"-ss", b"13",
|
||||
b"-y", b"-i", fsenc(sys.argv[1]),
|
||||
b"-map", b"0:a:0",
|
||||
b"-ac", b"1",
|
||||
b"-ar", b"22050",
|
||||
b"-t", b"300",
|
||||
b"-f", b"f32le",
|
||||
fsenc(tf)
|
||||
"ffmpeg",
|
||||
"-nostdin",
|
||||
"-hide_banner",
|
||||
"-v", "fatal",
|
||||
"-ss", "13",
|
||||
"-y", "-i", fsenc(sys.argv[1]),
|
||||
"-ac", "1",
|
||||
"-ar", "22050",
|
||||
"-t", "300",
|
||||
"-f", "f32le",
|
||||
tf
|
||||
])
|
||||
# fmt: on
|
||||
|
||||
|
||||
@@ -23,15 +23,14 @@ dep: ffmpeg
|
||||
def det(tf):
|
||||
# fmt: off
|
||||
sp.check_call([
|
||||
b"ffmpeg",
|
||||
b"-nostdin",
|
||||
b"-hide_banner",
|
||||
b"-v", b"fatal",
|
||||
b"-y", b"-i", fsenc(sys.argv[1]),
|
||||
b"-map", b"0:a:0",
|
||||
b"-t", b"300",
|
||||
b"-sample_fmt", b"s16",
|
||||
fsenc(tf)
|
||||
"ffmpeg",
|
||||
"-nostdin",
|
||||
"-hide_banner",
|
||||
"-v", "fatal",
|
||||
"-y", "-i", fsenc(sys.argv[1]),
|
||||
"-t", "300",
|
||||
"-sample_fmt", "s16",
|
||||
tf
|
||||
])
|
||||
# fmt: on
|
||||
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import json
|
||||
import zlib
|
||||
import struct
|
||||
import base64
|
||||
import hashlib
|
||||
|
||||
try:
|
||||
from copyparty.util import fsenc
|
||||
except:
|
||||
|
||||
def fsenc(p):
|
||||
return p
|
||||
|
||||
|
||||
"""
|
||||
calculates various checksums for uploads,
|
||||
usage: -mtp crc32,md5,sha1,sha256b=ad,bin/mtag/cksum.py
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
config = "crc32 md5 md5b sha1 sha1b sha256 sha256b sha512/240 sha512b/240"
|
||||
# b suffix = base64 encoded
|
||||
# slash = truncate to n bits
|
||||
|
||||
known = {
|
||||
"md5": hashlib.md5,
|
||||
"sha1": hashlib.sha1,
|
||||
"sha256": hashlib.sha256,
|
||||
"sha512": hashlib.sha512,
|
||||
}
|
||||
config = config.split()
|
||||
hashers = {
|
||||
k: v()
|
||||
for k, v in known.items()
|
||||
if k in [x.split("/")[0].rstrip("b") for x in known]
|
||||
}
|
||||
crc32 = 0 if "crc32" in config else None
|
||||
|
||||
with open(fsenc(sys.argv[1]), "rb", 512 * 1024) as f:
|
||||
while True:
|
||||
buf = f.read(64 * 1024)
|
||||
if not buf:
|
||||
break
|
||||
|
||||
for x in hashers.values():
|
||||
x.update(buf)
|
||||
|
||||
if crc32 is not None:
|
||||
crc32 = zlib.crc32(buf, crc32)
|
||||
|
||||
ret = {}
|
||||
for s in config:
|
||||
alg = s.split("/")[0]
|
||||
b64 = alg.endswith("b")
|
||||
alg = alg.rstrip("b")
|
||||
if alg in hashers:
|
||||
v = hashers[alg].digest()
|
||||
elif alg == "crc32":
|
||||
v = crc32
|
||||
if v < 0:
|
||||
v &= 2 ** 32 - 1
|
||||
v = struct.pack(">L", v)
|
||||
else:
|
||||
raise Exception("what is {}".format(s))
|
||||
|
||||
if "/" in s:
|
||||
v = v[: int(int(s.split("/")[1]) / 8)]
|
||||
|
||||
if b64:
|
||||
v = base64.b64encode(v).decode("ascii").rstrip("=")
|
||||
else:
|
||||
try:
|
||||
v = v.hex()
|
||||
except:
|
||||
import binascii
|
||||
|
||||
v = binascii.hexlify(v)
|
||||
|
||||
ret[s] = v
|
||||
|
||||
print(json.dumps(ret, indent=4))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,61 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
fetch latest msg from guestbook and return as tag
|
||||
|
||||
example copyparty config to use this:
|
||||
--urlform save,get -vsrv/hello:hello:w:c,e2ts,mtp=guestbook=t10,ad,p,bin/mtag/guestbook-read.py:mte=+guestbook
|
||||
|
||||
explained:
|
||||
for realpath srv/hello (served at /hello), write-only for eveyrone,
|
||||
enable file analysis on upload (e2ts),
|
||||
use mtp plugin "bin/mtag/guestbook-read.py" to provide metadata tag "guestbook",
|
||||
do this on all uploads regardless of extension,
|
||||
t10 = 10 seconds timeout for each dwonload,
|
||||
ad = parse file regardless if FFmpeg thinks it is audio or not
|
||||
p = request upload info as json on stdin (need ip)
|
||||
mte=+guestbook enabled indexing of that tag for this volume
|
||||
|
||||
PS: this requires e2ts to be functional,
|
||||
meaning you need to do at least one of these:
|
||||
* apt install ffmpeg
|
||||
* pip3 install mutagen
|
||||
"""
|
||||
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
|
||||
# set 0 to allow infinite msgs from one IP,
|
||||
# other values delete older messages to make space,
|
||||
# so 1 only keeps latest msg
|
||||
NUM_MSGS_TO_KEEP = 1
|
||||
|
||||
|
||||
def main():
|
||||
fp = os.path.abspath(sys.argv[1])
|
||||
fdir = os.path.dirname(fp)
|
||||
|
||||
zb = sys.stdin.buffer.read()
|
||||
zs = zb.decode("utf-8", "replace")
|
||||
md = json.loads(zs)
|
||||
|
||||
ip = md["up_ip"]
|
||||
|
||||
# can put the database inside `fdir` if you'd like,
|
||||
# by default it saves to PWD:
|
||||
# os.chdir(fdir)
|
||||
|
||||
db = sqlite3.connect("guestbook.db3")
|
||||
with db:
|
||||
t = "select msg from gb where ip = ? order by ts desc"
|
||||
r = db.execute(t, (ip,)).fetchone()
|
||||
if r:
|
||||
print(r[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,111 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
store messages from users in an sqlite database
|
||||
which can be read from another mtp for example
|
||||
|
||||
takes input from application/x-www-form-urlencoded POSTs,
|
||||
for example using the message/pager function on the website
|
||||
|
||||
example copyparty config to use this:
|
||||
--urlform save,get -vsrv/hello:hello:w:c,e2ts,mtp=xgb=ebin,t10,ad,p,bin/mtag/guestbook.py:mte=+xgb
|
||||
|
||||
explained:
|
||||
for realpath srv/hello (served at /hello),write-only for eveyrone,
|
||||
enable file analysis on upload (e2ts),
|
||||
use mtp plugin "bin/mtag/guestbook.py" to provide metadata tag "xgb",
|
||||
do this on all uploads with the file extension "bin",
|
||||
t300 = 300 seconds timeout for each dwonload,
|
||||
ad = parse file regardless if FFmpeg thinks it is audio or not
|
||||
p = request upload info as json on stdin
|
||||
mte=+xgb enabled indexing of that tag for this volume
|
||||
|
||||
PS: this requires e2ts to be functional,
|
||||
meaning you need to do at least one of these:
|
||||
* apt install ffmpeg
|
||||
* pip3 install mutagen
|
||||
"""
|
||||
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
|
||||
|
||||
# set 0 to allow infinite msgs from one IP,
|
||||
# other values delete older messages to make space,
|
||||
# so 1 only keeps latest msg
|
||||
NUM_MSGS_TO_KEEP = 1
|
||||
|
||||
|
||||
def main():
|
||||
fp = os.path.abspath(sys.argv[1])
|
||||
fdir = os.path.dirname(fp)
|
||||
fname = os.path.basename(fp)
|
||||
if not fname.startswith("put-") or not fname.endswith(".bin"):
|
||||
raise Exception("not a post file")
|
||||
|
||||
zb = sys.stdin.buffer.read()
|
||||
zs = zb.decode("utf-8", "replace")
|
||||
md = json.loads(zs)
|
||||
|
||||
buf = b""
|
||||
with open(fp, "rb") as f:
|
||||
while True:
|
||||
b = f.read(4096)
|
||||
buf += b
|
||||
if len(buf) > 4096:
|
||||
raise Exception("too big")
|
||||
|
||||
if not b:
|
||||
break
|
||||
|
||||
if not buf:
|
||||
raise Exception("file is empty")
|
||||
|
||||
buf = unquote(buf.replace(b"+", b" "))
|
||||
txt = buf.decode("utf-8")
|
||||
|
||||
if not txt.startswith("msg="):
|
||||
raise Exception("does not start with msg=")
|
||||
|
||||
ip = md["up_ip"]
|
||||
ts = md["up_at"]
|
||||
txt = txt[4:]
|
||||
|
||||
# can put the database inside `fdir` if you'd like,
|
||||
# by default it saves to PWD:
|
||||
# os.chdir(fdir)
|
||||
|
||||
db = sqlite3.connect("guestbook.db3")
|
||||
try:
|
||||
db.execute("select 1 from gb").fetchone()
|
||||
except:
|
||||
with db:
|
||||
db.execute("create table gb (ip text, ts real, msg text)")
|
||||
db.execute("create index gb_ip on gb(ip)")
|
||||
|
||||
with db:
|
||||
if NUM_MSGS_TO_KEEP == 1:
|
||||
t = "delete from gb where ip = ?"
|
||||
db.execute(t, (ip,))
|
||||
|
||||
t = "insert into gb values (?,?,?)"
|
||||
db.execute(t, (ip, ts, txt))
|
||||
|
||||
if NUM_MSGS_TO_KEEP > 1:
|
||||
t = "select ts from gb where ip = ? order by ts desc"
|
||||
hits = db.execute(t, (ip,)).fetchall()
|
||||
|
||||
if len(hits) > NUM_MSGS_TO_KEEP:
|
||||
lim = hits[NUM_MSGS_TO_KEEP][0]
|
||||
t = "delete from gb where ip = ? and ts <= ?"
|
||||
db.execute(t, (ip, lim))
|
||||
|
||||
print(txt)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,95 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
remove exif tags from uploaded images
|
||||
|
||||
dependencies:
|
||||
exiftool
|
||||
|
||||
about:
|
||||
creates a "noexif" subfolder and puts exif-stripped copies of each image there,
|
||||
the reason for the subfolder is to avoid issues with the up2k.db / deduplication:
|
||||
|
||||
if the original image is modified in-place, then copyparty will keep the original
|
||||
hash in up2k.db for a while (until the next volume rescan), so if the image is
|
||||
reuploaded after a rescan then the upload will be renamed and kept as a dupe
|
||||
|
||||
alternatively you could switch the logic around, making a copy of the original
|
||||
image into a subfolder named "exif" and modify the original in-place, but then
|
||||
up2k.db will be out of sync until the next rescan, so any additional uploads
|
||||
of the same image will get symlinked (deduplicated) to the modified copy
|
||||
instead of the original in "exif"
|
||||
|
||||
or maybe delete the original image after processing, that would kinda work too
|
||||
|
||||
example copyparty config to use this:
|
||||
-v/mnt/nas/pics:pics:rwmd,ed:c,e2ts,mte=+noexif:c,mtp=noexif=ejpg,ejpeg,ad,bin/mtag/image-noexif.py
|
||||
|
||||
explained:
|
||||
for realpath /mnt/nas/pics (served at /pics) with read-write-modify-delete for ed,
|
||||
enable file analysis on upload (e2ts),
|
||||
append "noexif" to the list of known tags (mtp),
|
||||
and use mtp plugin "bin/mtag/image-noexif.py" to provide that tag,
|
||||
do this on all uploads with the file extension "jpg" or "jpeg",
|
||||
ad = parse file regardless if FFmpeg thinks it is audio or not
|
||||
|
||||
PS: this requires e2ts to be functional,
|
||||
meaning you need to do at least one of these:
|
||||
* apt install ffmpeg
|
||||
* pip3 install mutagen
|
||||
and your python must have sqlite3 support compiled in
|
||||
"""
|
||||
|
||||
|
||||
import os
|
||||
import sys
|
||||
import filecmp
|
||||
import subprocess as sp
|
||||
|
||||
try:
|
||||
from copyparty.util import fsenc
|
||||
except:
|
||||
|
||||
def fsenc(p):
|
||||
return p.encode("utf-8")
|
||||
|
||||
|
||||
def main():
|
||||
cwd, fn = os.path.split(sys.argv[1])
|
||||
if os.path.basename(cwd) == "noexif":
|
||||
return
|
||||
|
||||
os.chdir(cwd)
|
||||
f1 = fsenc(fn)
|
||||
f2 = os.path.join(b"noexif", f1)
|
||||
cmd = [
|
||||
b"exiftool",
|
||||
b"-exif:all=",
|
||||
b"-iptc:all=",
|
||||
b"-xmp:all=",
|
||||
b"-P",
|
||||
b"-o",
|
||||
b"noexif/",
|
||||
b"--",
|
||||
f1,
|
||||
]
|
||||
sp.check_output(cmd)
|
||||
if not os.path.exists(f2):
|
||||
print("failed")
|
||||
return
|
||||
|
||||
if filecmp.cmp(f1, f2, shallow=False):
|
||||
print("clean")
|
||||
else:
|
||||
print("exif")
|
||||
|
||||
# lastmod = os.path.getmtime(f1)
|
||||
# times = (int(time.time()), int(lastmod))
|
||||
# os.utime(f2, times)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except:
|
||||
pass
|
||||
@@ -4,8 +4,7 @@ set -e
|
||||
|
||||
# install dependencies for audio-*.py
|
||||
#
|
||||
# linux/alpine: requires gcc g++ make cmake patchelf {python3,ffmpeg,fftw,libsndfile}-dev py3-{wheel,pip} py3-numpy{,-dev}
|
||||
# linux/debian: requires libav{codec,device,filter,format,resample,util}-dev {libfftw3,python3,libsndfile1}-dev python3-{numpy,pip} vamp-{plugin-sdk,examples} patchelf cmake
|
||||
# linux: requires {python3,ffmpeg,fftw}-dev py3-{wheel,pip} py3-numpy{,-dev} vamp-sdk-dev patchelf
|
||||
# win64: requires msys2-mingw64 environment
|
||||
# macos: requires macports
|
||||
#
|
||||
@@ -101,11 +100,8 @@ export -f dl_files
|
||||
|
||||
|
||||
github_tarball() {
|
||||
rm -rf g
|
||||
mkdir g
|
||||
cd g
|
||||
dl_text "$1" |
|
||||
tee ../json |
|
||||
tee json |
|
||||
(
|
||||
# prefer jq if available
|
||||
jq -r '.tarball_url' ||
|
||||
@@ -114,11 +110,8 @@ github_tarball() {
|
||||
awk -F\" '/"tarball_url": "/ {print$4}'
|
||||
) |
|
||||
tee /dev/stderr |
|
||||
head -n 1 |
|
||||
tr -d '\r' | tr '\n' '\0' |
|
||||
xargs -0 bash -c 'dl_files "$@"' _
|
||||
mv * ../tgz
|
||||
cd ..
|
||||
}
|
||||
|
||||
|
||||
@@ -133,7 +126,6 @@ gitlab_tarball() {
|
||||
tr \" '\n' | grep -E '\.tar\.gz$' | head -n 1
|
||||
) |
|
||||
tee /dev/stderr |
|
||||
head -n 1 |
|
||||
tr -d '\r' | tr '\n' '\0' |
|
||||
tee links |
|
||||
xargs -0 bash -c 'dl_files "$@"' _
|
||||
@@ -145,17 +137,10 @@ install_keyfinder() {
|
||||
# use msys2 in mingw-w64 mode
|
||||
# pacman -S --needed mingw-w64-x86_64-{ffmpeg,python}
|
||||
|
||||
[ -e $HOME/pe/keyfinder ] && {
|
||||
echo found a keyfinder build in ~/pe, skipping
|
||||
return
|
||||
}
|
||||
|
||||
cd "$td"
|
||||
github_tarball https://api.github.com/repos/mixxxdj/libkeyfinder/releases/latest
|
||||
ls -al
|
||||
|
||||
tar -xf tgz
|
||||
rm tgz
|
||||
tar -xf mixxxdj-libkeyfinder-*
|
||||
rm -- *.tar.gz
|
||||
cd mixxxdj-libkeyfinder*
|
||||
|
||||
h="$HOME"
|
||||
@@ -222,22 +207,6 @@ install_vamp() {
|
||||
|
||||
$pybin -m pip install --user vamp
|
||||
|
||||
cd "$td"
|
||||
echo '#include <vamp-sdk/Plugin.h>' | gcc -x c -c -o /dev/null - || [ -e ~/pe/vamp-sdk ] || {
|
||||
printf '\033[33mcould not find the vamp-sdk, building from source\033[0m\n'
|
||||
(dl_files yolo https://code.soundsoftware.ac.uk/attachments/download/2588/vamp-plugin-sdk-2.9.0.tar.gz)
|
||||
sha512sum -c <(
|
||||
echo "7ef7f837d19a08048b059e0da408373a7964ced452b290fae40b85d6d70ca9000bcfb3302cd0b4dc76cf2a848528456f78c1ce1ee0c402228d812bd347b6983b -"
|
||||
) <vamp-plugin-sdk-2.9.0.tar.gz
|
||||
tar -xf vamp-plugin-sdk-2.9.0.tar.gz
|
||||
rm -- *.tar.gz
|
||||
ls -al
|
||||
cd vamp-plugin-sdk-*
|
||||
./configure --prefix=$HOME/pe/vamp-sdk
|
||||
make -j1 install
|
||||
}
|
||||
|
||||
cd "$td"
|
||||
have_beatroot || {
|
||||
printf '\033[33mcould not find the vamp beatroot plugin, building from source\033[0m\n'
|
||||
(dl_files yolo https://code.soundsoftware.ac.uk/attachments/download/885/beatroot-vamp-v1.0.tar.gz)
|
||||
@@ -245,11 +214,8 @@ install_vamp() {
|
||||
echo "1f444d1d58ccf565c0adfe99f1a1aa62789e19f5071e46857e2adfbc9d453037bc1c4dcb039b02c16240e9b97f444aaff3afb625c86aa2470233e711f55b6874 -"
|
||||
) <beatroot-vamp-v1.0.tar.gz
|
||||
tar -xf beatroot-vamp-v1.0.tar.gz
|
||||
rm -- *.tar.gz
|
||||
cd beatroot-vamp-v1.0
|
||||
[ -e ~/pe/vamp-sdk ] &&
|
||||
sed -ri 's`^(CFLAGS :=.*)`\1 -I'$HOME'/pe/vamp-sdk/include`' Makefile.linux
|
||||
make -f Makefile.linux -j4 LDFLAGS=-L$HOME/pe/vamp-sdk/lib
|
||||
make -f Makefile.linux -j4
|
||||
# /home/ed/vamp /home/ed/.vamp /usr/local/lib/vamp
|
||||
mkdir ~/vamp
|
||||
cp -pv beatroot-vamp.* ~/vamp/
|
||||
@@ -263,7 +229,6 @@ install_vamp() {
|
||||
|
||||
# not in use because it kinda segfaults, also no windows support
|
||||
install_soundtouch() {
|
||||
cd "$td"
|
||||
gitlab_tarball https://gitlab.com/api/v4/projects/soundtouch%2Fsoundtouch/releases
|
||||
|
||||
tar -xvf soundtouch-*
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import base64
|
||||
import hashlib
|
||||
import subprocess as sp
|
||||
|
||||
try:
|
||||
from copyparty.util import fsenc
|
||||
except:
|
||||
|
||||
def fsenc(p):
|
||||
return p.encode("utf-8")
|
||||
|
||||
|
||||
"""
|
||||
dep: ffmpeg
|
||||
"""
|
||||
|
||||
|
||||
def det():
|
||||
# fmt: off
|
||||
cmd = [
|
||||
b"ffmpeg",
|
||||
b"-nostdin",
|
||||
b"-hide_banner",
|
||||
b"-v", b"fatal",
|
||||
b"-i", fsenc(sys.argv[1]),
|
||||
b"-f", b"framemd5",
|
||||
b"-"
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
p = sp.Popen(cmd, stdout=sp.PIPE)
|
||||
# ps = io.TextIOWrapper(p.stdout, encoding="utf-8")
|
||||
ps = p.stdout
|
||||
|
||||
chans = {}
|
||||
for ln in ps:
|
||||
if ln.startswith(b"#stream#"):
|
||||
break
|
||||
|
||||
m = re.match(r"^#media_type ([0-9]): ([a-zA-Z])", ln.decode("utf-8"))
|
||||
if m:
|
||||
chans[m.group(1)] = m.group(2)
|
||||
|
||||
hashers = [hashlib.sha512(), hashlib.sha512()]
|
||||
for ln in ps:
|
||||
n = int(ln[:1])
|
||||
v = ln.rsplit(b",", 1)[-1].strip()
|
||||
hashers[n].update(v)
|
||||
|
||||
r = {}
|
||||
for k, v in chans.items():
|
||||
dg = hashers[int(k)].digest()[:12]
|
||||
dg = base64.urlsafe_b64encode(dg).decode("ascii")
|
||||
r[v[0].lower() + "hash"] = dg
|
||||
|
||||
print(json.dumps(r, indent=4))
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
det()
|
||||
except:
|
||||
pass # mute
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,38 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess as sp
|
||||
|
||||
|
||||
"""
|
||||
mtp test -- opens a texteditor
|
||||
|
||||
usage:
|
||||
-vsrv/v1:v1:r:c,mte=+x1:c,mtp=x1=ad,p,bin/mtag/mousepad.py
|
||||
|
||||
explained:
|
||||
c,mte: list of tags to index in this volume
|
||||
c,mtp: add new tag provider
|
||||
x1: dummy tag to provide
|
||||
ad: dontcare if audio or not
|
||||
p: priority 1 (run after initial tag-scan with ffprobe or mutagen)
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
env = os.environ.copy()
|
||||
env["DISPLAY"] = ":0.0"
|
||||
|
||||
if False:
|
||||
# open the uploaded file
|
||||
fp = sys.argv[-1]
|
||||
else:
|
||||
# display stdin contents (`oth_tags`)
|
||||
fp = "/dev/stdin"
|
||||
|
||||
p = sp.Popen(["/usr/bin/mousepad", fp])
|
||||
p.communicate()
|
||||
|
||||
|
||||
main()
|
||||
@@ -1,76 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess as sp
|
||||
import sys
|
||||
import time
|
||||
|
||||
try:
|
||||
from copyparty.util import fsenc
|
||||
except:
|
||||
|
||||
def fsenc(p):
|
||||
return p.encode("utf-8")
|
||||
|
||||
|
||||
_ = r"""
|
||||
first checks the tag "vidchk" which must be "ok" to continue,
|
||||
then uploads all files to some cloud storage (RCLONE_REMOTE)
|
||||
and DELETES THE ORIGINAL FILES if rclone returns 0 ("success")
|
||||
|
||||
deps:
|
||||
rclone
|
||||
|
||||
usage:
|
||||
-mtp x2=t43200,ay,p2,bin/mtag/rclone-upload.py
|
||||
|
||||
explained:
|
||||
t43200: timeout 12h
|
||||
ay: only process files which contain audio (including video with audio)
|
||||
p2: set priority 2 (after vidchk's suggested priority of 1),
|
||||
so the output of vidchk will be passed in here
|
||||
|
||||
complete usage example as vflags along with vidchk:
|
||||
-vsrv/vidchk:vidchk:r:rw,ed:c,e2dsa,e2ts,mtp=vidchk=t600,p,bin/mtag/vidchk.py:c,mtp=rupload=t43200,ay,p2,bin/mtag/rclone-upload.py:c,mte=+vidchk,rupload
|
||||
|
||||
setup: see https://rclone.org/drive/
|
||||
|
||||
if you wanna use this script standalone / separately from copyparty,
|
||||
either set CONDITIONAL_UPLOAD False or provide the following stdin:
|
||||
{"vidchk":"ok"}
|
||||
"""
|
||||
|
||||
|
||||
RCLONE_REMOTE = "notmybox"
|
||||
CONDITIONAL_UPLOAD = True
|
||||
|
||||
|
||||
def main():
|
||||
fp = sys.argv[1]
|
||||
if CONDITIONAL_UPLOAD:
|
||||
zb = sys.stdin.buffer.read()
|
||||
zs = zb.decode("utf-8", "replace")
|
||||
md = json.loads(zs)
|
||||
|
||||
chk = md.get("vidchk", None)
|
||||
if chk != "ok":
|
||||
print(f"vidchk={chk}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
dst = f"{RCLONE_REMOTE}:".encode("utf-8")
|
||||
cmd = [b"rclone", b"copy", b"--", fsenc(fp), dst]
|
||||
|
||||
t0 = time.time()
|
||||
try:
|
||||
sp.check_call(cmd)
|
||||
except:
|
||||
print("rclone failed", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"{time.time() - t0:.1f} sec")
|
||||
os.unlink(fsenc(fp))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,21 +0,0 @@
|
||||
// ==UserScript==
|
||||
// @name twitter-unmute
|
||||
// @namespace http://ocv.me/
|
||||
// @version 0.1
|
||||
// @description memes
|
||||
// @author ed <irc.rizon.net>
|
||||
// @match https://twitter.com/*
|
||||
// @icon https://www.google.com/s2/favicons?domain=twitter.com
|
||||
// @grant GM_addStyle
|
||||
// ==/UserScript==
|
||||
|
||||
function grunnur() {
|
||||
setInterval(function () {
|
||||
//document.querySelector('div[aria-label="Unmute"]').click();
|
||||
document.querySelector('video').muted = false;
|
||||
}, 200);
|
||||
}
|
||||
|
||||
var scr = document.createElement('script');
|
||||
scr.textContent = '(' + grunnur.toString() + ')();';
|
||||
(document.head || document.getElementsByTagName('head')[0]).appendChild(scr);
|
||||
@@ -1,39 +0,0 @@
|
||||
# example config file to use copyparty as a youtube manifest collector,
|
||||
# use with copyparty like: python copyparty.py -c yt-ipr.conf
|
||||
#
|
||||
# see docs/example.conf for a better explanation of the syntax, but
|
||||
# newlines are block separators, so adding blank lines inside a volume definition is bad
|
||||
# (use comments as separators instead)
|
||||
|
||||
|
||||
# create user ed, password wark
|
||||
u ed:wark
|
||||
|
||||
|
||||
# create a volume at /ytm which stores files at ./srv/ytm
|
||||
./srv/ytm
|
||||
/ytm
|
||||
# write-only, but read-write for user ed
|
||||
w
|
||||
rw ed
|
||||
# rescan the volume on startup
|
||||
c e2dsa
|
||||
# collect tags from all new files since last scan
|
||||
c e2ts
|
||||
# optionally enable compression to make the files 50% smaller
|
||||
c pk
|
||||
# only allow uploads which are between 16k and 1m large
|
||||
c sz=16k-1m
|
||||
# allow up to 10 uploads over 5 minutes from each ip
|
||||
c maxn=10,300
|
||||
# move uploads into subfolders: YEAR-MONTH / DAY-HOUR / <upload>
|
||||
c rotf=%Y-%m/%d-%H
|
||||
# delete uploads when they are 24 hours old
|
||||
c lifetime=86400
|
||||
# add the parser and tell copyparty what tags it can expect from it
|
||||
c mtp=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires=bin/mtag/yt-ipr.py
|
||||
# decide which tags we want to index and in what order
|
||||
c mte=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires
|
||||
|
||||
|
||||
# create any other volumes you'd like down here, or merge this with an existing config file
|
||||
@@ -1,47 +0,0 @@
|
||||
// ==UserScript==
|
||||
// @name youtube-playerdata-hub
|
||||
// @match https://youtube.com/*
|
||||
// @match https://*.youtube.com/*
|
||||
// @version 1.0
|
||||
// @grant GM_addStyle
|
||||
// ==/UserScript==
|
||||
|
||||
function main() {
|
||||
var server = 'https://127.0.0.1:3923/ytm?pw=wark',
|
||||
interval = 60; // sec
|
||||
|
||||
var sent = {};
|
||||
function send(txt, mf_url, desc) {
|
||||
if (sent[mf_url])
|
||||
return;
|
||||
|
||||
fetch(server + '&_=' + Date.now(), { method: "PUT", body: txt });
|
||||
console.log('[yt-pdh] yeet %d bytes, %s', txt.length, desc);
|
||||
sent[mf_url] = 1;
|
||||
}
|
||||
|
||||
function collect() {
|
||||
try {
|
||||
var pd = document.querySelector('ytd-watch-flexy');
|
||||
if (!pd)
|
||||
return console.log('[yt-pdh] no video found');
|
||||
|
||||
pd = pd.playerData;
|
||||
var mu = pd.streamingData.dashManifestUrl || pd.streamingData.hlsManifestUrl;
|
||||
if (!mu || !mu.length)
|
||||
return console.log('[yt-pdh] no manifest found');
|
||||
|
||||
var desc = pd.videoDetails.videoId + ', ' + pd.videoDetails.title;
|
||||
send(JSON.stringify(pd), mu, desc);
|
||||
}
|
||||
catch (ex) {
|
||||
console.log("[yt-pdh]", ex);
|
||||
}
|
||||
}
|
||||
setInterval(collect, interval * 1000);
|
||||
}
|
||||
|
||||
var scr = document.createElement('script');
|
||||
scr.textContent = '(' + main.toString() + ')();';
|
||||
(document.head || document.getElementsByTagName('head')[0]).appendChild(scr);
|
||||
console.log('[yt-pdh] a');
|
||||
@@ -1,139 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
use copyparty as a chromecast replacement:
|
||||
* post a URL and it will open in the default browser
|
||||
* upload a file and it will open in the default application
|
||||
* the `key` command simulates keyboard input
|
||||
* the `x` command executes other xdotool commands
|
||||
* the `c` command executes arbitrary unix commands
|
||||
|
||||
the android app makes it a breeze to post pics and links:
|
||||
https://github.com/9001/party-up/releases
|
||||
(iOS devices have to rely on the web-UI)
|
||||
|
||||
goes without saying, but this is HELLA DANGEROUS,
|
||||
GIVES RCE TO ANYONE WHO HAVE UPLOAD PERMISSIONS
|
||||
|
||||
example copyparty config to use this:
|
||||
--urlform save,get -v.::w:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,kn,c0,bin/mtag/very-bad-idea.py
|
||||
|
||||
recommended deps:
|
||||
apt install xdotool libnotify-bin
|
||||
https://github.com/9001/copyparty/blob/hovudstraum/contrib/plugins/meadup.js
|
||||
|
||||
and you probably want `twitter-unmute.user.js` from the res folder
|
||||
|
||||
|
||||
-----------------------------------------------------------------------
|
||||
-- startup script:
|
||||
-----------------------------------------------------------------------
|
||||
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# create qr code
|
||||
ip=$(ip r | awk '/^default/{print$(NF-2)}'); echo http://$ip:3923/ | qrencode -o - -s 4 >/dev/shm/cpp-qr.png
|
||||
/usr/bin/feh -x /dev/shm/cpp-qr.png &
|
||||
|
||||
# reposition and make topmost (with janky raspbian support)
|
||||
( sleep 0.5
|
||||
xdotool search --name cpp-qr.png windowactivate --sync windowmove 1780 0
|
||||
wmctrl -r :ACTIVE: -b toggle,above || true
|
||||
|
||||
ps aux | grep -E 'sleep[ ]7\.27' ||
|
||||
while true; do
|
||||
w=$(xdotool getactivewindow)
|
||||
xdotool search --name cpp-qr.png windowactivate windowraise windowfocus
|
||||
xdotool windowactivate $w
|
||||
xdotool windowfocus $w
|
||||
sleep 7.27 || break
|
||||
done &
|
||||
xeyes # distraction window to prevent ^w from closing the qr-code
|
||||
) &
|
||||
|
||||
# bail if copyparty is already running
|
||||
ps aux | grep -E '[3] copy[p]arty' && exit 0
|
||||
|
||||
# dumb chrome wrapper to allow autoplay
|
||||
cat >/usr/local/bin/chromium-browser <<'EOF'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
/usr/bin/chromium-browser --autoplay-policy=no-user-gesture-required "$@"
|
||||
EOF
|
||||
chmod 755 /usr/local/bin/chromium-browser
|
||||
|
||||
# start the server (note: replace `-v.::rw:` with `-v.::w:` to disallow retrieving uploaded stuff)
|
||||
cd ~/Downloads; python3 copyparty-sfx.py --urlform save,get -v.::rw:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,kn,very-bad-idea.py
|
||||
|
||||
"""
|
||||
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import subprocess as sp
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
|
||||
|
||||
def main():
|
||||
fp = os.path.abspath(sys.argv[1])
|
||||
with open(fp, "rb") as f:
|
||||
txt = f.read(4096)
|
||||
|
||||
if txt.startswith(b"msg="):
|
||||
open_post(txt)
|
||||
else:
|
||||
open_url(fp)
|
||||
|
||||
|
||||
def open_post(txt):
|
||||
txt = unquote(txt.replace(b"+", b" ")).decode("utf-8")[4:]
|
||||
try:
|
||||
k, v = txt.split(" ", 1)
|
||||
except:
|
||||
open_url(txt)
|
||||
|
||||
if k == "key":
|
||||
sp.call(["xdotool", "key"] + v.split(" "))
|
||||
elif k == "x":
|
||||
sp.call(["xdotool"] + v.split(" "))
|
||||
elif k == "c":
|
||||
env = os.environ.copy()
|
||||
while " " in v:
|
||||
v1, v2 = v.split(" ", 1)
|
||||
if "=" not in v1:
|
||||
break
|
||||
|
||||
ek, ev = v1.split("=", 1)
|
||||
env[ek] = ev
|
||||
v = v2
|
||||
|
||||
sp.call(v.split(" "), env=env)
|
||||
else:
|
||||
open_url(txt)
|
||||
|
||||
|
||||
def open_url(txt):
|
||||
ext = txt.rsplit(".")[-1].lower()
|
||||
sp.call(["notify-send", "--", txt])
|
||||
if ext not in ["jpg", "jpeg", "png", "gif", "webp"]:
|
||||
# sp.call(["wmctrl", "-c", ":ACTIVE:"]) # closes the active window correctly
|
||||
sp.call(["killall", "vlc"])
|
||||
sp.call(["killall", "mpv"])
|
||||
sp.call(["killall", "feh"])
|
||||
time.sleep(0.5)
|
||||
for _ in range(20):
|
||||
sp.call(["xdotool", "key", "ctrl+w"]) # closes the open tab correctly
|
||||
# else:
|
||||
# sp.call(["xdotool", "getactivewindow", "windowminimize"]) # minimizes the focused windo
|
||||
|
||||
# close any error messages:
|
||||
sp.call(["xdotool", "search", "--name", "Error", "windowclose"])
|
||||
# sp.call(["xdotool", "key", "ctrl+alt+d"]) # doesnt work at all
|
||||
# sp.call(["xdotool", "keydown", "--delay", "100", "ctrl+alt+d"])
|
||||
# sp.call(["xdotool", "keyup", "ctrl+alt+d"])
|
||||
sp.call(["xdg-open", txt])
|
||||
|
||||
|
||||
main()
|
||||
@@ -1,131 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import subprocess as sp
|
||||
|
||||
try:
|
||||
from copyparty.util import fsenc
|
||||
except:
|
||||
|
||||
def fsenc(p):
|
||||
return p.encode("utf-8")
|
||||
|
||||
|
||||
_ = r"""
|
||||
inspects video files for errors and such
|
||||
plus stores a bunch of metadata to filename.ff.json
|
||||
|
||||
usage:
|
||||
-mtp vidchk=t600,ay,p,bin/mtag/vidchk.py
|
||||
|
||||
explained:
|
||||
t600: timeout 10min
|
||||
ay: only process files which contain audio (including video with audio)
|
||||
p: set priority 1 (lowest priority after initial ffprobe/mutagen for base tags),
|
||||
makes copyparty feed base tags into this script as json
|
||||
|
||||
if you wanna use this script standalone / separately from copyparty,
|
||||
provide the video resolution on stdin as json: {"res":"1920x1080"}
|
||||
"""
|
||||
|
||||
|
||||
FAST = True # parse entire file at container level
|
||||
# FAST = False # fully decode audio and video streams
|
||||
|
||||
|
||||
# warnings to ignore
|
||||
harmless = re.compile(
|
||||
r"Unsupported codec with id |Could not find codec parameters.*Attachment:|analyzeduration"
|
||||
+ r"|timescale not set"
|
||||
)
|
||||
|
||||
|
||||
def wfilter(lines):
|
||||
return [x for x in lines if x.strip() and not harmless.search(x)]
|
||||
|
||||
|
||||
def errchk(so, se, rc, dbg):
|
||||
if dbg:
|
||||
with open(dbg, "wb") as f:
|
||||
f.write(b"so:\n" + so + b"\nse:\n" + se + b"\n")
|
||||
|
||||
if rc:
|
||||
err = (so + se).decode("utf-8", "replace").split("\n", 1)
|
||||
err = wfilter(err) or err
|
||||
return f"ERROR {rc}: {err[0]}"
|
||||
|
||||
if se:
|
||||
err = se.decode("utf-8", "replace").split("\n", 1)
|
||||
err = wfilter(err)
|
||||
if err:
|
||||
return f"Warning: {err[0]}"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
fp = sys.argv[1]
|
||||
zb = sys.stdin.buffer.read()
|
||||
zs = zb.decode("utf-8", "replace")
|
||||
md = json.loads(zs)
|
||||
|
||||
fdir = os.path.dirname(os.path.realpath(fp))
|
||||
flag = os.path.join(fdir, ".processed")
|
||||
if os.path.exists(flag):
|
||||
return "already processed"
|
||||
|
||||
try:
|
||||
w, h = [int(x) for x in md["res"].split("x")]
|
||||
if not w + h:
|
||||
raise Exception()
|
||||
except:
|
||||
return "could not determine resolution"
|
||||
|
||||
# grab streams/format metadata + 2 seconds of frames at the start and end
|
||||
zs = "ffprobe -hide_banner -v warning -of json -show_streams -show_format -show_packets -show_data_hash crc32 -read_intervals %+2,999999%+2"
|
||||
cmd = zs.encode("ascii").split(b" ") + [fsenc(fp)]
|
||||
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
|
||||
so, se = p.communicate()
|
||||
|
||||
# spaces to tabs, drops filesize from 69k to 48k
|
||||
so = b"\n".join(
|
||||
[
|
||||
b"\t" * int((len(x) - len(x.lstrip())) / 4) + x.lstrip()
|
||||
for x in (so or b"").split(b"\n")
|
||||
]
|
||||
)
|
||||
with open(fsenc(f"{fp}.ff.json"), "wb") as f:
|
||||
f.write(so)
|
||||
|
||||
err = errchk(so, se, p.returncode, f"{fp}.vidchk")
|
||||
if err:
|
||||
return err
|
||||
|
||||
if max(w, h) < 1280 and min(w, h) < 720:
|
||||
return "resolution too small"
|
||||
|
||||
zs = (
|
||||
"ffmpeg -y -hide_banner -nostdin -v warning"
|
||||
+ " -err_detect +crccheck+bitstream+buffer+careful+compliant+aggressive+explode"
|
||||
+ " -xerror -i"
|
||||
)
|
||||
|
||||
cmd = zs.encode("ascii").split(b" ") + [fsenc(fp)]
|
||||
|
||||
if FAST:
|
||||
zs = "-c copy -f null -"
|
||||
else:
|
||||
zs = "-vcodec rawvideo -acodec pcm_s16le -f null -"
|
||||
|
||||
cmd += zs.encode("ascii").split(b" ")
|
||||
|
||||
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
|
||||
so, se = p.communicate()
|
||||
return errchk(so, se, p.returncode, f"{fp}.vidchk")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(main() or "ok")
|
||||
@@ -1,85 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
use copyparty as a file downloader by POSTing URLs as
|
||||
application/x-www-form-urlencoded (for example using the
|
||||
message/pager function on the website)
|
||||
|
||||
example copyparty config to use this:
|
||||
--urlform save,get -vsrv/wget:wget:rwmd,ed:c,e2ts,mtp=title=ebin,t300,ad,bin/mtag/wget.py
|
||||
|
||||
explained:
|
||||
for realpath srv/wget (served at /wget) with read-write-modify-delete for ed,
|
||||
enable file analysis on upload (e2ts),
|
||||
use mtp plugin "bin/mtag/wget.py" to provide metadata tag "title",
|
||||
do this on all uploads with the file extension "bin",
|
||||
t300 = 300 seconds timeout for each dwonload,
|
||||
ad = parse file regardless if FFmpeg thinks it is audio or not
|
||||
|
||||
PS: this requires e2ts to be functional,
|
||||
meaning you need to do at least one of these:
|
||||
* apt install ffmpeg
|
||||
* pip3 install mutagen
|
||||
"""
|
||||
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess as sp
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
|
||||
|
||||
def main():
|
||||
fp = os.path.abspath(sys.argv[1])
|
||||
fdir = os.path.dirname(fp)
|
||||
fname = os.path.basename(fp)
|
||||
if not fname.startswith("put-") or not fname.endswith(".bin"):
|
||||
raise Exception("not a post file")
|
||||
|
||||
buf = b""
|
||||
with open(fp, "rb") as f:
|
||||
while True:
|
||||
b = f.read(4096)
|
||||
buf += b
|
||||
if len(buf) > 4096:
|
||||
raise Exception("too big")
|
||||
|
||||
if not b:
|
||||
break
|
||||
|
||||
if not buf:
|
||||
raise Exception("file is empty")
|
||||
|
||||
buf = unquote(buf.replace(b"+", b" "))
|
||||
url = buf.decode("utf-8")
|
||||
|
||||
if not url.startswith("msg="):
|
||||
raise Exception("does not start with msg=")
|
||||
|
||||
url = url[4:]
|
||||
if "://" not in url:
|
||||
url = "https://" + url
|
||||
|
||||
os.chdir(fdir)
|
||||
|
||||
name = url.split("?")[0].split("/")[-1]
|
||||
tfn = "-- DOWNLOADING " + name
|
||||
open(tfn, "wb").close()
|
||||
|
||||
cmd = ["wget", "--trust-server-names", "--", url]
|
||||
|
||||
try:
|
||||
sp.check_call(cmd)
|
||||
|
||||
# OPTIONAL:
|
||||
# on success, delete the .bin file which contains the URL
|
||||
os.unlink(fp)
|
||||
except:
|
||||
open("-- FAILED TO DONWLOAD " + name, "wb").close()
|
||||
|
||||
os.unlink(tfn)
|
||||
print(url)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,198 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import gzip
|
||||
import json
|
||||
import base64
|
||||
import string
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
|
||||
"""
|
||||
youtube initial player response
|
||||
|
||||
it's probably best to use this through a config file; see res/yt-ipr.conf
|
||||
|
||||
but if you want to use plain arguments instead then:
|
||||
-v srv/ytm:ytm:w:rw,ed
|
||||
:c,e2ts,e2dsa
|
||||
:c,sz=16k-1m:c,maxn=10,300:c,rotf=%Y-%m/%d-%H
|
||||
:c,mtp=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires=bin/mtag/yt-ipr.py
|
||||
:c,mte=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires
|
||||
|
||||
see res/yt-ipr.user.js for the example userscript to go with this
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
with gzip.open(sys.argv[1], "rt", encoding="utf-8", errors="replace") as f:
|
||||
txt = f.read()
|
||||
except:
|
||||
with open(sys.argv[1], "r", encoding="utf-8", errors="replace") as f:
|
||||
txt = f.read()
|
||||
|
||||
txt = "{" + txt.split("{", 1)[1]
|
||||
|
||||
try:
|
||||
pd = json.loads(txt)
|
||||
except json.decoder.JSONDecodeError as ex:
|
||||
pd = json.loads(txt[: ex.pos])
|
||||
|
||||
# print(json.dumps(pd, indent=2))
|
||||
|
||||
if "videoDetails" in pd:
|
||||
parse_youtube(pd)
|
||||
else:
|
||||
parse_freg(pd)
|
||||
|
||||
|
||||
def get_expiration(url):
|
||||
et = re.search(r"[?&]expire=([0-9]+)", url).group(1)
|
||||
et = datetime.utcfromtimestamp(int(et))
|
||||
return et.strftime("%Y-%m-%d, %H:%M")
|
||||
|
||||
|
||||
def parse_youtube(pd):
|
||||
vd = pd["videoDetails"]
|
||||
sd = pd["streamingData"]
|
||||
|
||||
et = sd["adaptiveFormats"][0]["url"]
|
||||
et = get_expiration(et)
|
||||
|
||||
mf = []
|
||||
if "dashManifestUrl" in sd:
|
||||
mf.append("dash")
|
||||
if "hlsManifestUrl" in sd:
|
||||
mf.append("hls")
|
||||
|
||||
r = {
|
||||
"yt-id": vd["videoId"],
|
||||
"yt-title": vd["title"],
|
||||
"yt-author": vd["author"],
|
||||
"yt-channel": vd["channelId"],
|
||||
"yt-views": vd["viewCount"],
|
||||
"yt-private": vd["isPrivate"],
|
||||
# "yt-expires": sd["expiresInSeconds"],
|
||||
"yt-manifest": ",".join(mf),
|
||||
"yt-expires": et,
|
||||
}
|
||||
print(json.dumps(r))
|
||||
|
||||
freg_conv(pd)
|
||||
|
||||
|
||||
def parse_freg(pd):
|
||||
md = pd["metadata"]
|
||||
r = {
|
||||
"yt-id": md["id"],
|
||||
"yt-title": md["title"],
|
||||
"yt-author": md["channelName"],
|
||||
"yt-channel": md["channelURL"].strip("/").split("/")[-1],
|
||||
"yt-expires": get_expiration(list(pd["video"].values())[0]),
|
||||
}
|
||||
print(json.dumps(r))
|
||||
|
||||
|
||||
def freg_conv(pd):
|
||||
# based on getURLs.js v1.5 (2021-08-07)
|
||||
# fmt: off
|
||||
priority = {
|
||||
"video": [
|
||||
337, 315, 266, 138, # 2160p60
|
||||
313, 336, # 2160p
|
||||
308, # 1440p60
|
||||
271, 264, # 1440p
|
||||
335, 303, 299, # 1080p60
|
||||
248, 169, 137, # 1080p
|
||||
334, 302, 298, # 720p60
|
||||
247, 136 # 720p
|
||||
],
|
||||
"audio": [
|
||||
251, 141, 171, 140, 250, 249, 139
|
||||
]
|
||||
}
|
||||
|
||||
vid_id = pd["videoDetails"]["videoId"]
|
||||
chan_id = pd["videoDetails"]["channelId"]
|
||||
|
||||
try:
|
||||
thumb_url = pd["microformat"]["playerMicroformatRenderer"]["thumbnail"]["thumbnails"][0]["url"]
|
||||
start_ts = pd["microformat"]["playerMicroformatRenderer"]["liveBroadcastDetails"]["startTimestamp"]
|
||||
except:
|
||||
thumb_url = f"https://img.youtube.com/vi/{vid_id}/maxresdefault.jpg"
|
||||
start_ts = ""
|
||||
|
||||
# fmt: on
|
||||
|
||||
metadata = {
|
||||
"title": pd["videoDetails"]["title"],
|
||||
"id": vid_id,
|
||||
"channelName": pd["videoDetails"]["author"],
|
||||
"channelURL": "https://www.youtube.com/channel/" + chan_id,
|
||||
"description": pd["videoDetails"]["shortDescription"],
|
||||
"thumbnailUrl": thumb_url,
|
||||
"startTimestamp": start_ts,
|
||||
}
|
||||
|
||||
if [x for x in vid_id if x not in string.ascii_letters + string.digits + "_-"]:
|
||||
print(f"malicious json", file=sys.stderr)
|
||||
return
|
||||
|
||||
basepath = os.path.dirname(sys.argv[1])
|
||||
|
||||
thumb_fn = f"{basepath}/{vid_id}.jpg"
|
||||
tmp_fn = f"{thumb_fn}.{os.getpid()}"
|
||||
if not os.path.exists(thumb_fn) and (
|
||||
thumb_url.startswith("https://img.youtube.com/vi/")
|
||||
or thumb_url.startswith("https://i.ytimg.com/vi/")
|
||||
):
|
||||
try:
|
||||
with urllib.request.urlopen(thumb_url) as fi:
|
||||
with open(tmp_fn, "wb") as fo:
|
||||
fo.write(fi.read())
|
||||
|
||||
os.rename(tmp_fn, thumb_fn)
|
||||
except:
|
||||
if os.path.exists(tmp_fn):
|
||||
os.unlink(tmp_fn)
|
||||
|
||||
try:
|
||||
with open(thumb_fn, "rb") as f:
|
||||
thumb = base64.b64encode(f.read()).decode("ascii")
|
||||
except:
|
||||
thumb = "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/yQALCAABAAEBAREA/8wABgAQEAX/2gAIAQEAAD8A0s8g/9k="
|
||||
|
||||
metadata["thumbnail"] = "data:image/jpeg;base64," + thumb
|
||||
|
||||
ret = {
|
||||
"metadata": metadata,
|
||||
"version": "1.5",
|
||||
"createTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
}
|
||||
|
||||
for stream, itags in priority.items():
|
||||
for itag in itags:
|
||||
url = None
|
||||
for afmt in pd["streamingData"]["adaptiveFormats"]:
|
||||
if itag == afmt["itag"]:
|
||||
url = afmt["url"]
|
||||
break
|
||||
|
||||
if url:
|
||||
ret[stream] = {itag: url}
|
||||
break
|
||||
|
||||
fn = f"{basepath}/{vid_id}.urls.json"
|
||||
with open(fn, "w", encoding="utf-8", errors="replace") as f:
|
||||
f.write(json.dumps(ret, indent=4))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except:
|
||||
# raise
|
||||
pass
|
||||
@@ -1,177 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
partyjournal.py: chronological history of uploads
|
||||
2021-12-31, v0.1, ed <irc.rizon.net>, MIT-Licensed
|
||||
https://github.com/9001/copyparty/blob/hovudstraum/bin/partyjournal.py
|
||||
|
||||
produces a chronological list of all uploads,
|
||||
by collecting info from up2k databases and the filesystem
|
||||
|
||||
specify subnet `192.168.1.*` with argument `.=192.168.1.`,
|
||||
affecting all successive mappings
|
||||
|
||||
usage:
|
||||
./partyjournal.py > partyjournal.html .=192.168.1. cart=125 steen=114 steen=131 sleepy=121 fscarlet=144 ed=101 ed=123
|
||||
|
||||
"""
|
||||
|
||||
import sys
|
||||
import base64
|
||||
import sqlite3
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
from urllib.parse import quote_from_bytes as quote
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
|
||||
|
||||
FS_ENCODING = sys.getfilesystemencoding()
|
||||
|
||||
|
||||
class APF(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
|
||||
pass
|
||||
|
||||
|
||||
##
|
||||
## snibbed from copyparty
|
||||
|
||||
|
||||
def s3dec(v):
|
||||
if not v.startswith("//"):
|
||||
return v
|
||||
|
||||
v = base64.urlsafe_b64decode(v.encode("ascii")[2:])
|
||||
return v.decode(FS_ENCODING, "replace")
|
||||
|
||||
|
||||
def quotep(txt):
|
||||
btxt = txt.encode("utf-8", "replace")
|
||||
quot1 = quote(btxt, safe=b"/")
|
||||
quot1 = quot1.encode("ascii")
|
||||
quot2 = quot1.replace(b" ", b"+")
|
||||
return quot2.decode("utf-8", "replace")
|
||||
|
||||
|
||||
def html_escape(s, quote=False, crlf=False):
|
||||
"""html.escape but also newlines"""
|
||||
s = s.replace("&", "&").replace("<", "<").replace(">", ">")
|
||||
if quote:
|
||||
s = s.replace('"', """).replace("'", "'")
|
||||
if crlf:
|
||||
s = s.replace("\r", " ").replace("\n", " ")
|
||||
|
||||
return s
|
||||
|
||||
|
||||
## end snibs
|
||||
##
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(formatter_class=APF)
|
||||
ap.add_argument("who", nargs="*")
|
||||
ar = ap.parse_args()
|
||||
|
||||
imap = {}
|
||||
subnet = ""
|
||||
for v in ar.who:
|
||||
if "=" not in v:
|
||||
raise Exception("bad who: " + v)
|
||||
|
||||
k, v = v.split("=")
|
||||
if k == ".":
|
||||
subnet = v
|
||||
continue
|
||||
|
||||
imap["{}{}".format(subnet, v)] = k
|
||||
|
||||
print(repr(imap), file=sys.stderr)
|
||||
|
||||
print(
|
||||
"""\
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head><meta charset="utf-8"><style>
|
||||
|
||||
html, body {
|
||||
color: #ccc;
|
||||
background: #222;
|
||||
font-family: sans-serif;
|
||||
}
|
||||
a {
|
||||
color: #fc5;
|
||||
}
|
||||
td, th {
|
||||
padding: .2em .5em;
|
||||
border: 1px solid #999;
|
||||
border-width: 0 1px 1px 0;
|
||||
white-space: nowrap;
|
||||
}
|
||||
td:nth-child(1),
|
||||
td:nth-child(2),
|
||||
td:nth-child(3) {
|
||||
font-family: monospace, monospace;
|
||||
text-align: right;
|
||||
}
|
||||
tr:first-child {
|
||||
position: sticky;
|
||||
top: -1px;
|
||||
}
|
||||
th {
|
||||
background: #222;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
</style></head><body><table><tr>
|
||||
<th>wark</th>
|
||||
<th>time</th>
|
||||
<th>size</th>
|
||||
<th>who</th>
|
||||
<th>link</th>
|
||||
</tr>"""
|
||||
)
|
||||
|
||||
db_path = ".hist/up2k.db"
|
||||
conn = sqlite3.connect(db_path)
|
||||
q = r"pragma table_info(up)"
|
||||
inf = conn.execute(q).fetchall()
|
||||
cols = [x[1] for x in inf]
|
||||
print("<!-- " + str(cols) + " -->")
|
||||
# ['w', 'mt', 'sz', 'rd', 'fn', 'ip', 'at']
|
||||
|
||||
q = r"select * from up order by case when at > 0 then at else mt end"
|
||||
for w, mt, sz, rd, fn, ip, at in conn.execute(q):
|
||||
link = "/".join([s3dec(x) for x in [rd, fn] if x])
|
||||
if fn.startswith("put-") and sz < 4096:
|
||||
try:
|
||||
with open(link, "rb") as f:
|
||||
txt = f.read().decode("utf-8", "replace")
|
||||
except:
|
||||
continue
|
||||
|
||||
if txt.startswith("msg="):
|
||||
txt = txt.encode("utf-8", "replace")
|
||||
txt = unquote(txt.replace(b"+", b" "))
|
||||
link = txt.decode("utf-8")[4:]
|
||||
|
||||
sz = "{:,}".format(sz)
|
||||
v = [
|
||||
w[:16],
|
||||
datetime.utcfromtimestamp(at if at > 0 else mt).strftime(
|
||||
"%Y-%m-%d %H:%M:%S"
|
||||
),
|
||||
sz,
|
||||
imap.get(ip, ip),
|
||||
]
|
||||
|
||||
row = "<tr>\n "
|
||||
row += "\n ".join(["<td>{}</th>".format(x) for x in v])
|
||||
row += '\n <td><a href="{}">{}</a></td>'.format(link, html_escape(link))
|
||||
row += "\n</tr>"
|
||||
print(row)
|
||||
|
||||
print("</table></body></html>")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,128 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# runs copyparty (or any other program really) in a chroot
|
||||
#
|
||||
# assumption: these directories, and everything within, are owned by root
|
||||
sysdirs=( /bin /lib /lib32 /lib64 /sbin /usr )
|
||||
|
||||
|
||||
# error-handler
|
||||
help() { cat <<'EOF'
|
||||
|
||||
usage:
|
||||
./prisonparty.sh <ROOTDIR> <UID> <GID> [VOLDIR [VOLDIR...]] -- python3 copyparty-sfx.py [...]
|
||||
|
||||
example:
|
||||
./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 copyparty-sfx.py -v /mnt/nas/music::rwmd
|
||||
|
||||
example for running straight from source (instead of using an sfx):
|
||||
PYTHONPATH=$PWD ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 -um copyparty -v /mnt/nas/music::rwmd
|
||||
|
||||
note that if you have python modules installed as --user (such as bpm/key detectors),
|
||||
you should add /home/foo/.local as a VOLDIR
|
||||
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
|
||||
# read arguments
|
||||
trap help EXIT
|
||||
jail="$(realpath "$1")"; shift
|
||||
uid="$1"; shift
|
||||
gid="$1"; shift
|
||||
|
||||
vols=()
|
||||
while true; do
|
||||
v="$1"; shift
|
||||
[ "$v" = -- ] && break # end of volumes
|
||||
[ "$#" -eq 0 ] && break # invalid usage
|
||||
vols+=( "$(realpath "$v")" )
|
||||
done
|
||||
pybin="$1"; shift
|
||||
pybin="$(command -v "$pybin")"
|
||||
pyarg=
|
||||
while true; do
|
||||
v="$1"
|
||||
[ "${v:0:1}" = - ] || break
|
||||
pyarg="$pyarg $v"
|
||||
shift
|
||||
done
|
||||
cpp="$1"; shift
|
||||
[ -d "$cpp" ] && cppdir="$PWD" || {
|
||||
# sfx, not module
|
||||
cpp="$(realpath "$cpp")"
|
||||
cppdir="$(dirname "$cpp")"
|
||||
}
|
||||
trap - EXIT
|
||||
|
||||
|
||||
# debug/vis
|
||||
echo
|
||||
echo "chroot-dir = $jail"
|
||||
echo "user:group = $uid:$gid"
|
||||
echo " copyparty = $cpp"
|
||||
echo
|
||||
printf '\033[33m%s\033[0m\n' "copyparty can access these folders and all their subdirectories:"
|
||||
for v in "${vols[@]}"; do
|
||||
printf '\033[36m ├─\033[0m %s \033[36m ── added by (You)\033[0m\n' "$v"
|
||||
done
|
||||
printf '\033[36m ├─\033[0m %s \033[36m ── where the copyparty binary is\033[0m\n' "$cppdir"
|
||||
printf '\033[36m ╰─\033[0m %s \033[36m ── the folder you are currently in\033[0m\n' "$PWD"
|
||||
vols+=("$cppdir" "$PWD")
|
||||
echo
|
||||
|
||||
|
||||
# remove any trailing slashes
|
||||
jail="${jail%/}"
|
||||
|
||||
|
||||
# bind-mount system directories and volumes
|
||||
printf '%s\n' "${sysdirs[@]}" "${vols[@]}" | sed -r 's`/$``' | LC_ALL=C sort | uniq |
|
||||
while IFS= read -r v; do
|
||||
[ -e "$v" ] || {
|
||||
# printf '\033[1;31mfolder does not exist:\033[0m %s\n' "/$v"
|
||||
continue
|
||||
}
|
||||
i1=$(stat -c%D.%i "$v" 2>/dev/null || echo a)
|
||||
i2=$(stat -c%D.%i "$jail$v" 2>/dev/null || echo b)
|
||||
# echo "v [$v] i1 [$i1] i2 [$i2]"
|
||||
[ $i1 = $i2 ] && continue
|
||||
|
||||
mkdir -p "$jail$v"
|
||||
mount --bind "$v" "$jail$v"
|
||||
done
|
||||
|
||||
|
||||
cln() {
|
||||
rv=$?
|
||||
# cleanup if not in use
|
||||
lsof "$jail" | grep -qF "$jail" &&
|
||||
echo "chroot is in use, will not cleanup" ||
|
||||
{
|
||||
mount | grep -F " on $jail" |
|
||||
awk '{sub(/ type .*/,"");sub(/.* on /,"");print}' |
|
||||
LC_ALL=C sort -r | tee /dev/stderr | tr '\n' '\0' | xargs -r0 umount
|
||||
}
|
||||
exit $rv
|
||||
}
|
||||
trap cln EXIT
|
||||
|
||||
|
||||
# create a tmp
|
||||
mkdir -p "$jail/tmp"
|
||||
chmod 777 "$jail/tmp"
|
||||
|
||||
|
||||
# run copyparty
|
||||
export HOME=$(getent passwd $uid | cut -d: -f6)
|
||||
export USER=$(getent passwd $uid | cut -d: -f1)
|
||||
export LOGNAME="$USER"
|
||||
#echo "pybin [$pybin]"
|
||||
#echo "pyarg [$pyarg]"
|
||||
#echo "cpp [$cpp]"
|
||||
chroot --userspec=$uid:$gid "$jail" "$pybin" $pyarg "$cpp" "$@" &
|
||||
p=$!
|
||||
trap 'kill $p' INT TERM
|
||||
wait
|
||||
@@ -1,99 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
unforget.py: rebuild db from logfiles
|
||||
2022-09-07, v0.1, ed <irc.rizon.net>, MIT-Licensed
|
||||
https://github.com/9001/copyparty/blob/hovudstraum/bin/unforget.py
|
||||
|
||||
only makes sense if running copyparty with --no-forget
|
||||
(e.g. immediately shifting uploads to other storage)
|
||||
|
||||
usage:
|
||||
xz -d < log | ./unforget.py .hist/up2k.db
|
||||
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import base64
|
||||
import sqlite3
|
||||
import argparse
|
||||
|
||||
|
||||
FS_ENCODING = sys.getfilesystemencoding()
|
||||
|
||||
|
||||
class APF(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
|
||||
pass
|
||||
|
||||
|
||||
mem_cur = sqlite3.connect(":memory:").cursor()
|
||||
mem_cur.execute(r"create table a (b text)")
|
||||
|
||||
|
||||
def s3enc(rd: str, fn: str) -> tuple[str, str]:
|
||||
ret: list[str] = []
|
||||
for v in [rd, fn]:
|
||||
try:
|
||||
mem_cur.execute("select * from a where b = ?", (v,))
|
||||
ret.append(v)
|
||||
except:
|
||||
wtf8 = v.encode(FS_ENCODING, "surrogateescape")
|
||||
ret.append("//" + base64.urlsafe_b64encode(wtf8).decode("ascii"))
|
||||
|
||||
return ret[0], ret[1]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("db")
|
||||
ar = ap.parse_args()
|
||||
|
||||
db = sqlite3.connect(ar.db).cursor()
|
||||
ptn_times = re.compile(r"no more chunks, setting times \(([0-9]+)")
|
||||
at = 0
|
||||
ctr = 0
|
||||
|
||||
for ln in [x.decode("utf-8", "replace").rstrip() for x in sys.stdin.buffer]:
|
||||
if "no more chunks, setting times (" in ln:
|
||||
m = ptn_times.search(ln)
|
||||
if m:
|
||||
at = int(m.group(1))
|
||||
|
||||
if '"hash": []' in ln:
|
||||
try:
|
||||
ofs = ln.find("{")
|
||||
j = json.loads(ln[ofs:])
|
||||
except:
|
||||
pass
|
||||
|
||||
w = j["wark"]
|
||||
if db.execute("select w from up where w = ?", (w,)).fetchone():
|
||||
continue
|
||||
|
||||
# PYTHONPATH=/home/ed/dev/copyparty/ python3 -m copyparty -e2dsa -v foo:foo:rwmd,ed -aed:wark --no-forget
|
||||
# 05:34:43.845 127.0.0.1 42496 no more chunks, setting times (1662528883, 1658001882)
|
||||
# 05:34:43.863 127.0.0.1 42496 {"name": "f\"2", "purl": "/foo/bar/baz/", "size": 1674, "lmod": 1658001882, "sprs": true, "hash": [], "wark": "LKIWpp2jEAh9dH3fu-DobuURFGEKlODXDGTpZ1otMhUg"}
|
||||
# | w | mt | sz | rd | fn | ip | at |
|
||||
# | LKIWpp2jEAh9dH3fu-DobuURFGEKlODXDGTpZ1otMhUg | 1658001882 | 1674 | bar/baz | f"2 | 127.0.0.1 | 1662528883 |
|
||||
|
||||
rd, fn = s3enc(j["purl"].strip("/"), j["name"])
|
||||
ip = ln.split(" ")[1].split("m")[-1]
|
||||
|
||||
q = "insert into up values (?,?,?,?,?,?,?)"
|
||||
v = (w, int(j["lmod"]), int(j["size"]), rd, fn, ip, at)
|
||||
db.execute(q, v)
|
||||
ctr += 1
|
||||
if ctr % 1024 == 1023:
|
||||
print(f"{ctr} commit...")
|
||||
db.connection.commit()
|
||||
|
||||
if ctr:
|
||||
db.connection.commit()
|
||||
|
||||
print(f"unforgot {ctr} files")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
964
bin/up2k.py
964
bin/up2k.py
@@ -1,964 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
"""
|
||||
up2k.py: upload to copyparty
|
||||
2022-09-05, v0.19, ed <irc.rizon.net>, MIT-Licensed
|
||||
https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py
|
||||
|
||||
- dependencies: requests
|
||||
- supports python 2.6, 2.7, and 3.3 through 3.11
|
||||
|
||||
- almost zero error-handling
|
||||
- but if something breaks just try again and it'll autoresume
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import stat
|
||||
import math
|
||||
import time
|
||||
import atexit
|
||||
import signal
|
||||
import base64
|
||||
import hashlib
|
||||
import platform
|
||||
import threading
|
||||
import datetime
|
||||
|
||||
try:
|
||||
import argparse
|
||||
except:
|
||||
m = "\n ERROR: need 'argparse'; download it here:\n https://github.com/ThomasWaldmann/argparse/raw/master/argparse.py\n"
|
||||
print(m)
|
||||
raise
|
||||
|
||||
try:
|
||||
import requests
|
||||
except ImportError:
|
||||
if sys.version_info > (2, 7):
|
||||
m = "\nERROR: need 'requests'; please run this command:\n {0} -m pip install --user requests\n"
|
||||
else:
|
||||
m = "requests/2.18.4 urllib3/1.23 chardet/3.0.4 certifi/2020.4.5.1 idna/2.7"
|
||||
m = [" https://pypi.org/project/" + x + "/#files" for x in m.split()]
|
||||
m = "\n ERROR: need these:\n" + "\n".join(m) + "\n"
|
||||
|
||||
print(m.format(sys.executable))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# from copyparty/__init__.py
|
||||
PY2 = sys.version_info[0] == 2
|
||||
if PY2:
|
||||
from Queue import Queue
|
||||
from urllib import unquote
|
||||
from urllib import quote
|
||||
|
||||
sys.dont_write_bytecode = True
|
||||
bytes = str
|
||||
else:
|
||||
from queue import Queue
|
||||
from urllib.parse import unquote_to_bytes as unquote
|
||||
from urllib.parse import quote_from_bytes as quote
|
||||
|
||||
unicode = str
|
||||
|
||||
VT100 = platform.system() != "Windows"
|
||||
|
||||
|
||||
req_ses = requests.Session()
|
||||
|
||||
|
||||
class File(object):
|
||||
"""an up2k upload task; represents a single file"""
|
||||
|
||||
def __init__(self, top, rel, size, lmod):
|
||||
self.top = top # type: bytes
|
||||
self.rel = rel.replace(b"\\", b"/") # type: bytes
|
||||
self.size = size # type: int
|
||||
self.lmod = lmod # type: float
|
||||
|
||||
self.abs = os.path.join(top, rel) # type: bytes
|
||||
self.name = self.rel.split(b"/")[-1].decode("utf-8", "replace") # type: str
|
||||
|
||||
# set by get_hashlist
|
||||
self.cids = [] # type: list[tuple[str, int, int]] # [ hash, ofs, sz ]
|
||||
self.kchunks = {} # type: dict[str, tuple[int, int]] # hash: [ ofs, sz ]
|
||||
|
||||
# set by handshake
|
||||
self.ucids = [] # type: list[str] # chunks which need to be uploaded
|
||||
self.wark = None # type: str
|
||||
self.url = None # type: str
|
||||
|
||||
# set by upload
|
||||
self.up_b = 0 # type: int
|
||||
self.up_c = 0 # type: int
|
||||
|
||||
# t = "size({}) lmod({}) top({}) rel({}) abs({}) name({})\n"
|
||||
# eprint(t.format(self.size, self.lmod, self.top, self.rel, self.abs, self.name))
|
||||
|
||||
|
||||
class FileSlice(object):
|
||||
"""file-like object providing a fixed window into a file"""
|
||||
|
||||
def __init__(self, file, cid):
|
||||
# type: (File, str) -> None
|
||||
|
||||
self.car, self.len = file.kchunks[cid]
|
||||
self.cdr = self.car + self.len
|
||||
self.ofs = 0 # type: int
|
||||
self.f = open(file.abs, "rb", 512 * 1024)
|
||||
self.f.seek(self.car)
|
||||
|
||||
# https://stackoverflow.com/questions/4359495/what-is-exactly-a-file-like-object-in-python
|
||||
# IOBase, RawIOBase, BufferedIOBase
|
||||
funs = "close closed __enter__ __exit__ __iter__ isatty __next__ readable seekable writable"
|
||||
try:
|
||||
for fun in funs.split():
|
||||
setattr(self, fun, getattr(self.f, fun))
|
||||
except:
|
||||
pass # py27 probably
|
||||
|
||||
def tell(self):
|
||||
return self.ofs
|
||||
|
||||
def seek(self, ofs, wh=0):
|
||||
if wh == 1:
|
||||
ofs = self.ofs + ofs
|
||||
elif wh == 2:
|
||||
ofs = self.len + ofs # provided ofs is negative
|
||||
|
||||
if ofs < 0:
|
||||
ofs = 0
|
||||
elif ofs >= self.len:
|
||||
ofs = self.len - 1
|
||||
|
||||
self.ofs = ofs
|
||||
self.f.seek(self.car + ofs)
|
||||
|
||||
def read(self, sz):
|
||||
sz = min(sz, self.len - self.ofs)
|
||||
ret = self.f.read(sz)
|
||||
self.ofs += len(ret)
|
||||
return ret
|
||||
|
||||
|
||||
class MTHash(object):
|
||||
def __init__(self, cores):
|
||||
self.f = None
|
||||
self.sz = 0
|
||||
self.csz = 0
|
||||
self.omutex = threading.Lock()
|
||||
self.imutex = threading.Lock()
|
||||
self.work_q = Queue()
|
||||
self.done_q = Queue()
|
||||
self.thrs = []
|
||||
for _ in range(cores):
|
||||
t = threading.Thread(target=self.worker)
|
||||
t.daemon = True
|
||||
t.start()
|
||||
self.thrs.append(t)
|
||||
|
||||
def hash(self, f, fsz, chunksz, pcb=None, pcb_opaque=None):
|
||||
with self.omutex:
|
||||
self.f = f
|
||||
self.sz = fsz
|
||||
self.csz = chunksz
|
||||
|
||||
chunks = {}
|
||||
nchunks = int(math.ceil(fsz / chunksz))
|
||||
for nch in range(nchunks):
|
||||
self.work_q.put(nch)
|
||||
|
||||
ex = ""
|
||||
for nch in range(nchunks):
|
||||
qe = self.done_q.get()
|
||||
try:
|
||||
nch, dig, ofs, csz = qe
|
||||
chunks[nch] = [dig, ofs, csz]
|
||||
except:
|
||||
ex = ex or qe
|
||||
|
||||
if pcb:
|
||||
pcb(pcb_opaque, chunksz * nch)
|
||||
|
||||
if ex:
|
||||
raise Exception(ex)
|
||||
|
||||
ret = []
|
||||
for n in range(nchunks):
|
||||
ret.append(chunks[n])
|
||||
|
||||
self.f = None
|
||||
self.csz = 0
|
||||
self.sz = 0
|
||||
return ret
|
||||
|
||||
def worker(self):
|
||||
while True:
|
||||
ofs = self.work_q.get()
|
||||
try:
|
||||
v = self.hash_at(ofs)
|
||||
except Exception as ex:
|
||||
v = str(ex)
|
||||
|
||||
self.done_q.put(v)
|
||||
|
||||
def hash_at(self, nch):
|
||||
f = self.f
|
||||
ofs = ofs0 = nch * self.csz
|
||||
hashobj = hashlib.sha512()
|
||||
chunk_sz = chunk_rem = min(self.csz, self.sz - ofs)
|
||||
while chunk_rem > 0:
|
||||
with self.imutex:
|
||||
f.seek(ofs)
|
||||
buf = f.read(min(chunk_rem, 1024 * 1024 * 12))
|
||||
|
||||
if not buf:
|
||||
raise Exception("EOF at " + str(ofs))
|
||||
|
||||
hashobj.update(buf)
|
||||
chunk_rem -= len(buf)
|
||||
ofs += len(buf)
|
||||
|
||||
digest = hashobj.digest()[:33]
|
||||
digest = base64.urlsafe_b64encode(digest).decode("utf-8")
|
||||
return nch, digest, ofs0, chunk_sz
|
||||
|
||||
|
||||
_print = print
|
||||
|
||||
|
||||
def eprint(*a, **ka):
|
||||
ka["file"] = sys.stderr
|
||||
ka["end"] = ""
|
||||
if not PY2:
|
||||
ka["flush"] = True
|
||||
|
||||
_print(*a, **ka)
|
||||
if PY2 or not VT100:
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def flushing_print(*a, **ka):
|
||||
_print(*a, **ka)
|
||||
if "flush" not in ka:
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
if not VT100:
|
||||
print = flushing_print
|
||||
|
||||
|
||||
def termsize():
|
||||
env = os.environ
|
||||
|
||||
def ioctl_GWINSZ(fd):
|
||||
try:
|
||||
import fcntl, termios, struct
|
||||
|
||||
cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
|
||||
except:
|
||||
return
|
||||
return cr
|
||||
|
||||
cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
|
||||
if not cr:
|
||||
try:
|
||||
fd = os.open(os.ctermid(), os.O_RDONLY)
|
||||
cr = ioctl_GWINSZ(fd)
|
||||
os.close(fd)
|
||||
except:
|
||||
pass
|
||||
if not cr:
|
||||
try:
|
||||
cr = (env["LINES"], env["COLUMNS"])
|
||||
except:
|
||||
cr = (25, 80)
|
||||
return int(cr[1]), int(cr[0])
|
||||
|
||||
|
||||
class CTermsize(object):
|
||||
def __init__(self):
|
||||
self.ev = False
|
||||
self.margin = None
|
||||
self.g = None
|
||||
self.w, self.h = termsize()
|
||||
|
||||
try:
|
||||
signal.signal(signal.SIGWINCH, self.ev_sig)
|
||||
except:
|
||||
return
|
||||
|
||||
thr = threading.Thread(target=self.worker)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
def worker(self):
|
||||
while True:
|
||||
time.sleep(0.5)
|
||||
if not self.ev:
|
||||
continue
|
||||
|
||||
self.ev = False
|
||||
self.w, self.h = termsize()
|
||||
|
||||
if self.margin is not None:
|
||||
self.scroll_region(self.margin)
|
||||
|
||||
def ev_sig(self, *a, **ka):
|
||||
self.ev = True
|
||||
|
||||
def scroll_region(self, margin):
|
||||
self.margin = margin
|
||||
if margin is None:
|
||||
self.g = None
|
||||
eprint("\033[s\033[r\033[u")
|
||||
else:
|
||||
self.g = 1 + self.h - margin
|
||||
t = "{0}\033[{1}A".format("\n" * margin, margin)
|
||||
eprint("{0}\033[s\033[1;{1}r\033[u".format(t, self.g - 1))
|
||||
|
||||
|
||||
ss = CTermsize()
|
||||
|
||||
|
||||
def _scd(err, top):
|
||||
"""non-recursive listing of directory contents, along with stat() info"""
|
||||
with os.scandir(top) as dh:
|
||||
for fh in dh:
|
||||
abspath = os.path.join(top, fh.name)
|
||||
try:
|
||||
yield [abspath, fh.stat()]
|
||||
except Exception as ex:
|
||||
err.append((abspath, str(ex)))
|
||||
|
||||
|
||||
def _lsd(err, top):
|
||||
"""non-recursive listing of directory contents, along with stat() info"""
|
||||
for name in os.listdir(top):
|
||||
abspath = os.path.join(top, name)
|
||||
try:
|
||||
yield [abspath, os.stat(abspath)]
|
||||
except Exception as ex:
|
||||
err.append((abspath, str(ex)))
|
||||
|
||||
|
||||
if hasattr(os, "scandir") and sys.version_info > (3, 6):
|
||||
statdir = _scd
|
||||
else:
|
||||
statdir = _lsd
|
||||
|
||||
|
||||
def walkdir(err, top, seen):
|
||||
"""recursive statdir"""
|
||||
atop = os.path.abspath(os.path.realpath(top))
|
||||
if atop in seen:
|
||||
err.append((top, "recursive-symlink"))
|
||||
return
|
||||
|
||||
seen = seen[:] + [atop]
|
||||
for ap, inf in sorted(statdir(err, top)):
|
||||
if stat.S_ISDIR(inf.st_mode):
|
||||
try:
|
||||
for x in walkdir(err, ap, seen):
|
||||
yield x
|
||||
except Exception as ex:
|
||||
err.append((ap, str(ex)))
|
||||
else:
|
||||
yield ap, inf
|
||||
|
||||
|
||||
def walkdirs(err, tops):
|
||||
"""recursive statdir for a list of tops, yields [top, relpath, stat]"""
|
||||
sep = "{0}".format(os.sep).encode("ascii")
|
||||
for top in tops:
|
||||
if top[-1:] == sep:
|
||||
stop = top.rstrip(sep)
|
||||
else:
|
||||
stop = os.path.dirname(top)
|
||||
|
||||
if os.path.isdir(top):
|
||||
for ap, inf in walkdir(err, top, []):
|
||||
yield stop, ap[len(stop) :].lstrip(sep), inf
|
||||
else:
|
||||
d, n = top.rsplit(sep, 1)
|
||||
yield d, n, os.stat(top)
|
||||
|
||||
|
||||
# mostly from copyparty/util.py
|
||||
def quotep(btxt):
|
||||
quot1 = quote(btxt, safe=b"/")
|
||||
if not PY2:
|
||||
quot1 = quot1.encode("ascii")
|
||||
|
||||
return quot1.replace(b" ", b"+")
|
||||
|
||||
|
||||
# from copyparty/util.py
|
||||
def humansize(sz, terse=False):
|
||||
"""picks a sensible unit for the given extent"""
|
||||
for unit in ["B", "KiB", "MiB", "GiB", "TiB"]:
|
||||
if sz < 1024:
|
||||
break
|
||||
|
||||
sz /= 1024.0
|
||||
|
||||
ret = " ".join([str(sz)[:4].rstrip("."), unit])
|
||||
|
||||
if not terse:
|
||||
return ret
|
||||
|
||||
return ret.replace("iB", "").replace(" ", "")
|
||||
|
||||
|
||||
# from copyparty/up2k.py
|
||||
def up2k_chunksize(filesize):
|
||||
"""gives The correct chunksize for up2k hashing"""
|
||||
chunksize = 1024 * 1024
|
||||
stepsize = 512 * 1024
|
||||
while True:
|
||||
for mul in [1, 2]:
|
||||
nchunks = math.ceil(filesize * 1.0 / chunksize)
|
||||
if nchunks <= 256 or chunksize >= 32 * 1024 * 1024:
|
||||
return chunksize
|
||||
|
||||
chunksize += stepsize
|
||||
stepsize *= mul
|
||||
|
||||
|
||||
# mostly from copyparty/up2k.py
|
||||
def get_hashlist(file, pcb, mth):
|
||||
# type: (File, any, any) -> None
|
||||
"""generates the up2k hashlist from file contents, inserts it into `file`"""
|
||||
|
||||
chunk_sz = up2k_chunksize(file.size)
|
||||
file_rem = file.size
|
||||
file_ofs = 0
|
||||
ret = []
|
||||
with open(file.abs, "rb", 512 * 1024) as f:
|
||||
if mth and file.size >= 1024 * 512:
|
||||
ret = mth.hash(f, file.size, chunk_sz, pcb, file)
|
||||
file_rem = 0
|
||||
|
||||
while file_rem > 0:
|
||||
# same as `hash_at` except for `imutex` / bufsz
|
||||
hashobj = hashlib.sha512()
|
||||
chunk_sz = chunk_rem = min(chunk_sz, file_rem)
|
||||
while chunk_rem > 0:
|
||||
buf = f.read(min(chunk_rem, 64 * 1024))
|
||||
if not buf:
|
||||
raise Exception("EOF at " + str(f.tell()))
|
||||
|
||||
hashobj.update(buf)
|
||||
chunk_rem -= len(buf)
|
||||
|
||||
digest = hashobj.digest()[:33]
|
||||
digest = base64.urlsafe_b64encode(digest).decode("utf-8")
|
||||
|
||||
ret.append([digest, file_ofs, chunk_sz])
|
||||
file_ofs += chunk_sz
|
||||
file_rem -= chunk_sz
|
||||
|
||||
if pcb:
|
||||
pcb(file, file_ofs)
|
||||
|
||||
file.cids = ret
|
||||
file.kchunks = {}
|
||||
for k, v1, v2 in ret:
|
||||
file.kchunks[k] = [v1, v2]
|
||||
|
||||
|
||||
def handshake(req_ses, url, file, pw, search):
|
||||
# type: (requests.Session, str, File, any, bool) -> list[str]
|
||||
"""
|
||||
performs a handshake with the server; reply is:
|
||||
if search, a list of search results
|
||||
otherwise, a list of chunks to upload
|
||||
"""
|
||||
|
||||
req = {
|
||||
"hash": [x[0] for x in file.cids],
|
||||
"name": file.name,
|
||||
"lmod": file.lmod,
|
||||
"size": file.size,
|
||||
}
|
||||
if search:
|
||||
req["srch"] = 1
|
||||
|
||||
headers = {"Content-Type": "text/plain"} # wtf ed
|
||||
if pw:
|
||||
headers["Cookie"] = "=".join(["cppwd", pw])
|
||||
|
||||
if file.url:
|
||||
url = file.url
|
||||
elif b"/" in file.rel:
|
||||
url += quotep(file.rel.rsplit(b"/", 1)[0]).decode("utf-8", "replace")
|
||||
|
||||
while True:
|
||||
try:
|
||||
r = req_ses.post(url, headers=headers, json=req)
|
||||
break
|
||||
except Exception as ex:
|
||||
em = str(ex).split("SSLError(")[-1]
|
||||
eprint("handshake failed, retrying: {0}\n {1}\n\n".format(file.name, em))
|
||||
time.sleep(1)
|
||||
|
||||
try:
|
||||
r = r.json()
|
||||
except:
|
||||
raise Exception(r.text)
|
||||
|
||||
if search:
|
||||
return r["hits"], False
|
||||
|
||||
try:
|
||||
pre, url = url.split("://")
|
||||
pre += "://"
|
||||
except:
|
||||
pre = ""
|
||||
|
||||
file.url = pre + url.split("/")[0] + r["purl"]
|
||||
file.name = r["name"]
|
||||
file.wark = r["wark"]
|
||||
|
||||
return r["hash"], r["sprs"]
|
||||
|
||||
|
||||
def upload(req_ses, file, cid, pw):
|
||||
# type: (requests.Session, File, str, any) -> None
|
||||
"""upload one specific chunk, `cid` (a chunk-hash)"""
|
||||
|
||||
headers = {
|
||||
"X-Up2k-Hash": cid,
|
||||
"X-Up2k-Wark": file.wark,
|
||||
"Content-Type": "application/octet-stream",
|
||||
}
|
||||
if pw:
|
||||
headers["Cookie"] = "=".join(["cppwd", pw])
|
||||
|
||||
f = FileSlice(file, cid)
|
||||
try:
|
||||
r = req_ses.post(file.url, headers=headers, data=f)
|
||||
if not r:
|
||||
raise Exception(repr(r))
|
||||
|
||||
_ = r.content
|
||||
finally:
|
||||
f.f.close()
|
||||
|
||||
|
||||
class Daemon(threading.Thread):
|
||||
def __init__(self, *a, **ka):
|
||||
threading.Thread.__init__(self, *a, **ka)
|
||||
self.daemon = True
|
||||
|
||||
|
||||
class Ctl(object):
|
||||
"""
|
||||
this will be the coordinator which runs everything in parallel
|
||||
(hashing, handshakes, uploads) but right now it's p dumb
|
||||
"""
|
||||
|
||||
def __init__(self, ar):
|
||||
self.ar = ar
|
||||
ar.files = [
|
||||
os.path.abspath(os.path.realpath(x.encode("utf-8")))
|
||||
+ (x[-1:] if x[-1:] == os.sep else "").encode("utf-8")
|
||||
for x in ar.files
|
||||
]
|
||||
ar.url = ar.url.rstrip("/") + "/"
|
||||
if "://" not in ar.url:
|
||||
ar.url = "http://" + ar.url
|
||||
|
||||
eprint("\nscanning {0} locations\n".format(len(ar.files)))
|
||||
|
||||
nfiles = 0
|
||||
nbytes = 0
|
||||
err = []
|
||||
for _, _, inf in walkdirs(err, ar.files):
|
||||
nfiles += 1
|
||||
nbytes += inf.st_size
|
||||
|
||||
if err:
|
||||
eprint("\n# failed to access {0} paths:\n".format(len(err)))
|
||||
for ap, msg in err:
|
||||
if ar.v:
|
||||
eprint("{0}\n `-{1}\n\n".format(ap.decode("utf-8", "replace"), msg))
|
||||
else:
|
||||
eprint(ap.decode("utf-8", "replace") + "\n")
|
||||
|
||||
eprint("^ failed to access those {0} paths ^\n\n".format(len(err)))
|
||||
|
||||
if not ar.v:
|
||||
eprint("hint: set -v for detailed error messages\n")
|
||||
|
||||
if not ar.ok:
|
||||
eprint("hint: aborting because --ok is not set\n")
|
||||
return
|
||||
|
||||
eprint("found {0} files, {1}\n\n".format(nfiles, humansize(nbytes)))
|
||||
self.nfiles = nfiles
|
||||
self.nbytes = nbytes
|
||||
|
||||
if ar.td:
|
||||
requests.packages.urllib3.disable_warnings()
|
||||
req_ses.verify = False
|
||||
if ar.te:
|
||||
req_ses.verify = ar.te
|
||||
|
||||
self.filegen = walkdirs([], ar.files)
|
||||
if ar.safe:
|
||||
self._safe()
|
||||
else:
|
||||
self.hash_f = 0
|
||||
self.hash_c = 0
|
||||
self.hash_b = 0
|
||||
self.up_f = 0
|
||||
self.up_c = 0
|
||||
self.up_b = 0
|
||||
self.up_br = 0
|
||||
self.hasher_busy = 1
|
||||
self.handshaker_busy = 0
|
||||
self.uploader_busy = 0
|
||||
self.serialized = False
|
||||
|
||||
self.t0 = time.time()
|
||||
self.t0_up = None
|
||||
self.spd = None
|
||||
|
||||
self.mutex = threading.Lock()
|
||||
self.q_handshake = Queue() # type: Queue[File]
|
||||
self.q_recheck = Queue() # type: Queue[File] # partial upload exists [...]
|
||||
self.q_upload = Queue() # type: Queue[tuple[File, str]]
|
||||
|
||||
self.st_hash = [None, "(idle, starting...)"] # type: tuple[File, int]
|
||||
self.st_up = [None, "(idle, starting...)"] # type: tuple[File, int]
|
||||
|
||||
self.mth = MTHash(ar.J) if ar.J > 1 else None
|
||||
|
||||
self._fancy()
|
||||
|
||||
def _safe(self):
|
||||
"""minimal basic slow boring fallback codepath"""
|
||||
search = self.ar.s
|
||||
for nf, (top, rel, inf) in enumerate(self.filegen):
|
||||
file = File(top, rel, inf.st_size, inf.st_mtime)
|
||||
upath = file.abs.decode("utf-8", "replace")
|
||||
|
||||
print("{0} {1}\n hash...".format(self.nfiles - nf, upath))
|
||||
get_hashlist(file, None, None)
|
||||
|
||||
burl = self.ar.url[:12] + self.ar.url[8:].split("/")[0] + "/"
|
||||
while True:
|
||||
print(" hs...")
|
||||
hs, _ = handshake(req_ses, self.ar.url, file, self.ar.a, search)
|
||||
if search:
|
||||
if hs:
|
||||
for hit in hs:
|
||||
print(" found: {0}{1}".format(burl, hit["rp"]))
|
||||
else:
|
||||
print(" NOT found")
|
||||
break
|
||||
|
||||
file.ucids = hs
|
||||
if not hs:
|
||||
break
|
||||
|
||||
print("{0} {1}".format(self.nfiles - nf, upath))
|
||||
ncs = len(hs)
|
||||
for nc, cid in enumerate(hs):
|
||||
print(" {0} up {1}".format(ncs - nc, cid))
|
||||
upload(req_ses, file, cid, self.ar.a)
|
||||
|
||||
print(" ok!")
|
||||
|
||||
def _fancy(self):
|
||||
if VT100:
|
||||
atexit.register(self.cleanup_vt100)
|
||||
ss.scroll_region(3)
|
||||
|
||||
Daemon(target=self.hasher).start()
|
||||
for _ in range(self.ar.j):
|
||||
Daemon(target=self.handshaker).start()
|
||||
Daemon(target=self.uploader).start()
|
||||
|
||||
idles = 0
|
||||
while idles < 3:
|
||||
time.sleep(0.07)
|
||||
with self.mutex:
|
||||
if (
|
||||
self.q_handshake.empty()
|
||||
and self.q_upload.empty()
|
||||
and not self.hasher_busy
|
||||
and not self.handshaker_busy
|
||||
and not self.uploader_busy
|
||||
):
|
||||
idles += 1
|
||||
else:
|
||||
idles = 0
|
||||
|
||||
if VT100:
|
||||
maxlen = ss.w - len(str(self.nfiles)) - 14
|
||||
txt = "\033[s\033[{0}H".format(ss.g)
|
||||
for y, k, st, f in [
|
||||
[0, "hash", self.st_hash, self.hash_f],
|
||||
[1, "send", self.st_up, self.up_f],
|
||||
]:
|
||||
txt += "\033[{0}H{1}:".format(ss.g + y, k)
|
||||
file, arg = st
|
||||
if not file:
|
||||
txt += " {0}\033[K".format(arg)
|
||||
else:
|
||||
if y:
|
||||
p = 100 * file.up_b / file.size
|
||||
else:
|
||||
p = 100 * arg / file.size
|
||||
|
||||
name = file.abs.decode("utf-8", "replace")[-maxlen:]
|
||||
if "/" in name:
|
||||
name = "\033[36m{0}\033[0m/{1}".format(*name.rsplit("/", 1))
|
||||
|
||||
t = "{0:6.1f}% {1} {2}\033[K"
|
||||
txt += t.format(p, self.nfiles - f, name)
|
||||
|
||||
txt += "\033[{0}H ".format(ss.g + 2)
|
||||
else:
|
||||
txt = " "
|
||||
|
||||
if not self.up_br:
|
||||
spd = self.hash_b / (time.time() - self.t0)
|
||||
eta = (self.nbytes - self.hash_b) / (spd + 1)
|
||||
else:
|
||||
spd = self.up_br / (time.time() - self.t0_up)
|
||||
spd = self.spd = (self.spd or spd) * 0.9 + spd * 0.1
|
||||
eta = (self.nbytes - self.up_b) / (spd + 1)
|
||||
|
||||
spd = humansize(spd)
|
||||
eta = str(datetime.timedelta(seconds=int(eta)))
|
||||
sleft = humansize(self.nbytes - self.up_b)
|
||||
nleft = self.nfiles - self.up_f
|
||||
tail = "\033[K\033[u" if VT100 else "\r"
|
||||
|
||||
t = "{0} eta @ {1}/s, {2}, {3}# left".format(eta, spd, sleft, nleft)
|
||||
eprint(txt + "\033]0;{0}\033\\\r{0}{1}".format(t, tail))
|
||||
|
||||
def cleanup_vt100(self):
|
||||
ss.scroll_region(None)
|
||||
eprint("\033[J\033]0;\033\\")
|
||||
|
||||
def cb_hasher(self, file, ofs):
|
||||
self.st_hash = [file, ofs]
|
||||
|
||||
def hasher(self):
|
||||
prd = None
|
||||
ls = {}
|
||||
for top, rel, inf in self.filegen:
|
||||
if self.ar.z:
|
||||
rd = os.path.dirname(rel)
|
||||
if prd != rd:
|
||||
prd = rd
|
||||
headers = {}
|
||||
if self.ar.a:
|
||||
headers["Cookie"] = "=".join(["cppwd", self.ar.a])
|
||||
|
||||
ls = {}
|
||||
try:
|
||||
print(" ls ~{0}".format(rd.decode("utf-8", "replace")))
|
||||
r = req_ses.get(
|
||||
self.ar.url.encode("utf-8") + quotep(rd) + b"?ls",
|
||||
headers=headers,
|
||||
)
|
||||
for f in r.json()["files"]:
|
||||
rfn = f["href"].split("?")[0].encode("utf-8", "replace")
|
||||
ls[unquote(rfn)] = f
|
||||
except:
|
||||
print(" mkdir ~{0}".format(rd.decode("utf-8", "replace")))
|
||||
|
||||
rf = ls.get(os.path.basename(rel), None)
|
||||
if rf and rf["sz"] == inf.st_size and abs(rf["ts"] - inf.st_mtime) <= 1:
|
||||
self.nfiles -= 1
|
||||
self.nbytes -= inf.st_size
|
||||
continue
|
||||
|
||||
file = File(top, rel, inf.st_size, inf.st_mtime)
|
||||
while True:
|
||||
with self.mutex:
|
||||
if (
|
||||
self.hash_b - self.up_b < 1024 * 1024 * 128
|
||||
and self.hash_c - self.up_c < 64
|
||||
and (
|
||||
not self.ar.nh
|
||||
or (
|
||||
self.q_upload.empty()
|
||||
and self.q_handshake.empty()
|
||||
and not self.uploader_busy
|
||||
)
|
||||
)
|
||||
):
|
||||
break
|
||||
|
||||
time.sleep(0.05)
|
||||
|
||||
get_hashlist(file, self.cb_hasher, self.mth)
|
||||
with self.mutex:
|
||||
self.hash_f += 1
|
||||
self.hash_c += len(file.cids)
|
||||
self.hash_b += file.size
|
||||
|
||||
self.q_handshake.put(file)
|
||||
|
||||
self.hasher_busy = 0
|
||||
self.st_hash = [None, "(finished)"]
|
||||
|
||||
def handshaker(self):
|
||||
search = self.ar.s
|
||||
q = self.q_handshake
|
||||
burl = self.ar.url[:8] + self.ar.url[8:].split("/")[0] + "/"
|
||||
while True:
|
||||
file = q.get()
|
||||
if not file:
|
||||
if q == self.q_handshake:
|
||||
q = self.q_recheck
|
||||
q.put(None)
|
||||
continue
|
||||
|
||||
self.q_upload.put(None)
|
||||
break
|
||||
|
||||
with self.mutex:
|
||||
self.handshaker_busy += 1
|
||||
|
||||
upath = file.abs.decode("utf-8", "replace")
|
||||
|
||||
try:
|
||||
hs, sprs = handshake(req_ses, self.ar.url, file, self.ar.a, search)
|
||||
except Exception as ex:
|
||||
if q == self.q_handshake and "<pre>partial upload exists" in str(ex):
|
||||
self.q_recheck.put(file)
|
||||
hs = []
|
||||
else:
|
||||
raise
|
||||
|
||||
if search:
|
||||
if hs:
|
||||
for hit in hs:
|
||||
t = "found: {0}\n {1}{2}\n"
|
||||
print(t.format(upath, burl, hit["rp"]), end="")
|
||||
else:
|
||||
print("NOT found: {0}\n".format(upath), end="")
|
||||
|
||||
with self.mutex:
|
||||
self.up_f += 1
|
||||
self.up_c += len(file.cids)
|
||||
self.up_b += file.size
|
||||
self.handshaker_busy -= 1
|
||||
|
||||
continue
|
||||
|
||||
with self.mutex:
|
||||
if not sprs and not self.serialized:
|
||||
t = "server filesystem does not support sparse files; serializing uploads\n"
|
||||
eprint(t)
|
||||
self.serialized = True
|
||||
for _ in range(self.ar.j - 1):
|
||||
self.q_upload.put(None)
|
||||
if not hs:
|
||||
# all chunks done
|
||||
self.up_f += 1
|
||||
self.up_c += len(file.cids) - file.up_c
|
||||
self.up_b += file.size - file.up_b
|
||||
|
||||
if hs and file.up_c:
|
||||
# some chunks failed
|
||||
self.up_c -= len(hs)
|
||||
file.up_c -= len(hs)
|
||||
for cid in hs:
|
||||
sz = file.kchunks[cid][1]
|
||||
self.up_b -= sz
|
||||
file.up_b -= sz
|
||||
|
||||
file.ucids = hs
|
||||
self.handshaker_busy -= 1
|
||||
|
||||
if not hs:
|
||||
kw = "uploaded" if file.up_b else " found"
|
||||
print("{0} {1}".format(kw, upath))
|
||||
for cid in hs:
|
||||
self.q_upload.put([file, cid])
|
||||
|
||||
def uploader(self):
|
||||
while True:
|
||||
task = self.q_upload.get()
|
||||
if not task:
|
||||
self.st_up = [None, "(finished)"]
|
||||
break
|
||||
|
||||
with self.mutex:
|
||||
self.uploader_busy += 1
|
||||
self.t0_up = self.t0_up or time.time()
|
||||
|
||||
file, cid = task
|
||||
try:
|
||||
upload(req_ses, file, cid, self.ar.a)
|
||||
except:
|
||||
eprint("upload failed, retrying: {0} #{1}\n".format(file.name, cid[:8]))
|
||||
pass # handshake will fix it
|
||||
|
||||
with self.mutex:
|
||||
sz = file.kchunks[cid][1]
|
||||
file.ucids = [x for x in file.ucids if x != cid]
|
||||
if not file.ucids:
|
||||
self.q_handshake.put(file)
|
||||
|
||||
self.st_up = [file, cid]
|
||||
file.up_b += sz
|
||||
self.up_b += sz
|
||||
self.up_br += sz
|
||||
file.up_c += 1
|
||||
self.up_c += 1
|
||||
self.uploader_busy -= 1
|
||||
|
||||
|
||||
class APF(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
time.strptime("19970815", "%Y%m%d") # python#7980
|
||||
if not VT100:
|
||||
os.system("rem") # enables colors
|
||||
|
||||
cores = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 2
|
||||
hcores = min(cores, 3) # 4% faster than 4+ on py3.9 @ r5-4500U
|
||||
|
||||
# fmt: off
|
||||
ap = app = argparse.ArgumentParser(formatter_class=APF, epilog="""
|
||||
NOTE:
|
||||
source file/folder selection uses rsync syntax, meaning that:
|
||||
"foo" uploads the entire folder to URL/foo/
|
||||
"foo/" uploads the CONTENTS of the folder into URL/
|
||||
""")
|
||||
|
||||
ap.add_argument("url", type=unicode, help="server url, including destination folder")
|
||||
ap.add_argument("files", type=unicode, nargs="+", help="files and/or folders to process")
|
||||
ap.add_argument("-v", action="store_true", help="verbose")
|
||||
ap.add_argument("-a", metavar="PASSWORD", help="password")
|
||||
ap.add_argument("-s", action="store_true", help="file-search (disables upload)")
|
||||
ap.add_argument("--ok", action="store_true", help="continue even if some local files are inaccessible")
|
||||
ap = app.add_argument_group("performance tweaks")
|
||||
ap.add_argument("-j", type=int, metavar="THREADS", default=4, help="parallel connections")
|
||||
ap.add_argument("-J", type=int, metavar="THREADS", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
|
||||
ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
|
||||
ap.add_argument("--safe", action="store_true", help="use simple fallback approach")
|
||||
ap.add_argument("-z", action="store_true", help="ZOOMIN' (skip uploading files if they exist at the destination with the ~same last-modified timestamp, so same as yolo / turbo with date-chk but even faster)")
|
||||
ap = app.add_argument_group("tls")
|
||||
ap.add_argument("-te", metavar="PEM_FILE", help="certificate to expect/verify")
|
||||
ap.add_argument("-td", action="store_true", help="disable certificate check")
|
||||
# fmt: on
|
||||
|
||||
Ctl(app.parse_args())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
12
bin/up2k.sh
Normal file → Executable file
12
bin/up2k.sh
Normal file → Executable file
@@ -8,7 +8,7 @@ set -e
|
||||
##
|
||||
## config
|
||||
|
||||
datalen=$((128*1024*1024))
|
||||
datalen=$((2*1024*1024*1024))
|
||||
target=127.0.0.1
|
||||
posturl=/inc
|
||||
passwd=wark
|
||||
@@ -37,10 +37,10 @@ gendata() {
|
||||
# pipe a chunk, get the base64 checksum
|
||||
gethash() {
|
||||
printf $(
|
||||
sha512sum | cut -c-66 |
|
||||
sha512sum | cut -c-64 |
|
||||
sed -r 's/ .*//;s/(..)/\\x\1/g'
|
||||
) |
|
||||
base64 -w0 | cut -c-44 |
|
||||
base64 -w0 | cut -c-43 |
|
||||
tr '+/' '-_'
|
||||
}
|
||||
|
||||
@@ -123,7 +123,7 @@ printf '\033[36m'
|
||||
{
|
||||
{
|
||||
cat <<EOF
|
||||
POST $posturl/ HTTP/1.1
|
||||
POST $posturl/handshake.php HTTP/1.1
|
||||
Connection: Close
|
||||
Cookie: cppwd=$passwd
|
||||
Content-Type: text/plain;charset=UTF-8
|
||||
@@ -145,7 +145,6 @@ printf '\033[0m\nwark: %s\n' $wark
|
||||
##
|
||||
## wait for signal to continue
|
||||
|
||||
true || {
|
||||
w8=/dev/shm/$salt.w8
|
||||
touch $w8
|
||||
|
||||
@@ -154,7 +153,6 @@ true || {
|
||||
while [ -e $w8 ]; do
|
||||
sleep 0.2
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
##
|
||||
@@ -177,7 +175,7 @@ while [ $remains -gt 0 ]; do
|
||||
|
||||
{
|
||||
cat <<EOF
|
||||
POST $posturl/ HTTP/1.1
|
||||
POST $posturl/chunkpit.php HTTP/1.1
|
||||
Connection: Keep-Alive
|
||||
Cookie: cppwd=$passwd
|
||||
Content-Type: application/octet-stream
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
### [`plugins/`](plugins/)
|
||||
* example extensions
|
||||
|
||||
### [`copyparty.bat`](copyparty.bat)
|
||||
* launches copyparty with no arguments (anon read+write within same folder)
|
||||
* intended for windows machines with no python.exe in PATH
|
||||
@@ -22,9 +19,6 @@ however if your copyparty is behind a reverse-proxy, you may want to use [`share
|
||||
* `URL`: full URL to the root folder (with trailing slash) followed by `$regex:1|1$`
|
||||
* `pw`: password (remove `Parameters` if anon-write)
|
||||
|
||||
### [`media-osd-bgone.ps1`](media-osd-bgone.ps1)
|
||||
* disables the [windows OSD popup](https://user-images.githubusercontent.com/241032/122821375-0e08df80-d2dd-11eb-9fd9-184e8aacf1d0.png) (the thing on the left) which appears every time you hit media hotkeys to adjust volume or change song while playing music with the copyparty web-ui, or most other audio players really
|
||||
|
||||
### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
|
||||
* disables thumbnails and folder-type detection in windows explorer
|
||||
* makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
|
||||
@@ -32,13 +26,10 @@ however if your copyparty is behind a reverse-proxy, you may want to use [`share
|
||||
### [`cfssl.sh`](cfssl.sh)
|
||||
* creates CA and server certificates using cfssl
|
||||
* give a 3rd argument to install it to your copyparty config
|
||||
* systemd service at [`systemd/cfssl.service`](systemd/cfssl.service)
|
||||
|
||||
# OS integration
|
||||
init-scripts to start copyparty as a service
|
||||
* [`systemd/copyparty.service`](systemd/copyparty.service) runs the sfx normally
|
||||
* [`rc/copyparty`](rc/copyparty) runs sfx normally on freebsd, create a `copyparty` user
|
||||
* [`systemd/prisonparty.service`](systemd/prisonparty.service) runs the sfx in a chroot
|
||||
* [`systemd/copyparty.service`](systemd/copyparty.service)
|
||||
* [`openrc/copyparty`](openrc/copyparty)
|
||||
|
||||
# Reverse-proxy
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# ca-name and server-fqdn
|
||||
# ca-name and server-name
|
||||
ca_name="$1"
|
||||
srv_fqdn="$2"
|
||||
srv_name="$2"
|
||||
|
||||
[ -z "$srv_fqdn" ] && {
|
||||
[ -z "$srv_name" ] && {
|
||||
echo "need arg 1: ca name"
|
||||
echo "need arg 2: server fqdn and/or IPs, comma-separated"
|
||||
echo "optional arg 3: if set, write cert into copyparty cfg"
|
||||
echo "need arg 2: server name"
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -32,15 +31,15 @@ EOF
|
||||
gen_srv() {
|
||||
(tee /dev/stderr <<EOF
|
||||
{"key": {"algo":"rsa", "size":4096},
|
||||
"names": [{"O":"$ca_name - $srv_fqdn"}]}
|
||||
"names": [{"O":"$ca_name - $srv_name"}]}
|
||||
EOF
|
||||
)|
|
||||
cfssl gencert -ca ca.pem -ca-key ca.key \
|
||||
-profile=www -hostname="$srv_fqdn" - |
|
||||
cfssljson -bare "$srv_fqdn"
|
||||
-profile=www -hostname="$srv_name.$ca_name" - |
|
||||
cfssljson -bare "$srv_name"
|
||||
|
||||
mv "$srv_fqdn-key.pem" "$srv_fqdn.key"
|
||||
rm "$srv_fqdn.csr"
|
||||
mv "$srv_name-key.pem" "$srv_name.key"
|
||||
rm "$srv_name.csr"
|
||||
}
|
||||
|
||||
|
||||
@@ -58,13 +57,13 @@ show() {
|
||||
awk '!o; {o=0} /[0-9a-f:]{16}/{o=1}'
|
||||
}
|
||||
show ca.pem
|
||||
show "$srv_fqdn.pem"
|
||||
show "$srv_name.pem"
|
||||
|
||||
|
||||
# write cert into copyparty config
|
||||
[ -z "$3" ] || {
|
||||
mkdir -p ~/.config/copyparty
|
||||
cat "$srv_fqdn".{key,pem} ca.pem >~/.config/copyparty/cert.pem
|
||||
cat "$srv_name".{key,pem} ca.pem >~/.config/copyparty/cert.pem
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
# media-osd-bgone.ps1: disable media-control OSD on win10do
|
||||
# v1.1, 2021-06-25, ed <irc.rizon.net>, MIT-licensed
|
||||
# https://github.com/9001/copyparty/blob/hovudstraum/contrib/media-osd-bgone.ps1
|
||||
#
|
||||
# locates the first window that looks like the media OSD and minimizes it;
|
||||
# doing this once after each reboot should do the trick
|
||||
# (adjust the width/height filter if it doesn't work)
|
||||
#
|
||||
# ---------------------------------------------------------------------
|
||||
#
|
||||
# tip: save the following as "media-osd-bgone.bat" next to this script:
|
||||
# start cmd /c "powershell -command ""set-executionpolicy -scope process bypass; .\media-osd-bgone.ps1"" & ping -n 2 127.1 >nul"
|
||||
#
|
||||
# then create a shortcut to that bat-file and move the shortcut here:
|
||||
# %appdata%\Microsoft\Windows\Start Menu\Programs\Startup
|
||||
#
|
||||
# and now this will autorun on bootup
|
||||
|
||||
|
||||
Add-Type -TypeDefinition @"
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Windows.Forms;
|
||||
|
||||
namespace A {
|
||||
public class B : Control {
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, int dwExtraInfo);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
static extern IntPtr FindWindowEx(IntPtr hwndParent, IntPtr hwndChildAfter, string lpszClass, string lpszWindow);
|
||||
|
||||
[DllImport("user32.dll", SetLastError=true)]
|
||||
static extern bool GetWindowRect(IntPtr hwnd, out RECT lpRect);
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct RECT {
|
||||
public int x;
|
||||
public int y;
|
||||
public int x2;
|
||||
public int y2;
|
||||
}
|
||||
|
||||
bool fa() {
|
||||
RECT r;
|
||||
IntPtr it = IntPtr.Zero;
|
||||
while ((it = FindWindowEx(IntPtr.Zero, it, "NativeHWNDHost", "")) != IntPtr.Zero) {
|
||||
if (FindWindowEx(it, IntPtr.Zero, "DirectUIHWND", "") == IntPtr.Zero)
|
||||
continue;
|
||||
|
||||
if (!GetWindowRect(it, out r))
|
||||
continue;
|
||||
|
||||
int w = r.x2 - r.x + 1;
|
||||
int h = r.y2 - r.y + 1;
|
||||
|
||||
Console.WriteLine("[*] hwnd {0:x} @ {1}x{2} sz {3}x{4}", it, r.x, r.y, w, h);
|
||||
if (h != 141)
|
||||
continue;
|
||||
|
||||
ShowWindow(it, 6);
|
||||
Console.WriteLine("[+] poof");
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void fb() {
|
||||
keybd_event((byte)Keys.VolumeMute, 0, 0, 0);
|
||||
keybd_event((byte)Keys.VolumeMute, 0, 2, 0);
|
||||
Thread.Sleep(500);
|
||||
keybd_event((byte)Keys.VolumeMute, 0, 0, 0);
|
||||
keybd_event((byte)Keys.VolumeMute, 0, 2, 0);
|
||||
|
||||
while (true) {
|
||||
if (fa()) {
|
||||
break;
|
||||
}
|
||||
Console.WriteLine("[!] not found");
|
||||
Thread.Sleep(1000);
|
||||
}
|
||||
this.Invoke((MethodInvoker)delegate {
|
||||
Application.Exit();
|
||||
});
|
||||
}
|
||||
|
||||
public void Run() {
|
||||
Console.WriteLine("[+] hi");
|
||||
new Thread(new ThreadStart(fb)).Start();
|
||||
Application.Run();
|
||||
Console.WriteLine("[+] bye");
|
||||
}
|
||||
}
|
||||
}
|
||||
"@ -ReferencedAssemblies System.Windows.Forms
|
||||
|
||||
(New-Object -TypeName A.B).Run()
|
||||
@@ -1,19 +1,11 @@
|
||||
# when running copyparty behind a reverse proxy,
|
||||
# the following arguments are recommended:
|
||||
#
|
||||
# -nc 512 important, see next paragraph
|
||||
# --http-only lower latency on initial connection
|
||||
# -i 127.0.0.1 only accept connections from nginx
|
||||
#
|
||||
# -nc must match or exceed the webserver's max number of concurrent clients;
|
||||
# nginx default is 512 (worker_processes 1, worker_connections 512)
|
||||
#
|
||||
# you may also consider adding -j0 for CPU-intensive configurations
|
||||
# (not that i can really think of any good examples)
|
||||
# when running copyparty behind a reverse-proxy,
|
||||
# make sure that copyparty allows at least as many clients as the proxy does,
|
||||
# so run copyparty with -nc 512 if your nginx has the default limits
|
||||
# (worker_processes 1, worker_connections 512)
|
||||
|
||||
upstream cpp {
|
||||
server 127.0.0.1:3923;
|
||||
keepalive 1;
|
||||
keepalive 120;
|
||||
}
|
||||
server {
|
||||
listen 443 ssl;
|
||||
|
||||
@@ -8,11 +8,11 @@
|
||||
#
|
||||
# you may want to:
|
||||
# change '/usr/bin/python' to another interpreter
|
||||
# change '/mnt::rw' to another location or permission-set
|
||||
# change '/mnt::a' to another location or permission-set
|
||||
|
||||
name="$SVCNAME"
|
||||
command_background=true
|
||||
pidfile="/var/run/$SVCNAME.pid"
|
||||
|
||||
command="/usr/bin/python /usr/local/bin/copyparty-sfx.py"
|
||||
command_args="-q -v /mnt::rw"
|
||||
command_args="-q -v /mnt::a"
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
# example resource files
|
||||
|
||||
can be provided to copyparty to tweak things
|
||||
|
||||
|
||||
|
||||
## example `.epilogue.html`
|
||||
save one of these as `.epilogue.html` inside a folder to customize it:
|
||||
|
||||
* [`minimal-up2k.html`](minimal-up2k.html) will [simplify the upload ui](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
|
||||
|
||||
|
||||
|
||||
## example browser-js
|
||||
point `--js-browser` to one of these by URL:
|
||||
|
||||
* [`minimal-up2k.js`](minimal-up2k.js) is similar to the above `minimal-up2k.html` except it applies globally to all write-only folders
|
||||
* [`up2k-hooks.js`](up2k-hooks.js) lets you specify a ruleset for files to skip uploading
|
||||
* [`up2k-hook-ytid.js`](up2k-hook-ytid.js) is a more specific example checking youtube-IDs against some API
|
||||
|
||||
|
||||
|
||||
## example browser-css
|
||||
point `--css-browser` to one of these by URL:
|
||||
|
||||
* [`browser-icons.css`](browser-icons.css) adds filetype icons
|
||||
|
||||
|
||||
|
||||
## meadup.js
|
||||
|
||||
* turns copyparty into chromecast just more flexible (and probably way more buggy)
|
||||
* usage: put the js somewhere in the webroot and `--js-browser /memes/meadup.js`
|
||||
@@ -1,71 +0,0 @@
|
||||
/* video, alternative 1:
|
||||
top-left icon, just like the other formats
|
||||
=======================================================================
|
||||
|
||||
#ggrid>a:is(
|
||||
[href$=".mkv"i],
|
||||
[href$=".mp4"i],
|
||||
[href$=".webm"i],
|
||||
):before {
|
||||
content: '📺';
|
||||
}
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/* video, alternative 2:
|
||||
play-icon in the middle of the thumbnail
|
||||
=======================================================================
|
||||
*/
|
||||
#ggrid>a:is(
|
||||
[href$=".mkv"i],
|
||||
[href$=".mp4"i],
|
||||
[href$=".webm"i],
|
||||
) {
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
}
|
||||
#ggrid>a:is(
|
||||
[href$=".mkv"i],
|
||||
[href$=".mp4"i],
|
||||
[href$=".webm"i],
|
||||
):before {
|
||||
content: '▶';
|
||||
opacity: .8;
|
||||
margin: 0;
|
||||
padding: 1em .5em 1em .7em;
|
||||
border-radius: 9em;
|
||||
line-height: 0;
|
||||
color: #fff;
|
||||
text-shadow: none;
|
||||
background: rgba(0, 0, 0, 0.7);
|
||||
left: calc(50% - 1em);
|
||||
top: calc(50% - 1.4em);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* audio */
|
||||
#ggrid>a:is(
|
||||
[href$=".mp3"i],
|
||||
[href$=".ogg"i],
|
||||
[href$=".opus"i],
|
||||
[href$=".flac"i],
|
||||
[href$=".m4a"i],
|
||||
[href$=".aac"i],
|
||||
):before {
|
||||
content: '🎵';
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* image */
|
||||
#ggrid>a:is(
|
||||
[href$=".jpg"i],
|
||||
[href$=".jpeg"i],
|
||||
[href$=".png"i],
|
||||
[href$=".gif"i],
|
||||
[href$=".webp"i],
|
||||
):before {
|
||||
content: '🎨';
|
||||
}
|
||||
@@ -1,506 +0,0 @@
|
||||
// USAGE:
|
||||
// place this file somewhere in the webroot and then
|
||||
// python3 -m copyparty --js-browser /memes/meadup.js
|
||||
//
|
||||
// FEATURES:
|
||||
// * adds an onscreen keyboard for operating a media center remotely,
|
||||
// relies on https://github.com/9001/copyparty/blob/hovudstraum/bin/mtag/very-bad-idea.py
|
||||
// * adds an interactive anime girl (if you can find the dependencies)
|
||||
|
||||
var hambagas = [
|
||||
"https://www.youtube.com/watch?v=pFA3KGp4GuU"
|
||||
];
|
||||
|
||||
// keybaord,
|
||||
// onscreen keyboard by @steinuil
|
||||
function initKeybaord(BASE_URL, HAMBAGA, consoleLog, consoleError) {
|
||||
document.querySelector('.keybaord-container').innerHTML = `
|
||||
<div class="keybaord-body">
|
||||
<div class="keybaord-row keybaord-row-1">
|
||||
<div class="keybaord-key" data-keybaord-key="Escape">
|
||||
esc
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F1">
|
||||
F1
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F2">
|
||||
F2
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F3">
|
||||
F3
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F4">
|
||||
F4
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F5">
|
||||
F5
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F6">
|
||||
F6
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F7">
|
||||
F7
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F8">
|
||||
F8
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F9">
|
||||
F9
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F10">
|
||||
F10
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F11">
|
||||
F11
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="F12">
|
||||
F12
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Insert">
|
||||
ins
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Delete">
|
||||
del
|
||||
</div>
|
||||
</div>
|
||||
<div class="keybaord-row keybaord-row-2">
|
||||
<div class="keybaord-key" data-keybaord-key="\`">
|
||||
\`
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="1">
|
||||
1
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="2">
|
||||
2
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="3">
|
||||
3
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="4">
|
||||
4
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="5">
|
||||
5
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="6">
|
||||
6
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="7">
|
||||
7
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="8">
|
||||
8
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="9">
|
||||
9
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="0">
|
||||
0
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="-">
|
||||
-
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="=">
|
||||
=
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-backspace" data-keybaord-key="BackSpace">
|
||||
backspace
|
||||
</div>
|
||||
</div>
|
||||
<div class="keybaord-row keybaord-row-3">
|
||||
<div class="keybaord-key keybaord-tab" data-keybaord-key="Tab">
|
||||
tab
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="q">
|
||||
q
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="w">
|
||||
w
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="e">
|
||||
e
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="r">
|
||||
r
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="t">
|
||||
t
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="y">
|
||||
y
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="u">
|
||||
u
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="i">
|
||||
i
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="o">
|
||||
o
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="p">
|
||||
p
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="[">
|
||||
[
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="]">
|
||||
]
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-enter" data-keybaord-key="Return">
|
||||
enter
|
||||
</div>
|
||||
</div>
|
||||
<div class="keybaord-row keybaord-row-4">
|
||||
<div class="keybaord-key keybaord-capslock" data-keybaord-key="HAMBAGA">
|
||||
🍔
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="a">
|
||||
a
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="s">
|
||||
s
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="d">
|
||||
d
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="f">
|
||||
f
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="g">
|
||||
g
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="h">
|
||||
h
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="j">
|
||||
j
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="k">
|
||||
k
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="l">
|
||||
l
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key=";">
|
||||
;
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="'">
|
||||
'
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-backslash" data-keybaord-key="\\">
|
||||
\\
|
||||
</div>
|
||||
</div>
|
||||
<div class="keybaord-row keybaord-row-5">
|
||||
<div class="keybaord-key keybaord-lshift" data-keybaord-key="Shift_L">
|
||||
shift
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="\\">
|
||||
\\
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="z">
|
||||
z
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="x">
|
||||
x
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="c">
|
||||
c
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="v">
|
||||
v
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="b">
|
||||
b
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="n">
|
||||
n
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="m">
|
||||
m
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key=",">
|
||||
,
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key=".">
|
||||
.
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="/">
|
||||
/
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-rshift" data-keybaord-key="Shift_R">
|
||||
shift
|
||||
</div>
|
||||
</div>
|
||||
<div class="keybaord-row keybaord-row-6">
|
||||
<div class="keybaord-key keybaord-lctrl" data-keybaord-key="Control_L">
|
||||
ctrl
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-super" data-keybaord-key="Meta_L">
|
||||
win
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-alt" data-keybaord-key="Alt_L">
|
||||
alt
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-spacebar" data-keybaord-key="space">
|
||||
space
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-altgr" data-keybaord-key="Alt_R">
|
||||
altgr
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-what" data-keybaord-key="Menu">
|
||||
menu
|
||||
</div>
|
||||
<div class="keybaord-key keybaord-rctrl" data-keybaord-key="Control_R">
|
||||
ctrl
|
||||
</div>
|
||||
</div>
|
||||
<div class="keybaord-row">
|
||||
<div class="keybaord-key" data-keybaord-key="XF86AudioLowerVolume">
|
||||
🔉
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="XF86AudioRaiseVolume">
|
||||
🔊
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Left">
|
||||
⬅️
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Down">
|
||||
⬇️
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Up">
|
||||
⬆️
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Right">
|
||||
➡️
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Page_Up">
|
||||
PgUp
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Page_Down">
|
||||
PgDn
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="Home">
|
||||
🏠
|
||||
</div>
|
||||
<div class="keybaord-key" data-keybaord-key="End">
|
||||
End
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
`;
|
||||
|
||||
function arraySample(array) {
|
||||
return array[Math.floor(Math.random() * array.length)];
|
||||
}
|
||||
|
||||
function sendMessage(msg) {
|
||||
return fetch(BASE_URL, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
|
||||
},
|
||||
body: "msg=" + encodeURIComponent(msg),
|
||||
}).then(
|
||||
(r) => r.text(), // so the response body shows up in network tab
|
||||
(err) => consoleError(err)
|
||||
);
|
||||
}
|
||||
const MODIFIER_ON_CLASS = "keybaord-modifier-on";
|
||||
const KEY_DATASET = "data-keybaord-key";
|
||||
const KEY_CLASS = "keybaord-key";
|
||||
|
||||
const modifiers = new Set()
|
||||
|
||||
function toggleModifier(button, key) {
|
||||
button.classList.toggle(MODIFIER_ON_CLASS);
|
||||
if (modifiers.has(key)) {
|
||||
modifiers.delete(key);
|
||||
} else {
|
||||
modifiers.add(key);
|
||||
}
|
||||
}
|
||||
|
||||
function popModifiers() {
|
||||
let modifierString = "";
|
||||
|
||||
modifiers.forEach((mod) => {
|
||||
document.querySelector("[" + KEY_DATASET + "='" + mod + "']")
|
||||
.classList.remove(MODIFIER_ON_CLASS);
|
||||
|
||||
modifierString += mod + "+";
|
||||
});
|
||||
|
||||
modifiers.clear();
|
||||
|
||||
return modifierString;
|
||||
}
|
||||
|
||||
Array.from(document.querySelectorAll("." + KEY_CLASS)).forEach((button) => {
|
||||
const key = button.dataset.keybaordKey;
|
||||
|
||||
button.addEventListener("click", (ev) => {
|
||||
switch (key) {
|
||||
case "HAMBAGA":
|
||||
sendMessage(arraySample(HAMBAGA));
|
||||
break;
|
||||
|
||||
case "Shift_L":
|
||||
case "Shift_R":
|
||||
|
||||
case "Control_L":
|
||||
case "Control_R":
|
||||
|
||||
case "Meta_L":
|
||||
|
||||
case "Alt_L":
|
||||
case "Alt_R":
|
||||
toggleModifier(button, key);
|
||||
break;
|
||||
|
||||
default: {
|
||||
const keyWithModifiers = popModifiers() + key;
|
||||
|
||||
consoleLog(keyWithModifiers);
|
||||
|
||||
sendMessage("key " + keyWithModifiers)
|
||||
.then(() => consoleLog(keyWithModifiers + " OK"));
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
// keybaord integration
|
||||
(function () {
|
||||
var o = mknod('div');
|
||||
clmod(o, 'keybaord-container', 1);
|
||||
ebi('op_msg').appendChild(o);
|
||||
|
||||
o = mknod('style');
|
||||
o.innerHTML = `
|
||||
.keybaord-body {
|
||||
display: flex;
|
||||
flex-flow: column nowrap;
|
||||
margin: .6em 0;
|
||||
}
|
||||
|
||||
.keybaord-row {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.keybaord-key {
|
||||
border: 1px solid rgba(128,128,128,0.2);
|
||||
width: 41px;
|
||||
height: 40px;
|
||||
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.keybaord-key:active {
|
||||
background-color: lightgrey;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-modifier-on {
|
||||
background-color: lightblue;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-backspace {
|
||||
width: 82px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-tab {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-enter {
|
||||
width: 69px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-capslock {
|
||||
width: 80px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-backslash {
|
||||
width: 88px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-lshift {
|
||||
width: 65px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-rshift {
|
||||
width: 103px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-lctrl {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-super {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-alt {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-altgr {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-what {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-rctrl {
|
||||
width: 55px;
|
||||
}
|
||||
|
||||
.keybaord-key.keybaord-spacebar {
|
||||
width: 302px;
|
||||
}
|
||||
`;
|
||||
document.head.appendChild(o);
|
||||
|
||||
initKeybaord('/', hambagas,
|
||||
(msg) => { toast.inf(2, msg.toString()) },
|
||||
(msg) => { toast.err(30, msg.toString()) });
|
||||
})();
|
||||
|
||||
|
||||
// live2d (dumb pointless meme)
|
||||
// dependencies for this part are not tracked in git
|
||||
// so delete this section if you wanna use this file
|
||||
// (or supply your own l2d model and js)
|
||||
(function () {
|
||||
var o = mknod('link');
|
||||
o.setAttribute('rel', 'stylesheet');
|
||||
o.setAttribute('href', "/bad-memes/pio.css");
|
||||
document.head.appendChild(o);
|
||||
|
||||
o = mknod('style');
|
||||
o.innerHTML = '.pio-container{text-shadow:none;z-index:1}';
|
||||
document.head.appendChild(o);
|
||||
|
||||
o = mknod('div');
|
||||
clmod(o, 'pio-container', 1);
|
||||
o.innerHTML = '<div class="pio-action"></div><canvas id="pio" width="280" height="500"></canvas>';
|
||||
document.body.appendChild(o);
|
||||
|
||||
var remaining = 3;
|
||||
for (var a of ['pio', 'l2d', 'fireworks']) {
|
||||
import_js(`/bad-memes/${a}.js`, function () {
|
||||
if (remaining --> 1)
|
||||
return;
|
||||
|
||||
o = mknod('script');
|
||||
o.innerHTML = 'var pio = new Paul_Pio({"selector":[],"mode":"fixed","hidden":false,"content":{"close":"ok bye"},"model":["/bad-memes/sagiri/model.json"]});';
|
||||
document.body.appendChild(o);
|
||||
});
|
||||
}
|
||||
})();
|
||||
@@ -1,59 +0,0 @@
|
||||
/*
|
||||
|
||||
makes the up2k ui REALLY minimal by hiding a bunch of stuff
|
||||
|
||||
almost the same as minimal-up2k.html except this one...:
|
||||
|
||||
-- applies to every write-only folder when used with --js-browser
|
||||
|
||||
-- only applies if javascript is enabled
|
||||
|
||||
-- doesn't hide the total upload ETA display
|
||||
|
||||
-- looks slightly better
|
||||
|
||||
*/
|
||||
|
||||
var u2min = `
|
||||
<style>
|
||||
|
||||
#ops, #path, #tree, #files, #epi+div+h2,
|
||||
#u2conf td.c+.c, #u2cards, #srch_dz, #srch_zd {
|
||||
display: none !important;
|
||||
}
|
||||
#u2conf {margin:5em auto 0 auto !important}
|
||||
#u2conf.ww {width:70em}
|
||||
#u2conf.w {width:50em}
|
||||
#u2conf.w .c,
|
||||
#u2conf.w #u2btn_cw {text-align:left}
|
||||
#u2conf.w #u2btn_cw {width:70%}
|
||||
#u2etaw {margin:3em auto}
|
||||
#u2etaw.w {
|
||||
text-align: center;
|
||||
margin: -3.5em auto 5em auto;
|
||||
}
|
||||
#u2etaw.w #u2etas {margin-right:-37em}
|
||||
#u2etaw.w #u2etas.o {margin-top:-2.2em}
|
||||
#u2etaw.ww {margin:-1em auto}
|
||||
#u2etaw.ww #u2etas {padding-left:4em}
|
||||
#u2etas {
|
||||
background: none !important;
|
||||
border: none !important;
|
||||
}
|
||||
#wrap {margin-left:2em !important}
|
||||
.logue {
|
||||
border: none !important;
|
||||
margin: 2em auto !important;
|
||||
}
|
||||
.logue:before {content:'' !important}
|
||||
|
||||
</style>
|
||||
|
||||
<a href="#" onclick="this.parentNode.innerHTML='';">show advanced options</a>
|
||||
`;
|
||||
|
||||
if (!has(perms, 'read')) {
|
||||
var e2 = mknod('div');
|
||||
e2.innerHTML = u2min;
|
||||
ebi('wrap').insertBefore(e2, QS('#epi+h2'));
|
||||
}
|
||||
@@ -1,297 +0,0 @@
|
||||
// way more specific example --
|
||||
// assumes all files dropped into the uploader have a youtube-id somewhere in the filename,
|
||||
// locates the youtube-ids and passes them to an API which returns a list of IDs which should be uploaded
|
||||
//
|
||||
// also tries to find the youtube-id in the embedded metadata
|
||||
//
|
||||
// assumes copyparty is behind nginx as /ytq is a standalone service which must be rproxied in place
|
||||
|
||||
function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
|
||||
var passthru = up2k.uc.fsearch;
|
||||
if (passthru)
|
||||
return hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
|
||||
|
||||
a_up2k_namefilter(good_files, nil_files, bad_files, hooks).then(() => { });
|
||||
}
|
||||
|
||||
// ebi('op_up2k').appendChild(mknod('input','unick'));
|
||||
|
||||
function bstrpos(buf, ptn) {
|
||||
var ofs = 0,
|
||||
ch0 = ptn[0],
|
||||
sz = buf.byteLength;
|
||||
|
||||
while (true) {
|
||||
ofs = buf.indexOf(ch0, ofs);
|
||||
if (ofs < 0 || ofs >= sz)
|
||||
return -1;
|
||||
|
||||
for (var a = 1; a < ptn.length; a++)
|
||||
if (buf[ofs + a] !== ptn[a])
|
||||
break;
|
||||
|
||||
if (a === ptn.length)
|
||||
return ofs;
|
||||
|
||||
++ofs;
|
||||
}
|
||||
}
|
||||
|
||||
async function a_up2k_namefilter(good_files, nil_files, bad_files, hooks) {
|
||||
var t0 = Date.now(),
|
||||
yt_ids = new Set(),
|
||||
textdec = new TextDecoder('latin1'),
|
||||
md_ptn = new TextEncoder().encode('youtube.com/watch?v='),
|
||||
file_ids = [], // all IDs found for each good_files
|
||||
md_only = [], // `${id} ${fn}` where ID was only found in metadata
|
||||
mofs = 0,
|
||||
mnchk = 0,
|
||||
mfile = '',
|
||||
myid = localStorage.getItem('ytid_t0');
|
||||
|
||||
if (!myid)
|
||||
localStorage.setItem('ytid_t0', myid = Date.now());
|
||||
|
||||
for (var a = 0; a < good_files.length; a++) {
|
||||
var [fobj, name] = good_files[a],
|
||||
cname = name, // will clobber
|
||||
sz = fobj.size,
|
||||
ids = [],
|
||||
fn_ids = [],
|
||||
md_ids = [],
|
||||
id_ok = false,
|
||||
m;
|
||||
|
||||
// all IDs found in this file
|
||||
file_ids.push(ids);
|
||||
|
||||
// look for ID in filename; reduce the
|
||||
// metadata-scan intensity if the id looks safe
|
||||
m = /[\[(-]([\w-]{11})[\])]?\.(?:mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name);
|
||||
id_ok = !!m;
|
||||
|
||||
while (true) {
|
||||
// fuzzy catch-all;
|
||||
// some ytdl fork did %(title)-%(id).%(ext) ...
|
||||
m = /(?:^|[^\w])([\w-]{11})(?:$|[^\w-])/.exec(cname);
|
||||
if (!m)
|
||||
break;
|
||||
|
||||
cname = cname.replace(m[1], '');
|
||||
yt_ids.add(m[1]);
|
||||
fn_ids.unshift(m[1]);
|
||||
}
|
||||
|
||||
// look for IDs in video metadata,
|
||||
if (/\.(mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name)) {
|
||||
toast.show('inf r', 0, `analyzing file ${a + 1} / ${good_files.length} :\n${name}\n\nhave analysed ${++mnchk} files in ${(Date.now() - t0) / 1000} seconds, ${humantime((good_files.length - (a + 1)) * (((Date.now() - t0) / 1000) / mnchk))} remaining,\n\nbiggest offset so far is ${mofs}, in this file:\n\n${mfile}`);
|
||||
|
||||
// check first and last 128 MiB;
|
||||
// pWxOroN5WCo.mkv @ 6edb98 (6.92M)
|
||||
// Nf-nN1wF5Xo.mp4 @ 4a98034 (74.6M)
|
||||
var chunksz = 1024 * 1024 * 2, // byte
|
||||
aspan = id_ok ? 128 : 512; // MiB
|
||||
|
||||
aspan = parseInt(Math.min(sz / 2, aspan * 1024 * 1024) / chunksz) * chunksz;
|
||||
if (!aspan)
|
||||
aspan = Math.min(sz, chunksz);
|
||||
|
||||
for (var side = 0; side < 2; side++) {
|
||||
var ofs = side ? Math.max(0, sz - aspan) : 0,
|
||||
nchunks = aspan / chunksz;
|
||||
|
||||
for (var chunk = 0; chunk < nchunks; chunk++) {
|
||||
var bchunk = await fobj.slice(ofs, ofs + chunksz + 16).arrayBuffer(),
|
||||
uchunk = new Uint8Array(bchunk, 0, bchunk.byteLength),
|
||||
bofs = bstrpos(uchunk, md_ptn),
|
||||
absofs = Math.min(ofs + bofs, (sz - ofs) + bofs),
|
||||
txt = bofs < 0 ? '' : textdec.decode(uchunk.subarray(bofs)),
|
||||
m;
|
||||
|
||||
//console.log(`side ${ side }, chunk ${ chunk }, ofs ${ ofs }, bchunk ${ bchunk.byteLength }, txt ${ txt.length }`);
|
||||
while (true) {
|
||||
// mkv/webm have [a-z] immediately after url
|
||||
m = /(youtube\.com\/watch\?v=[\w-]{11})/.exec(txt);
|
||||
if (!m)
|
||||
break;
|
||||
|
||||
txt = txt.replace(m[1], '');
|
||||
m = m[1].slice(-11);
|
||||
|
||||
console.log(`found ${m} @${bofs}, ${name} `);
|
||||
yt_ids.add(m);
|
||||
if (!has(fn_ids, m) && !has(md_ids, m)) {
|
||||
md_ids.push(m);
|
||||
md_only.push(`${m} ${name}`);
|
||||
}
|
||||
else
|
||||
// id appears several times; make it preferred
|
||||
md_ids.unshift(m);
|
||||
|
||||
// bail after next iteration
|
||||
chunk = nchunks - 1;
|
||||
side = 9;
|
||||
|
||||
if (mofs < absofs) {
|
||||
mofs = absofs;
|
||||
mfile = name;
|
||||
}
|
||||
}
|
||||
ofs += chunksz;
|
||||
if (ofs >= sz)
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (var yi of md_ids)
|
||||
ids.push(yi);
|
||||
|
||||
for (var yi of fn_ids)
|
||||
if (!has(ids, yi))
|
||||
ids.push(yi);
|
||||
}
|
||||
|
||||
if (md_only.length)
|
||||
console.log('recovered the following youtube-IDs by inspecting metadata:\n\n' + md_only.join('\n'));
|
||||
else if (yt_ids.size)
|
||||
console.log('did not discover any additional youtube-IDs by inspecting metadata; all the IDs also existed in the filenames');
|
||||
else
|
||||
console.log('failed to find any youtube-IDs at all, sorry');
|
||||
|
||||
if (false) {
|
||||
var msg = `finished analysing ${mnchk} files in ${(Date.now() - t0) / 1000} seconds,\n\nbiggest offset was ${mofs} in this file:\n\n${mfile}`,
|
||||
mfun = function () { toast.ok(0, msg); };
|
||||
|
||||
mfun();
|
||||
setTimeout(mfun, 200);
|
||||
|
||||
return hooks[0]([], [], [], hooks.slice(1));
|
||||
}
|
||||
|
||||
var el = ebi('unick'), unick = el ? el.value : '';
|
||||
if (unick) {
|
||||
console.log(`sending uploader nickname [${unick}]`);
|
||||
fetch(document.location, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' },
|
||||
body: 'msg=' + encodeURIComponent(unick)
|
||||
});
|
||||
}
|
||||
|
||||
toast.inf(5, `running query for ${yt_ids.size} youtube-IDs...`);
|
||||
|
||||
var xhr = new XHR();
|
||||
xhr.open('POST', '/ytq', true);
|
||||
xhr.setRequestHeader('Content-Type', 'text/plain');
|
||||
xhr.onload = xhr.onerror = function () {
|
||||
if (this.status != 200)
|
||||
return toast.err(0, `sorry, database query failed ;_;\n\nplease let us know so we can look at it, thx!!\n\nerror ${this.status}: ${(this.response && this.response.err) || this.responseText}`);
|
||||
|
||||
process_id_list(this.responseText);
|
||||
};
|
||||
xhr.send(Array.from(yt_ids).join('\n'));
|
||||
|
||||
function process_id_list(txt) {
|
||||
var wanted_ids = new Set(txt.trim().split('\n')),
|
||||
name_id = {},
|
||||
wanted_names = new Set(), // basenames with a wanted ID -- not including relpath
|
||||
wanted_names_scoped = {}, // basenames with a wanted ID -> list of dirs to search under
|
||||
wanted_files = new Set(); // filedrops
|
||||
|
||||
for (var a = 0; a < good_files.length; a++) {
|
||||
var name = good_files[a][1];
|
||||
for (var b = 0; b < file_ids[a].length; b++)
|
||||
if (wanted_ids.has(file_ids[a][b])) {
|
||||
// let the next stage handle this to prevent dupes
|
||||
//wanted_files.add(good_files[a]);
|
||||
|
||||
var m = /(.*)\.(mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name);
|
||||
if (!m)
|
||||
continue;
|
||||
|
||||
var [rd, fn] = vsplit(m[1]);
|
||||
|
||||
if (fn in wanted_names_scoped)
|
||||
wanted_names_scoped[fn].push(rd);
|
||||
else
|
||||
wanted_names_scoped[fn] = [rd];
|
||||
|
||||
wanted_names.add(fn);
|
||||
name_id[m[1]] = file_ids[a][b];
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// add all files with the same basename as each explicitly wanted file
|
||||
// (infojson/chatlog/etc when ID was discovered from metadata)
|
||||
for (var a = 0; a < good_files.length; a++) {
|
||||
var [rd, name] = vsplit(good_files[a][1]);
|
||||
for (var b = 0; b < 3; b++) {
|
||||
name = name.replace(/\.[^\.]+$/, '');
|
||||
if (!wanted_names.has(name))
|
||||
continue;
|
||||
|
||||
var vid_fp = false;
|
||||
for (var c of wanted_names_scoped[name])
|
||||
if (rd.startsWith(c))
|
||||
vid_fp = c + name;
|
||||
|
||||
if (!vid_fp)
|
||||
continue;
|
||||
|
||||
var subdir = name_id[vid_fp];
|
||||
subdir = `v${subdir.slice(0, 1)}/${subdir}-${myid}`;
|
||||
var newpath = subdir + '/' + good_files[a][1].split(/\//g).pop();
|
||||
|
||||
// check if this file is a dupe
|
||||
for (var c of good_files)
|
||||
if (c[1] == newpath)
|
||||
newpath = null;
|
||||
|
||||
if (!newpath)
|
||||
break;
|
||||
|
||||
good_files[a][1] = newpath;
|
||||
wanted_files.add(good_files[a]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
function upload_filtered() {
|
||||
if (!wanted_files.size)
|
||||
return modal.alert('Good news -- turns out we already have all those.\n\nBut thank you for checking in!');
|
||||
|
||||
hooks[0](Array.from(wanted_files), nil_files, bad_files, hooks.slice(1));
|
||||
}
|
||||
|
||||
function upload_all() {
|
||||
hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
|
||||
}
|
||||
|
||||
var n_skip = good_files.length - wanted_files.size,
|
||||
msg = `you added ${good_files.length} files; ${good_files.length == n_skip ? 'all' : n_skip} of them were skipped --\neither because we already have them,\nor because there is no youtube-ID in your filenames.\n\n<code>OK</code> / <code>Enter</code> = continue uploading just the ${wanted_files.size} files we definitely need\n\n<code>Cancel</code> / <code>ESC</code> = override the filter; upload ALL the files you added`;
|
||||
|
||||
if (!n_skip)
|
||||
upload_filtered();
|
||||
else
|
||||
modal.confirm(msg, upload_filtered, upload_all);
|
||||
};
|
||||
}
|
||||
|
||||
up2k_hooks.push(function () {
|
||||
up2k.gotallfiles.unshift(up2k_namefilter);
|
||||
});
|
||||
|
||||
// persist/restore nickname field if present
|
||||
setInterval(function () {
|
||||
var o = ebi('unick');
|
||||
if (!o || document.activeElement == o)
|
||||
return;
|
||||
|
||||
o.oninput = function () {
|
||||
localStorage.setItem('unick', o.value);
|
||||
};
|
||||
o.value = localStorage.getItem('unick') || '';
|
||||
}, 1000);
|
||||
@@ -1,45 +0,0 @@
|
||||
// hooks into up2k
|
||||
|
||||
function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
|
||||
// is called when stuff is dropped into the browser,
|
||||
// after iterating through the directory tree and discovering all files,
|
||||
// before the upload confirmation dialogue is shown
|
||||
|
||||
// good_files will successfully upload
|
||||
// nil_files are empty files and will show an alert in the final hook
|
||||
// bad_files are unreadable and cannot be uploaded
|
||||
var file_lists = [good_files, nil_files, bad_files];
|
||||
|
||||
// build a list of filenames
|
||||
var filenames = [];
|
||||
for (var lst of file_lists)
|
||||
for (var ent of lst)
|
||||
filenames.push(ent[1]);
|
||||
|
||||
toast.inf(5, "running database query...");
|
||||
|
||||
// simulate delay while passing the list to some api for checking
|
||||
setTimeout(function () {
|
||||
|
||||
// only keep webm files as an example
|
||||
var new_lists = [];
|
||||
for (var lst of file_lists) {
|
||||
var keep = [];
|
||||
new_lists.push(keep);
|
||||
|
||||
for (var ent of lst)
|
||||
if (/\.webm$/.test(ent[1]))
|
||||
keep.push(ent);
|
||||
}
|
||||
|
||||
// finally, call the next hook in the chain
|
||||
[good_files, nil_files, bad_files] = new_lists;
|
||||
hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
|
||||
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
// register
|
||||
up2k_hooks.push(function () {
|
||||
up2k.gotallfiles.unshift(up2k_namefilter);
|
||||
});
|
||||
@@ -1,31 +0,0 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# PROVIDE: copyparty
|
||||
# REQUIRE: networking
|
||||
# KEYWORD:
|
||||
|
||||
. /etc/rc.subr
|
||||
|
||||
name="copyparty"
|
||||
rcvar="copyparty_enable"
|
||||
copyparty_user="copyparty"
|
||||
copyparty_args="-e2dsa -v /storage:/storage:r" # change as you see fit
|
||||
copyparty_command="/usr/local/bin/python3.8 /usr/local/copyparty/copyparty-sfx.py ${copyparty_args}"
|
||||
pidfile="/var/run/copyparty/${name}.pid"
|
||||
command="/usr/sbin/daemon"
|
||||
command_args="-P ${pidfile} -r -f ${copyparty_command}"
|
||||
|
||||
stop_postcmd="copyparty_shutdown"
|
||||
|
||||
copyparty_shutdown()
|
||||
{
|
||||
if [ -e "${pidfile}" ]; then
|
||||
echo "Stopping supervising daemon."
|
||||
kill -s TERM `cat ${pidfile}`
|
||||
fi
|
||||
}
|
||||
|
||||
load_rc_config $name
|
||||
: ${copyparty_enable:=no}
|
||||
|
||||
run_rc_command "$1"
|
||||
@@ -1,23 +0,0 @@
|
||||
# systemd service which generates a new TLS certificate on each boot,
|
||||
# that way the one-year expiry time won't cause any issues --
|
||||
# just have everyone trust the ca.pem once every 10 years
|
||||
#
|
||||
# assumptions/placeholder values:
|
||||
# * this script and copyparty runs as user "cpp"
|
||||
# * copyparty repo is at ~cpp/dev/copyparty
|
||||
# * CA is named partylan
|
||||
# * server IPs = 10.1.2.3 and 192.168.123.1
|
||||
# * server hostname = party.lan
|
||||
|
||||
[Unit]
|
||||
Description=copyparty certificate generator
|
||||
Before=copyparty.service
|
||||
|
||||
[Service]
|
||||
User=cpp
|
||||
Type=oneshot
|
||||
SyslogIdentifier=cpp-cert
|
||||
ExecStart=/bin/bash -c 'cd ~/dev/copyparty/contrib && ./cfssl.sh partylan 10.1.2.3,192.168.123.1,party.lan y'
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -2,60 +2,18 @@
|
||||
# and share '/mnt' with anonymous read+write
|
||||
#
|
||||
# installation:
|
||||
# cp -pv copyparty.service /etc/systemd/system
|
||||
# restorecon -vr /etc/systemd/system/copyparty.service
|
||||
# firewall-cmd --permanent --add-port={80,443,3923}/tcp # --zone=libvirt
|
||||
# firewall-cmd --reload
|
||||
# systemctl daemon-reload && systemctl enable --now copyparty
|
||||
# cp -pv copyparty.service /etc/systemd/system && systemctl enable --now copyparty
|
||||
#
|
||||
# you may want to:
|
||||
# change "User=cpp" and "/home/cpp/" to another user
|
||||
# remove the nft lines to only listen on port 3923
|
||||
# and in the ExecStart= line:
|
||||
# change '/usr/bin/python3' to another interpreter
|
||||
# change '/mnt::rw' to another location or permission-set
|
||||
# add '-q' to disable logging on busy servers
|
||||
# add '-i 127.0.0.1' to only allow local connections
|
||||
# add '-e2dsa' to enable filesystem scanning + indexing
|
||||
# add '-e2ts' to enable metadata indexing
|
||||
#
|
||||
# with `Type=notify`, copyparty will signal systemd when it is ready to
|
||||
# accept connections; correctly delaying units depending on copyparty.
|
||||
# But note that journalctl will get the timestamps wrong due to
|
||||
# python disabling line-buffering, so messages are out-of-order:
|
||||
# https://user-images.githubusercontent.com/241032/126040249-cb535cc7-c599-4931-a796-a5d9af691bad.png
|
||||
#
|
||||
# unless you add -q to disable logging, you may want to remove the
|
||||
# following line to allow buffering (slightly better performance):
|
||||
# Environment=PYTHONUNBUFFERED=x
|
||||
#
|
||||
# keep ExecStartPre before ExecStart, at least on rhel8
|
||||
# change '/usr/bin/python' to another interpreter
|
||||
# change '/mnt::a' to another location or permission-set
|
||||
|
||||
[Unit]
|
||||
Description=copyparty file server
|
||||
|
||||
[Service]
|
||||
Type=notify
|
||||
SyslogIdentifier=copyparty
|
||||
Environment=PYTHONUNBUFFERED=x
|
||||
ExecReload=/bin/kill -s USR1 $MAINPID
|
||||
|
||||
# user to run as + where the TLS certificate is (if any)
|
||||
User=cpp
|
||||
Environment=XDG_CONFIG_HOME=/home/cpp/.config
|
||||
|
||||
# setup forwarding from ports 80 and 443 to port 3923
|
||||
ExecStartPre=+/bin/bash -c 'nft -n -a list table nat | awk "/ to :3923 /{print\$NF}" | xargs -rL1 nft delete rule nat prerouting handle; true'
|
||||
ExecStartPre=+nft add table ip nat
|
||||
ExecStartPre=+nft -- add chain ip nat prerouting { type nat hook prerouting priority -100 \; }
|
||||
ExecStartPre=+nft add rule ip nat prerouting tcp dport 80 redirect to :3923
|
||||
ExecStartPre=+nft add rule ip nat prerouting tcp dport 443 redirect to :3923
|
||||
|
||||
# stop systemd-tmpfiles-clean.timer from deleting copyparty while it's running
|
||||
ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
|
||||
|
||||
# copyparty settings
|
||||
ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -e2d -v /mnt::rw
|
||||
ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
|
||||
ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# this will start `/usr/local/bin/copyparty-sfx.py`
|
||||
# in a chroot, preventing accidental access elsewhere
|
||||
# and share '/mnt' with anonymous read+write
|
||||
#
|
||||
# installation:
|
||||
# 1) put copyparty-sfx.py and prisonparty.sh in /usr/local/bin
|
||||
# 2) cp -pv prisonparty.service /etc/systemd/system && systemctl enable --now prisonparty
|
||||
#
|
||||
# you may want to:
|
||||
# change '/mnt::rw' to another location or permission-set
|
||||
# (remember to change the '/mnt' chroot arg too)
|
||||
#
|
||||
# enable line-buffering for realtime logging (slight performance cost):
|
||||
# inside the [Service] block, add the following line:
|
||||
# Environment=PYTHONUNBUFFERED=x
|
||||
|
||||
[Unit]
|
||||
Description=copyparty file server
|
||||
|
||||
[Service]
|
||||
SyslogIdentifier=prisonparty
|
||||
WorkingDirectory=/usr/local/bin
|
||||
ExecStart=/bin/bash /usr/local/bin/prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt -- \
|
||||
/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::rw
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,48 +1,50 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
import time
|
||||
|
||||
try:
|
||||
from typing import TYPE_CHECKING, Any
|
||||
except:
|
||||
TYPE_CHECKING = False
|
||||
import sys
|
||||
import os
|
||||
|
||||
PY2 = sys.version_info[0] == 2
|
||||
if PY2:
|
||||
sys.dont_write_bytecode = True
|
||||
unicode = unicode # noqa: F821 # pylint: disable=undefined-variable,self-assigning-variable
|
||||
else:
|
||||
unicode = str
|
||||
|
||||
WINDOWS: Any = (
|
||||
[int(x) for x in platform.version().split(".")]
|
||||
if platform.system() == "Windows"
|
||||
else False
|
||||
)
|
||||
WINDOWS = False
|
||||
if platform.system() == "Windows":
|
||||
WINDOWS = [int(x) for x in platform.version().split(".")]
|
||||
|
||||
VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
|
||||
# introduced in anniversary update
|
||||
|
||||
ANYWIN = WINDOWS or sys.platform in ["msys", "cygwin"]
|
||||
ANYWIN = WINDOWS or sys.platform in ["msys"]
|
||||
|
||||
MACOS = platform.system() == "Darwin"
|
||||
|
||||
try:
|
||||
CORES = len(os.sched_getaffinity(0))
|
||||
except:
|
||||
CORES = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 2
|
||||
|
||||
|
||||
class EnvParams(object):
|
||||
def __init__(self) -> None:
|
||||
def __init__(self):
|
||||
self.t0 = time.time()
|
||||
self.mod = None
|
||||
self.cfg = None
|
||||
self.ox = getattr(sys, "oxidized", None)
|
||||
self.mod = os.path.dirname(os.path.realpath(__file__))
|
||||
if self.mod.endswith("__init__"):
|
||||
self.mod = os.path.dirname(self.mod)
|
||||
|
||||
if sys.platform == "win32":
|
||||
self.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
|
||||
elif sys.platform == "darwin":
|
||||
self.cfg = os.path.expanduser("~/Library/Preferences/copyparty")
|
||||
else:
|
||||
self.cfg = os.path.normpath(
|
||||
os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config"))
|
||||
+ "/copyparty"
|
||||
)
|
||||
|
||||
self.cfg = self.cfg.replace("\\", "/")
|
||||
try:
|
||||
os.makedirs(self.cfg)
|
||||
except:
|
||||
if not os.path.isdir(self.cfg):
|
||||
raise
|
||||
|
||||
|
||||
E = EnvParams()
|
||||
|
||||
798
copyparty/__main__.py
Executable file → Normal file
798
copyparty/__main__.py
Executable file → Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,8 @@
|
||||
# coding: utf-8
|
||||
|
||||
VERSION = (1, 4, 1)
|
||||
CODENAME = "mostly reliable"
|
||||
BUILD_DT = (2022, 9, 24)
|
||||
VERSION = (0, 11, 29)
|
||||
CODENAME = "the grid"
|
||||
BUILD_DT = (2021, 6, 30)
|
||||
|
||||
S_VERSION = ".".join(map(str, VERSION))
|
||||
S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)
|
||||
|
||||
1143
copyparty/authsrv.py
1143
copyparty/authsrv.py
File diff suppressed because it is too large
Load Diff
@@ -1,76 +0,0 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
|
||||
from ..util import SYMTIME, fsdec, fsenc
|
||||
from . import path
|
||||
|
||||
try:
|
||||
from typing import Optional
|
||||
except:
|
||||
pass
|
||||
|
||||
_ = (path,)
|
||||
|
||||
# grep -hRiE '(^|[^a-zA-Z_\.-])os\.' . | gsed -r 's/ /\n/g;s/\(/(\n/g' | grep -hRiE '(^|[^a-zA-Z_\.-])os\.' | sort | uniq -c
|
||||
# printf 'os\.(%s)' "$(grep ^def bos/__init__.py | gsed -r 's/^def //;s/\(.*//' | tr '\n' '|' | gsed -r 's/.$//')"
|
||||
|
||||
|
||||
def chmod(p: str, mode: int) -> None:
|
||||
return os.chmod(fsenc(p), mode)
|
||||
|
||||
|
||||
def listdir(p: str = ".") -> list[str]:
|
||||
return [fsdec(x) for x in os.listdir(fsenc(p))]
|
||||
|
||||
|
||||
def makedirs(name: str, mode: int = 0o755, exist_ok: bool = True) -> None:
|
||||
bname = fsenc(name)
|
||||
try:
|
||||
os.makedirs(bname, mode)
|
||||
except:
|
||||
if not exist_ok or not os.path.isdir(bname):
|
||||
raise
|
||||
|
||||
|
||||
def mkdir(p: str, mode: int = 0o755) -> None:
|
||||
return os.mkdir(fsenc(p), mode)
|
||||
|
||||
|
||||
def rename(src: str, dst: str) -> None:
|
||||
return os.rename(fsenc(src), fsenc(dst))
|
||||
|
||||
|
||||
def replace(src: str, dst: str) -> None:
|
||||
return os.replace(fsenc(src), fsenc(dst))
|
||||
|
||||
|
||||
def rmdir(p: str) -> None:
|
||||
return os.rmdir(fsenc(p))
|
||||
|
||||
|
||||
def stat(p: str) -> os.stat_result:
|
||||
return os.stat(fsenc(p))
|
||||
|
||||
|
||||
def unlink(p: str) -> None:
|
||||
return os.unlink(fsenc(p))
|
||||
|
||||
|
||||
def utime(
|
||||
p: str, times: Optional[tuple[float, float]] = None, follow_symlinks: bool = True
|
||||
) -> None:
|
||||
if SYMTIME:
|
||||
return os.utime(fsenc(p), times, follow_symlinks=follow_symlinks)
|
||||
else:
|
||||
return os.utime(fsenc(p), times)
|
||||
|
||||
|
||||
if hasattr(os, "lstat"):
|
||||
|
||||
def lstat(p: str) -> os.stat_result:
|
||||
return os.lstat(fsenc(p))
|
||||
|
||||
else:
|
||||
lstat = stat
|
||||
@@ -1,45 +0,0 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
|
||||
from ..util import SYMTIME, fsdec, fsenc
|
||||
|
||||
|
||||
def abspath(p: str) -> str:
|
||||
return fsdec(os.path.abspath(fsenc(p)))
|
||||
|
||||
|
||||
def exists(p: str) -> bool:
|
||||
return os.path.exists(fsenc(p))
|
||||
|
||||
|
||||
def getmtime(p: str, follow_symlinks: bool = True) -> float:
|
||||
if not follow_symlinks and SYMTIME:
|
||||
return os.lstat(fsenc(p)).st_mtime
|
||||
else:
|
||||
return os.path.getmtime(fsenc(p))
|
||||
|
||||
|
||||
def getsize(p: str) -> int:
|
||||
return os.path.getsize(fsenc(p))
|
||||
|
||||
|
||||
def isfile(p: str) -> bool:
|
||||
return os.path.isfile(fsenc(p))
|
||||
|
||||
|
||||
def isdir(p: str) -> bool:
|
||||
return os.path.isdir(fsenc(p))
|
||||
|
||||
|
||||
def islink(p: str) -> bool:
|
||||
return os.path.islink(fsenc(p))
|
||||
|
||||
|
||||
def lexists(p: str) -> bool:
|
||||
return os.path.lexists(fsenc(p))
|
||||
|
||||
|
||||
def realpath(p: str) -> str:
|
||||
return fsdec(os.path.realpath(fsenc(p)))
|
||||
@@ -1,59 +1,51 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import threading
|
||||
import time
|
||||
import threading
|
||||
|
||||
import queue
|
||||
|
||||
from .__init__ import CORES, TYPE_CHECKING
|
||||
from .broker_mpw import MpWorker
|
||||
from .__init__ import PY2, WINDOWS, VT100
|
||||
from .broker_util import try_exec
|
||||
from .broker_mpw import MpWorker
|
||||
from .util import mp
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
|
||||
try:
|
||||
from typing import Any
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class MProcess(mp.Process):
|
||||
def __init__(
|
||||
self,
|
||||
q_pend: queue.Queue[tuple[int, str, list[Any]]],
|
||||
q_yield: queue.Queue[tuple[int, str, list[Any]]],
|
||||
target: Any,
|
||||
args: Any,
|
||||
) -> None:
|
||||
super(MProcess, self).__init__(target=target, args=args)
|
||||
self.q_pend = q_pend
|
||||
self.q_yield = q_yield
|
||||
if PY2 and not WINDOWS:
|
||||
from multiprocessing.reduction import ForkingPickler
|
||||
from StringIO import StringIO as MemesIO # pylint: disable=import-error
|
||||
|
||||
|
||||
class BrokerMp(object):
|
||||
"""external api; manages MpWorkers"""
|
||||
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
def __init__(self, hub):
|
||||
self.hub = hub
|
||||
self.log = hub.log
|
||||
self.args = hub.args
|
||||
|
||||
self.procs = []
|
||||
self.retpend = {}
|
||||
self.retpend_mutex = threading.Lock()
|
||||
self.mutex = threading.Lock()
|
||||
|
||||
self.num_workers = self.args.j or CORES
|
||||
self.log("broker", "booting {} subprocesses".format(self.num_workers))
|
||||
for n in range(1, self.num_workers + 1):
|
||||
q_pend: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(1)
|
||||
q_yield: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(64)
|
||||
cores = self.args.j
|
||||
if not cores:
|
||||
cores = mp.cpu_count()
|
||||
|
||||
proc = MProcess(q_pend, q_yield, MpWorker, (q_pend, q_yield, self.args, n))
|
||||
self.log("broker", "booting {} subprocesses".format(cores))
|
||||
for n in range(cores):
|
||||
q_pend = mp.Queue(1)
|
||||
q_yield = mp.Queue(64)
|
||||
|
||||
proc = mp.Process(target=MpWorker, args=(q_pend, q_yield, self.args, n))
|
||||
proc.q_pend = q_pend
|
||||
proc.q_yield = q_yield
|
||||
proc.nid = n
|
||||
proc.clients = {}
|
||||
proc.workload = 0
|
||||
|
||||
thr = threading.Thread(
|
||||
target=self.collector, args=(proc,), name="mp-sink-{}".format(n)
|
||||
target=self.collector, args=(proc,), name="mp-collector"
|
||||
)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
@@ -61,11 +53,18 @@ class BrokerMp(object):
|
||||
self.procs.append(proc)
|
||||
proc.start()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if not self.args.q:
|
||||
thr = threading.Thread(
|
||||
target=self.debug_load_balancer, name="mp-dbg-loadbalancer"
|
||||
)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
def shutdown(self):
|
||||
self.log("broker", "shutting down")
|
||||
for n, proc in enumerate(self.procs):
|
||||
thr = threading.Thread(
|
||||
target=proc.q_pend.put((0, "shutdown", [])),
|
||||
target=proc.q_pend.put([0, "shutdown", []]),
|
||||
name="mp-shutdown-{}-{}".format(n, len(self.procs)),
|
||||
)
|
||||
thr.start()
|
||||
@@ -81,12 +80,7 @@ class BrokerMp(object):
|
||||
|
||||
procs.pop()
|
||||
|
||||
def reload(self) -> None:
|
||||
self.log("broker", "reloading")
|
||||
for _, proc in enumerate(self.procs):
|
||||
proc.q_pend.put((0, "reload", []))
|
||||
|
||||
def collector(self, proc: MProcess) -> None:
|
||||
def collector(self, proc):
|
||||
"""receive message from hub in other process"""
|
||||
while True:
|
||||
msg = proc.q_yield.get()
|
||||
@@ -95,9 +89,26 @@ class BrokerMp(object):
|
||||
if dest == "log":
|
||||
self.log(*args)
|
||||
|
||||
elif dest == "workload":
|
||||
with self.mutex:
|
||||
proc.workload = args[0]
|
||||
|
||||
elif dest == "httpdrop":
|
||||
addr = args[0]
|
||||
|
||||
with self.mutex:
|
||||
del proc.clients[addr]
|
||||
if not proc.clients:
|
||||
proc.workload = 0
|
||||
|
||||
self.hub.tcpsrv.num_clients.add(-1)
|
||||
|
||||
elif dest == "retq":
|
||||
# response from previous ipc call
|
||||
raise Exception("invalid broker_mp usage")
|
||||
with self.retpend_mutex:
|
||||
retq = self.retpend.pop(retq_id)
|
||||
|
||||
retq.put(args)
|
||||
|
||||
else:
|
||||
# new ipc invoking managed service in hub
|
||||
@@ -109,20 +120,46 @@ class BrokerMp(object):
|
||||
rv = try_exec(retq_id, obj, *args)
|
||||
|
||||
if retq_id:
|
||||
proc.q_pend.put((retq_id, "retq", rv))
|
||||
proc.q_pend.put([retq_id, "retq", rv])
|
||||
|
||||
def say(self, dest: str, *args: Any) -> None:
|
||||
def put(self, want_retval, dest, *args):
|
||||
"""
|
||||
send message to non-hub component in other process,
|
||||
returns a Queue object which eventually contains the response if want_retval
|
||||
(not-impl here since nothing uses it yet)
|
||||
"""
|
||||
if dest == "listen":
|
||||
for p in self.procs:
|
||||
p.q_pend.put((0, dest, [args[0], len(self.procs)]))
|
||||
if dest == "httpconn":
|
||||
sck, addr = args
|
||||
sck2 = sck
|
||||
if PY2:
|
||||
buf = MemesIO()
|
||||
ForkingPickler(buf).dump(sck)
|
||||
sck2 = buf.getvalue()
|
||||
|
||||
elif dest == "cb_httpsrv_up":
|
||||
self.hub.cb_httpsrv_up()
|
||||
proc = sorted(self.procs, key=lambda x: x.workload)[0]
|
||||
proc.q_pend.put([0, dest, [sck2, addr]])
|
||||
|
||||
with self.mutex:
|
||||
proc.clients[addr] = 50
|
||||
proc.workload += 50
|
||||
|
||||
else:
|
||||
raise Exception("what is " + str(dest))
|
||||
|
||||
def debug_load_balancer(self):
|
||||
fmt = "\033[1m{}\033[0;36m{:4}\033[0m "
|
||||
if not VT100:
|
||||
fmt = "({}{:4})"
|
||||
|
||||
last = ""
|
||||
while self.procs:
|
||||
msg = ""
|
||||
for proc in self.procs:
|
||||
msg += fmt.format(len(proc.clients), proc.workload)
|
||||
|
||||
if msg != last:
|
||||
last = msg
|
||||
with self.hub.log_mutex:
|
||||
print(msg)
|
||||
|
||||
time.sleep(0.1)
|
||||
|
||||
@@ -1,62 +1,46 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
from copyparty.authsrv import AuthSrv
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
import signal
|
||||
import threading
|
||||
|
||||
import queue
|
||||
|
||||
from .authsrv import AuthSrv
|
||||
from .broker_util import BrokerCli, ExceptionalQueue
|
||||
from .__init__ import PY2, WINDOWS
|
||||
from .broker_util import ExceptionalQueue
|
||||
from .httpsrv import HttpSrv
|
||||
from .util import FAKE_MP, HMaccas
|
||||
from .util import FAKE_MP
|
||||
|
||||
try:
|
||||
from types import FrameType
|
||||
|
||||
from typing import Any, Optional, Union
|
||||
except:
|
||||
pass
|
||||
if PY2 and not WINDOWS:
|
||||
import pickle # nosec
|
||||
|
||||
|
||||
class MpWorker(BrokerCli):
|
||||
class MpWorker(object):
|
||||
"""one single mp instance"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
q_pend: queue.Queue[tuple[int, str, list[Any]]],
|
||||
q_yield: queue.Queue[tuple[int, str, list[Any]]],
|
||||
args: argparse.Namespace,
|
||||
n: int,
|
||||
) -> None:
|
||||
super(MpWorker, self).__init__()
|
||||
|
||||
def __init__(self, q_pend, q_yield, args, n):
|
||||
self.q_pend = q_pend
|
||||
self.q_yield = q_yield
|
||||
self.args = args
|
||||
self.n = n
|
||||
|
||||
self.log = self._log_disabled if args.q and not args.lo else self._log_enabled
|
||||
|
||||
self.retpend: dict[int, Any] = {}
|
||||
self.retpend = {}
|
||||
self.retpend_mutex = threading.Lock()
|
||||
self.mutex = threading.Lock()
|
||||
self.workload_thr_alive = False
|
||||
|
||||
# we inherited signal_handler from parent,
|
||||
# replace it with something harmless
|
||||
if not FAKE_MP:
|
||||
for sig in [signal.SIGINT, signal.SIGTERM, signal.SIGUSR1]:
|
||||
signal.signal(sig, self.signal_handler)
|
||||
signal.signal(signal.SIGINT, self.signal_handler)
|
||||
|
||||
# starting to look like a good idea
|
||||
self.asrv = AuthSrv(args, None, False)
|
||||
|
||||
# instantiate all services here (TODO: inheritance?)
|
||||
self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
|
||||
self.httpsrv = HttpSrv(self, n)
|
||||
self.httpsrv = HttpSrv(self, True)
|
||||
self.httpsrv.disconnect_func = self.httpdrop
|
||||
|
||||
# on winxp and some other platforms,
|
||||
# use thr.join() to block all signals
|
||||
@@ -65,20 +49,20 @@ class MpWorker(BrokerCli):
|
||||
thr.start()
|
||||
thr.join()
|
||||
|
||||
def signal_handler(self, sig: Optional[int], frame: Optional[FrameType]) -> None:
|
||||
def signal_handler(self, signal, frame):
|
||||
# print('k')
|
||||
pass
|
||||
|
||||
def _log_enabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
|
||||
self.q_yield.put((0, "log", [src, msg, c]))
|
||||
def log(self, src, msg, c=0):
|
||||
self.q_yield.put([0, "log", [src, msg, c]])
|
||||
|
||||
def _log_disabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
|
||||
pass
|
||||
|
||||
def logw(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
def logw(self, msg, c=0):
|
||||
self.log("mp{}".format(self.n), msg, c)
|
||||
|
||||
def main(self) -> None:
|
||||
def httpdrop(self, addr):
|
||||
self.q_yield.put([0, "httpdrop", [addr]])
|
||||
|
||||
def main(self):
|
||||
while True:
|
||||
retq_id, dest, args = self.q_pend.get()
|
||||
|
||||
@@ -89,13 +73,24 @@ class MpWorker(BrokerCli):
|
||||
sys.exit(0)
|
||||
return
|
||||
|
||||
elif dest == "reload":
|
||||
self.logw("mpw.asrv reloading")
|
||||
self.asrv.reload()
|
||||
self.logw("mpw.asrv reloaded")
|
||||
elif dest == "httpconn":
|
||||
sck, addr = args
|
||||
if PY2:
|
||||
sck = pickle.loads(sck) # nosec
|
||||
|
||||
elif dest == "listen":
|
||||
self.httpsrv.listen(args[0], args[1])
|
||||
if self.args.log_conn:
|
||||
self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
|
||||
|
||||
self.httpsrv.accept(sck, addr)
|
||||
|
||||
with self.mutex:
|
||||
if not self.workload_thr_alive:
|
||||
self.workload_thr_alive = True
|
||||
thr = threading.Thread(
|
||||
target=self.thr_workload, name="mpw-workload"
|
||||
)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
elif dest == "retq":
|
||||
# response from previous ipc call
|
||||
@@ -107,14 +102,28 @@ class MpWorker(BrokerCli):
|
||||
else:
|
||||
raise Exception("what is " + str(dest))
|
||||
|
||||
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
|
||||
def put(self, want_retval, dest, *args):
|
||||
if want_retval:
|
||||
retq = ExceptionalQueue(1)
|
||||
retq_id = id(retq)
|
||||
with self.retpend_mutex:
|
||||
self.retpend[retq_id] = retq
|
||||
else:
|
||||
retq = None
|
||||
retq_id = 0
|
||||
|
||||
self.q_yield.put((retq_id, dest, list(args)))
|
||||
self.q_yield.put([retq_id, dest, args])
|
||||
return retq
|
||||
|
||||
def say(self, dest: str, *args: Any) -> None:
|
||||
self.q_yield.put((0, dest, list(args)))
|
||||
def thr_workload(self):
|
||||
"""announce workloads to MpSrv (the mp controller / loadbalancer)"""
|
||||
# avoid locking in extract_filedata by tracking difference here
|
||||
while True:
|
||||
time.sleep(0.2)
|
||||
with self.mutex:
|
||||
if self.httpsrv.num_clients() == 0:
|
||||
# no clients rn, termiante thread
|
||||
self.workload_thr_alive = False
|
||||
return
|
||||
|
||||
self.q_yield.put([0, "workload", [self.httpsrv.workload]])
|
||||
|
||||
@@ -1,71 +1,56 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import threading
|
||||
|
||||
from .__init__ import TYPE_CHECKING
|
||||
from .broker_util import BrokerCli, ExceptionalQueue, try_exec
|
||||
from .authsrv import AuthSrv
|
||||
from .httpsrv import HttpSrv
|
||||
from .util import HMaccas
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
|
||||
try:
|
||||
from typing import Any
|
||||
except:
|
||||
pass
|
||||
from .broker_util import ExceptionalQueue, try_exec
|
||||
|
||||
|
||||
class BrokerThr(BrokerCli):
|
||||
class BrokerThr(object):
|
||||
"""external api; behaves like BrokerMP but using plain threads"""
|
||||
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
super(BrokerThr, self).__init__()
|
||||
|
||||
def __init__(self, hub):
|
||||
self.hub = hub
|
||||
self.log = hub.log
|
||||
self.args = hub.args
|
||||
self.asrv = hub.asrv
|
||||
|
||||
self.mutex = threading.Lock()
|
||||
self.num_workers = 1
|
||||
|
||||
# instantiate all services here (TODO: inheritance?)
|
||||
self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
|
||||
self.httpsrv = HttpSrv(self, None)
|
||||
self.reload = self.noop
|
||||
self.httpsrv = HttpSrv(self)
|
||||
self.httpsrv.disconnect_func = self.httpdrop
|
||||
|
||||
def shutdown(self) -> None:
|
||||
def shutdown(self):
|
||||
# self.log("broker", "shutting down")
|
||||
self.httpsrv.shutdown()
|
||||
|
||||
def noop(self) -> None:
|
||||
pass
|
||||
|
||||
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
|
||||
def put(self, want_retval, dest, *args):
|
||||
if dest == "httpconn":
|
||||
sck, addr = args
|
||||
if self.args.log_conn:
|
||||
self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
|
||||
|
||||
self.httpsrv.accept(sck, addr)
|
||||
|
||||
else:
|
||||
# new ipc invoking managed service in hub
|
||||
obj = self.hub
|
||||
for node in dest.split("."):
|
||||
obj = getattr(obj, node)
|
||||
|
||||
rv = try_exec(True, obj, *args)
|
||||
# TODO will deadlock if dest performs another ipc
|
||||
rv = try_exec(want_retval, obj, *args)
|
||||
if not want_retval:
|
||||
return
|
||||
|
||||
# pretend we're broker_mp
|
||||
retq = ExceptionalQueue(1)
|
||||
retq.put(rv)
|
||||
return retq
|
||||
|
||||
def say(self, dest: str, *args: Any) -> None:
|
||||
if dest == "listen":
|
||||
self.httpsrv.listen(args[0], 1)
|
||||
return
|
||||
|
||||
# new ipc invoking managed service in hub
|
||||
obj = self.hub
|
||||
for node in dest.split("."):
|
||||
obj = getattr(obj, node)
|
||||
|
||||
try_exec(False, obj, *args)
|
||||
def httpdrop(self, addr):
|
||||
self.hub.tcpsrv.num_clients.add(-1)
|
||||
|
||||
@@ -1,30 +1,17 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
|
||||
import traceback
|
||||
|
||||
from queue import Queue
|
||||
|
||||
from .__init__ import TYPE_CHECKING
|
||||
from .authsrv import AuthSrv
|
||||
from .util import HMaccas, Pebkac
|
||||
|
||||
try:
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from .util import RootLogger
|
||||
except:
|
||||
pass
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .httpsrv import HttpSrv
|
||||
from .util import Pebkac, Queue
|
||||
|
||||
|
||||
class ExceptionalQueue(Queue, object):
|
||||
def get(self, block: bool = True, timeout: Optional[float] = None) -> Any:
|
||||
def get(self, block=True, timeout=None):
|
||||
rv = super(ExceptionalQueue, self).get(block, timeout)
|
||||
|
||||
# TODO: how expensive is this?
|
||||
if isinstance(rv, list):
|
||||
if rv[0] == "exception":
|
||||
if rv[1] == "pebkac":
|
||||
@@ -35,27 +22,7 @@ class ExceptionalQueue(Queue, object):
|
||||
return rv
|
||||
|
||||
|
||||
class BrokerCli(object):
|
||||
"""
|
||||
helps mypy understand httpsrv.broker but still fails a few levels deeper,
|
||||
for example resolving httpconn.* in httpcli -- see lines tagged #mypy404
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.log: "RootLogger" = None
|
||||
self.args: argparse.Namespace = None
|
||||
self.asrv: AuthSrv = None
|
||||
self.httpsrv: "HttpSrv" = None
|
||||
self.iphash: HMaccas = None
|
||||
|
||||
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
|
||||
return ExceptionalQueue(1)
|
||||
|
||||
def say(self, dest: str, *args: Any) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def try_exec(want_retval: Union[bool, int], func: Any, *args: list[Any]) -> Any:
|
||||
def try_exec(want_retval, func, *args):
|
||||
try:
|
||||
return func(*args)
|
||||
|
||||
|
||||
@@ -1,154 +0,0 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
from .__init__ import ANYWIN, MACOS
|
||||
from .authsrv import AXS, VFS
|
||||
from .bos import bos
|
||||
from .util import chkcmd, min_ex
|
||||
|
||||
try:
|
||||
from typing import Optional, Union
|
||||
|
||||
from .util import RootLogger
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class Fstab(object):
|
||||
def __init__(self, log: "RootLogger"):
|
||||
self.log_func = log
|
||||
|
||||
self.trusted = False
|
||||
self.tab: Optional[VFS] = None
|
||||
self.cache: dict[str, str] = {}
|
||||
self.age = 0.0
|
||||
|
||||
def log(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
self.log_func("fstab", msg + "\033[K", c)
|
||||
|
||||
def get(self, path: str) -> str:
|
||||
if len(self.cache) > 9000:
|
||||
self.age = time.time()
|
||||
self.tab = None
|
||||
self.cache = {}
|
||||
|
||||
fs = "ext4"
|
||||
msg = "failed to determine filesystem at [{}]; assuming {}\n{}"
|
||||
|
||||
if ANYWIN:
|
||||
fs = "vfat"
|
||||
try:
|
||||
path = self._winpath(path)
|
||||
except:
|
||||
self.log(msg.format(path, fs, min_ex()), 3)
|
||||
return fs
|
||||
|
||||
path = path.lstrip("/")
|
||||
try:
|
||||
return self.cache[path]
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
fs = self.get_w32(path) if ANYWIN else self.get_unix(path)
|
||||
except:
|
||||
self.log(msg.format(path, fs, min_ex()), 3)
|
||||
|
||||
fs = fs.lower()
|
||||
self.cache[path] = fs
|
||||
self.log("found {} at {}".format(fs, path))
|
||||
return fs
|
||||
|
||||
def _winpath(self, path: str) -> str:
|
||||
# try to combine volume-label + st_dev (vsn)
|
||||
path = path.replace("/", "\\")
|
||||
vid = path.split(":", 1)[0].strip("\\").split("\\", 1)[0]
|
||||
try:
|
||||
return "{}*{}".format(vid, bos.stat(path).st_dev)
|
||||
except:
|
||||
return vid
|
||||
|
||||
def build_fallback(self) -> None:
|
||||
self.tab = VFS(self.log_func, "idk", "/", AXS(), {})
|
||||
self.trusted = False
|
||||
|
||||
def build_tab(self) -> None:
|
||||
self.log("building tab")
|
||||
|
||||
sptn = r"^.*? on (.*) type ([^ ]+) \(.*"
|
||||
if MACOS:
|
||||
sptn = r"^.*? on (.*) \(([^ ]+), .*"
|
||||
|
||||
ptn = re.compile(sptn)
|
||||
so, _ = chkcmd(["mount"])
|
||||
tab1: list[tuple[str, str]] = []
|
||||
for ln in so.split("\n"):
|
||||
m = ptn.match(ln)
|
||||
if not m:
|
||||
continue
|
||||
|
||||
zs1, zs2 = m.groups()
|
||||
tab1.append((str(zs1), str(zs2)))
|
||||
|
||||
tab1.sort(key=lambda x: (len(x[0]), x[0]))
|
||||
path1, fs1 = tab1[0]
|
||||
tab = VFS(self.log_func, fs1, path1, AXS(), {})
|
||||
for path, fs in tab1[1:]:
|
||||
tab.add(fs, path.lstrip("/"))
|
||||
|
||||
self.tab = tab
|
||||
|
||||
def relabel(self, path: str, nval: str) -> None:
|
||||
assert self.tab
|
||||
self.cache = {}
|
||||
if ANYWIN:
|
||||
path = self._winpath(path)
|
||||
|
||||
path = path.lstrip("/")
|
||||
ptn = re.compile(r"^[^\\/]*")
|
||||
vn, rem = self.tab._find(path)
|
||||
if not self.trusted:
|
||||
# no mtab access; have to build as we go
|
||||
if "/" in rem:
|
||||
self.tab.add("idk", os.path.join(vn.vpath, rem.split("/")[0]))
|
||||
if rem:
|
||||
self.tab.add(nval, path)
|
||||
else:
|
||||
vn.realpath = nval
|
||||
|
||||
return
|
||||
|
||||
visit = [vn]
|
||||
while visit:
|
||||
vn = visit.pop()
|
||||
vn.realpath = ptn.sub(nval, vn.realpath)
|
||||
visit.extend(list(vn.nodes.values()))
|
||||
|
||||
def get_unix(self, path: str) -> str:
|
||||
if not self.tab:
|
||||
try:
|
||||
self.build_tab()
|
||||
self.trusted = True
|
||||
except:
|
||||
# prisonparty or other restrictive environment
|
||||
self.log("failed to build tab:\n{}".format(min_ex()), 3)
|
||||
self.build_fallback()
|
||||
|
||||
assert self.tab
|
||||
ret = self.tab._find(path)[0]
|
||||
if self.trusted or path == ret.vpath:
|
||||
return ret.realpath.split("/")[0]
|
||||
else:
|
||||
return "idk"
|
||||
|
||||
def get_w32(self, path: str) -> str:
|
||||
if not self.tab:
|
||||
self.build_fallback()
|
||||
|
||||
assert self.tab
|
||||
ret = self.tab._find(path)[0]
|
||||
return ret.realpath
|
||||
@@ -1,403 +0,0 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from pyftpdlib.authorizers import AuthenticationFailed, DummyAuthorizer
|
||||
from pyftpdlib.filesystems import AbstractedFS, FilesystemError
|
||||
from pyftpdlib.handlers import FTPHandler
|
||||
from pyftpdlib.log import config_logging
|
||||
from pyftpdlib.servers import FTPServer
|
||||
|
||||
from .__init__ import PY2, TYPE_CHECKING, E
|
||||
from .bos import bos
|
||||
from .util import Pebkac, exclude_dotfiles, fsenc
|
||||
|
||||
try:
|
||||
from pyftpdlib.ioloop import IOLoop
|
||||
except ImportError:
|
||||
p = os.path.join(E.mod, "vend")
|
||||
print("loading asynchat from " + p)
|
||||
sys.path.append(p)
|
||||
from pyftpdlib.ioloop import IOLoop
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
|
||||
try:
|
||||
import typing
|
||||
from typing import Any, Optional
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class FtpAuth(DummyAuthorizer):
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
super(FtpAuth, self).__init__()
|
||||
self.hub = hub
|
||||
|
||||
def validate_authentication(
|
||||
self, username: str, password: str, handler: Any
|
||||
) -> None:
|
||||
asrv = self.hub.asrv
|
||||
if username == "anonymous":
|
||||
password = ""
|
||||
|
||||
uname = "*"
|
||||
if password:
|
||||
uname = asrv.iacct.get(password, "")
|
||||
|
||||
handler.username = uname
|
||||
|
||||
if (password and not uname) or not (
|
||||
asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)
|
||||
):
|
||||
raise AuthenticationFailed("Authentication failed.")
|
||||
|
||||
def get_home_dir(self, username: str) -> str:
|
||||
return "/"
|
||||
|
||||
def has_user(self, username: str) -> bool:
|
||||
asrv = self.hub.asrv
|
||||
return username in asrv.acct
|
||||
|
||||
def has_perm(self, username: str, perm: int, path: Optional[str] = None) -> bool:
|
||||
return True # handled at filesystem layer
|
||||
|
||||
def get_perms(self, username: str) -> str:
|
||||
return "elradfmwMT"
|
||||
|
||||
def get_msg_login(self, username: str) -> str:
|
||||
return "sup {}".format(username)
|
||||
|
||||
def get_msg_quit(self, username: str) -> str:
|
||||
return "cya"
|
||||
|
||||
|
||||
class FtpFs(AbstractedFS):
|
||||
def __init__(
|
||||
self, root: str, cmd_channel: Any
|
||||
) -> None: # pylint: disable=super-init-not-called
|
||||
self.h = self.cmd_channel = cmd_channel # type: FTPHandler
|
||||
self.hub: "SvcHub" = cmd_channel.hub
|
||||
self.args = cmd_channel.args
|
||||
|
||||
self.uname = self.hub.asrv.iacct.get(cmd_channel.password, "*")
|
||||
|
||||
self.cwd = "/" # pyftpdlib convention of leading slash
|
||||
self.root = "/var/lib/empty"
|
||||
|
||||
self.listdirinfo = self.listdir
|
||||
self.chdir(".")
|
||||
|
||||
def v2a(
|
||||
self,
|
||||
vpath: str,
|
||||
r: bool = False,
|
||||
w: bool = False,
|
||||
m: bool = False,
|
||||
d: bool = False,
|
||||
) -> str:
|
||||
try:
|
||||
vpath = vpath.replace("\\", "/").lstrip("/")
|
||||
vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, r, w, m, d)
|
||||
if not vfs.realpath:
|
||||
raise FilesystemError("no filesystem mounted at this path")
|
||||
|
||||
return os.path.join(vfs.realpath, rem)
|
||||
except Pebkac as ex:
|
||||
raise FilesystemError(str(ex))
|
||||
|
||||
def rv2a(
|
||||
self,
|
||||
vpath: str,
|
||||
r: bool = False,
|
||||
w: bool = False,
|
||||
m: bool = False,
|
||||
d: bool = False,
|
||||
) -> str:
|
||||
return self.v2a(os.path.join(self.cwd, vpath), r, w, m, d)
|
||||
|
||||
def ftp2fs(self, ftppath: str) -> str:
|
||||
# return self.v2a(ftppath)
|
||||
return ftppath # self.cwd must be vpath
|
||||
|
||||
def fs2ftp(self, fspath: str) -> str:
|
||||
# raise NotImplementedError()
|
||||
return fspath
|
||||
|
||||
def validpath(self, path: str) -> bool:
|
||||
if "/.hist/" in path:
|
||||
if "/up2k." in path or path.endswith("/dir.txt"):
|
||||
raise FilesystemError("access to this file is forbidden")
|
||||
|
||||
return True
|
||||
|
||||
def open(self, filename: str, mode: str) -> typing.IO[Any]:
|
||||
r = "r" in mode
|
||||
w = "w" in mode or "a" in mode or "+" in mode
|
||||
|
||||
ap = self.rv2a(filename, r, w)
|
||||
if w and bos.path.exists(ap):
|
||||
raise FilesystemError("cannot open existing file for writing")
|
||||
|
||||
self.validpath(ap)
|
||||
return open(fsenc(ap), mode)
|
||||
|
||||
def chdir(self, path: str) -> None:
|
||||
self.cwd = join(self.cwd, path)
|
||||
x = self.hub.asrv.vfs.can_access(self.cwd.lstrip("/"), self.h.username)
|
||||
self.can_read, self.can_write, self.can_move, self.can_delete, self.can_get = x
|
||||
|
||||
def mkdir(self, path: str) -> None:
|
||||
ap = self.rv2a(path, w=True)
|
||||
bos.mkdir(ap)
|
||||
|
||||
def listdir(self, path: str) -> list[str]:
|
||||
vpath = join(self.cwd, path).lstrip("/")
|
||||
try:
|
||||
vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, True, False)
|
||||
|
||||
fsroot, vfs_ls1, vfs_virt = vfs.ls(
|
||||
rem, self.uname, not self.args.no_scandir, [[True], [False, True]]
|
||||
)
|
||||
vfs_ls = [x[0] for x in vfs_ls1]
|
||||
vfs_ls.extend(vfs_virt.keys())
|
||||
|
||||
if not self.args.ed:
|
||||
vfs_ls = exclude_dotfiles(vfs_ls)
|
||||
|
||||
vfs_ls.sort()
|
||||
return vfs_ls
|
||||
except:
|
||||
if vpath:
|
||||
# display write-only folders as empty
|
||||
return []
|
||||
|
||||
# return list of volumes
|
||||
r = {x.split("/")[0]: 1 for x in self.hub.asrv.vfs.all_vols.keys()}
|
||||
return list(sorted(list(r.keys())))
|
||||
|
||||
def rmdir(self, path: str) -> None:
|
||||
ap = self.rv2a(path, d=True)
|
||||
bos.rmdir(ap)
|
||||
|
||||
def remove(self, path: str) -> None:
|
||||
if self.args.no_del:
|
||||
raise FilesystemError("the delete feature is disabled in server config")
|
||||
|
||||
vp = join(self.cwd, path).lstrip("/")
|
||||
try:
|
||||
self.hub.up2k.handle_rm(self.uname, self.h.remote_ip, [vp])
|
||||
except Exception as ex:
|
||||
raise FilesystemError(str(ex))
|
||||
|
||||
def rename(self, src: str, dst: str) -> None:
|
||||
if not self.can_move:
|
||||
raise FilesystemError("not allowed for user " + self.h.username)
|
||||
|
||||
if self.args.no_mv:
|
||||
t = "the rename/move feature is disabled in server config"
|
||||
raise FilesystemError(t)
|
||||
|
||||
svp = join(self.cwd, src).lstrip("/")
|
||||
dvp = join(self.cwd, dst).lstrip("/")
|
||||
try:
|
||||
self.hub.up2k.handle_mv(self.uname, svp, dvp)
|
||||
except Exception as ex:
|
||||
raise FilesystemError(str(ex))
|
||||
|
||||
def chmod(self, path: str, mode: str) -> None:
|
||||
pass
|
||||
|
||||
def stat(self, path: str) -> os.stat_result:
|
||||
try:
|
||||
ap = self.rv2a(path, r=True)
|
||||
return bos.stat(ap)
|
||||
except:
|
||||
ap = self.rv2a(path)
|
||||
st = bos.stat(ap)
|
||||
if not stat.S_ISDIR(st.st_mode):
|
||||
raise
|
||||
|
||||
return st
|
||||
|
||||
def utime(self, path: str, timeval: float) -> None:
|
||||
ap = self.rv2a(path, w=True)
|
||||
return bos.utime(ap, (timeval, timeval))
|
||||
|
||||
def lstat(self, path: str) -> os.stat_result:
|
||||
ap = self.rv2a(path)
|
||||
return bos.lstat(ap)
|
||||
|
||||
def isfile(self, path: str) -> bool:
|
||||
st = self.stat(path)
|
||||
return stat.S_ISREG(st.st_mode)
|
||||
|
||||
def islink(self, path: str) -> bool:
|
||||
ap = self.rv2a(path)
|
||||
return bos.path.islink(ap)
|
||||
|
||||
def isdir(self, path: str) -> bool:
|
||||
try:
|
||||
st = self.stat(path)
|
||||
return stat.S_ISDIR(st.st_mode)
|
||||
except:
|
||||
return True
|
||||
|
||||
def getsize(self, path: str) -> int:
|
||||
ap = self.rv2a(path)
|
||||
return bos.path.getsize(ap)
|
||||
|
||||
def getmtime(self, path: str) -> float:
|
||||
ap = self.rv2a(path)
|
||||
return bos.path.getmtime(ap)
|
||||
|
||||
def realpath(self, path: str) -> str:
|
||||
return path
|
||||
|
||||
def lexists(self, path: str) -> bool:
|
||||
ap = self.rv2a(path)
|
||||
return bos.path.lexists(ap)
|
||||
|
||||
def get_user_by_uid(self, uid: int) -> str:
|
||||
return "root"
|
||||
|
||||
def get_group_by_uid(self, gid: int) -> str:
|
||||
return "root"
|
||||
|
||||
|
||||
class FtpHandler(FTPHandler):
|
||||
abstracted_fs = FtpFs
|
||||
hub: "SvcHub" = None
|
||||
args: argparse.Namespace = None
|
||||
|
||||
def __init__(self, conn: Any, server: Any, ioloop: Any = None) -> None:
|
||||
self.hub: "SvcHub" = FtpHandler.hub
|
||||
self.args: argparse.Namespace = FtpHandler.args
|
||||
|
||||
if PY2:
|
||||
FTPHandler.__init__(self, conn, server, ioloop)
|
||||
else:
|
||||
super(FtpHandler, self).__init__(conn, server, ioloop)
|
||||
|
||||
# abspath->vpath mapping to resolve log_transfer paths
|
||||
self.vfs_map: dict[str, str] = {}
|
||||
|
||||
def ftp_STOR(self, file: str, mode: str = "w") -> Any:
|
||||
# Optional[str]
|
||||
vp = join(self.fs.cwd, file).lstrip("/")
|
||||
ap = self.fs.v2a(vp)
|
||||
self.vfs_map[ap] = vp
|
||||
# print("ftp_STOR: {} {} => {}".format(vp, mode, ap))
|
||||
ret = FTPHandler.ftp_STOR(self, file, mode)
|
||||
# print("ftp_STOR: {} {} OK".format(vp, mode))
|
||||
return ret
|
||||
|
||||
def log_transfer(
|
||||
self,
|
||||
cmd: str,
|
||||
filename: bytes,
|
||||
receive: bool,
|
||||
completed: bool,
|
||||
elapsed: float,
|
||||
bytes: int,
|
||||
) -> Any:
|
||||
# None
|
||||
ap = filename.decode("utf-8", "replace")
|
||||
vp = self.vfs_map.pop(ap, None)
|
||||
# print("xfer_end: {} => {}".format(ap, vp))
|
||||
if vp:
|
||||
vp, fn = os.path.split(vp)
|
||||
vfs, rem = self.hub.asrv.vfs.get(vp, self.username, False, True)
|
||||
vfs, rem = vfs.get_dbv(rem)
|
||||
self.hub.up2k.hash_file(
|
||||
vfs.realpath,
|
||||
vfs.flags,
|
||||
rem,
|
||||
fn,
|
||||
self.remote_ip,
|
||||
time.time(),
|
||||
)
|
||||
|
||||
return FTPHandler.log_transfer(
|
||||
self, cmd, filename, receive, completed, elapsed, bytes
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
from pyftpdlib.handlers import TLS_FTPHandler
|
||||
|
||||
class SftpHandler(FtpHandler, TLS_FTPHandler):
|
||||
pass
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class Ftpd(object):
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
self.hub = hub
|
||||
self.args = hub.args
|
||||
|
||||
hs = []
|
||||
if self.args.ftp:
|
||||
hs.append([FtpHandler, self.args.ftp])
|
||||
if self.args.ftps:
|
||||
try:
|
||||
h1 = SftpHandler
|
||||
except:
|
||||
t = "\nftps requires pyopenssl;\nplease run the following:\n\n {} -m pip install --user pyopenssl\n"
|
||||
print(t.format(sys.executable))
|
||||
sys.exit(1)
|
||||
|
||||
h1.certfile = os.path.join(self.args.E.cfg, "cert.pem")
|
||||
h1.tls_control_required = True
|
||||
h1.tls_data_required = True
|
||||
|
||||
hs.append([h1, self.args.ftps])
|
||||
|
||||
for h_lp in hs:
|
||||
h2, lp = h_lp
|
||||
h2.hub = hub
|
||||
h2.args = hub.args
|
||||
h2.authorizer = FtpAuth(hub)
|
||||
|
||||
if self.args.ftp_pr:
|
||||
p1, p2 = [int(x) for x in self.args.ftp_pr.split("-")]
|
||||
if self.args.ftp and self.args.ftps:
|
||||
# divide port range in half
|
||||
d = int((p2 - p1) / 2)
|
||||
if lp == self.args.ftp:
|
||||
p2 = p1 + d
|
||||
else:
|
||||
p1 += d + 1
|
||||
|
||||
h2.passive_ports = list(range(p1, p2 + 1))
|
||||
|
||||
if self.args.ftp_nat:
|
||||
h2.masquerade_address = self.args.ftp_nat
|
||||
|
||||
if self.args.ftp_dbg:
|
||||
config_logging(level=logging.DEBUG)
|
||||
|
||||
ioloop = IOLoop()
|
||||
for ip in self.args.i:
|
||||
for h, lp in hs:
|
||||
FTPServer((ip, int(lp)), h, ioloop)
|
||||
|
||||
thr = threading.Thread(target=ioloop.loop, name="ftp")
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
|
||||
def join(p1: str, p2: str) -> str:
|
||||
w = os.path.join(p1, p2.replace("\\", "/"))
|
||||
return os.path.normpath(w).replace("\\", "/")
|
||||
1980
copyparty/httpcli.py
1980
copyparty/httpcli.py
File diff suppressed because it is too large
Load Diff
@@ -1,37 +1,24 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import argparse # typechk
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import threading # typechk
|
||||
import os
|
||||
import time
|
||||
import socket
|
||||
|
||||
try:
|
||||
HAVE_SSL = True
|
||||
try:
|
||||
import ssl
|
||||
except:
|
||||
HAVE_SSL = False
|
||||
|
||||
from . import util as Util
|
||||
from .__init__ import TYPE_CHECKING, EnvParams
|
||||
from .authsrv import AuthSrv # typechk
|
||||
from .__init__ import E
|
||||
from .util import Unrecv
|
||||
from .httpcli import HttpCli
|
||||
from .ico import Ico
|
||||
from .mtag import HAVE_FFMPEG
|
||||
from .th_cli import ThumbCli
|
||||
from .th_srv import HAVE_PIL, HAVE_VIPS
|
||||
from .u2idx import U2idx
|
||||
from .util import HMaccas, shut_socket
|
||||
|
||||
try:
|
||||
from typing import Optional, Pattern, Union
|
||||
except:
|
||||
pass
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .httpsrv import HttpSrv
|
||||
from .th_cli import ThumbCli
|
||||
from .th_srv import HAVE_PIL
|
||||
from .ico import Ico
|
||||
|
||||
|
||||
class HttpConn(object):
|
||||
@@ -40,46 +27,39 @@ class HttpConn(object):
|
||||
creates an HttpCli for each request (Connection: Keep-Alive)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, sck: socket.socket, addr: tuple[str, int], hsrv: "HttpSrv"
|
||||
) -> None:
|
||||
def __init__(self, sck, addr, hsrv):
|
||||
self.s = sck
|
||||
self.sr: Optional[Util._Unrecv] = None
|
||||
self.addr = addr
|
||||
self.hsrv = hsrv
|
||||
|
||||
self.mutex: threading.Lock = hsrv.mutex # mypy404
|
||||
self.args: argparse.Namespace = hsrv.args # mypy404
|
||||
self.E: EnvParams = self.args.E
|
||||
self.asrv: AuthSrv = hsrv.asrv # mypy404
|
||||
self.args = hsrv.args
|
||||
self.asrv = hsrv.asrv
|
||||
self.is_mp = hsrv.is_mp
|
||||
self.cert_path = hsrv.cert_path
|
||||
self.u2fh: Util.FHC = hsrv.u2fh # mypy404
|
||||
self.iphash: HMaccas = hsrv.broker.iphash
|
||||
|
||||
enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb
|
||||
self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None # mypy404
|
||||
self.ico: Ico = Ico(self.args) # mypy404
|
||||
enth = HAVE_PIL and not self.args.no_thumb
|
||||
self.thumbcli = ThumbCli(hsrv.broker) if enth else None
|
||||
self.ico = Ico(self.args)
|
||||
|
||||
self.t0: float = time.time() # mypy404
|
||||
self.t0 = time.time()
|
||||
self.stopping = False
|
||||
self.nreq: int = 0 # mypy404
|
||||
self.nbyte: int = 0 # mypy404
|
||||
self.u2idx: Optional[U2idx] = None
|
||||
self.log_func: "Util.RootLogger" = hsrv.log # mypy404
|
||||
self.log_src: str = "httpconn" # mypy404
|
||||
self.lf_url: Optional[Pattern[str]] = (
|
||||
re.compile(self.args.lf_url) if self.args.lf_url else None
|
||||
) # mypy404
|
||||
self.nreq = 0
|
||||
self.nbyte = 0
|
||||
self.workload = 0
|
||||
self.u2idx = None
|
||||
self.log_func = hsrv.log
|
||||
self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
|
||||
self.set_rproxy()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
def shutdown(self):
|
||||
self.stopping = True
|
||||
try:
|
||||
shut_socket(self.log, self.s, 1)
|
||||
self.s.shutdown(socket.SHUT_RDWR)
|
||||
self.s.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
def set_rproxy(self, ip: Optional[str] = None) -> str:
|
||||
def set_rproxy(self, ip=None):
|
||||
if ip is None:
|
||||
color = 36
|
||||
ip = self.addr[0]
|
||||
@@ -92,37 +72,35 @@ class HttpConn(object):
|
||||
self.log_src = "{} \033[{}m{}".format(ip, color, self.addr[1]).ljust(26)
|
||||
return self.log_src
|
||||
|
||||
def respath(self, res_name: str) -> str:
|
||||
return os.path.join(self.E.mod, "web", res_name)
|
||||
def respath(self, res_name):
|
||||
return os.path.join(E.mod, "web", res_name)
|
||||
|
||||
def log(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
def log(self, msg, c=0):
|
||||
self.log_func(self.log_src, msg, c)
|
||||
|
||||
def get_u2idx(self) -> U2idx:
|
||||
# one u2idx per tcp connection;
|
||||
# sqlite3 fully parallelizes under python threads
|
||||
def get_u2idx(self):
|
||||
if not self.u2idx:
|
||||
self.u2idx = U2idx(self)
|
||||
|
||||
return self.u2idx
|
||||
|
||||
def _detect_https(self) -> bool:
|
||||
def _detect_https(self):
|
||||
method = None
|
||||
if self.cert_path:
|
||||
try:
|
||||
method = self.s.recv(4, socket.MSG_PEEK)
|
||||
except socket.timeout:
|
||||
return False
|
||||
return
|
||||
except AttributeError:
|
||||
# jython does not support msg_peek; forget about https
|
||||
method = self.s.recv(4)
|
||||
self.sr = Util.Unrecv(self.s, self.log)
|
||||
self.sr = Unrecv(self.s)
|
||||
self.sr.buf = method
|
||||
|
||||
# jython used to do this, they stopped since it's broken
|
||||
# but reimplementing sendall is out of scope for now
|
||||
if not getattr(self.s, "sendall", None):
|
||||
self.s.sendall = self.s.send # type: ignore
|
||||
self.s.sendall = self.s.send
|
||||
|
||||
if len(method) != 4:
|
||||
err = "need at least 4 bytes in the first packet; got {}".format(
|
||||
@@ -132,18 +110,17 @@ class HttpConn(object):
|
||||
self.log(err)
|
||||
|
||||
self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
|
||||
return False
|
||||
return
|
||||
|
||||
return method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]
|
||||
|
||||
def run(self) -> None:
|
||||
def run(self):
|
||||
self.sr = None
|
||||
if self.args.https_only:
|
||||
is_https = True
|
||||
elif self.args.http_only or not HAVE_SSL:
|
||||
is_https = False
|
||||
else:
|
||||
# raise Exception("asdf")
|
||||
is_https = self._detect_https()
|
||||
|
||||
if is_https:
|
||||
@@ -172,15 +149,14 @@ class HttpConn(object):
|
||||
self.s = ctx.wrap_socket(self.s, server_side=True)
|
||||
msg = [
|
||||
"\033[1;3{:d}m{}".format(c, s)
|
||||
for c, s in zip([0, 5, 0], self.s.cipher()) # type: ignore
|
||||
for c, s in zip([0, 5, 0], self.s.cipher())
|
||||
]
|
||||
self.log(" ".join(msg) + "\033[0m")
|
||||
|
||||
if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
|
||||
ciphers = self.s.shared_ciphers()
|
||||
assert ciphers
|
||||
overlap = [str(y[::-1]) for y in ciphers]
|
||||
self.log("TLS cipher overlap:" + "\n".join(overlap))
|
||||
overlap = [y[::-1] for y in self.s.shared_ciphers()]
|
||||
lines = [str(x) for x in (["TLS cipher overlap:"] + overlap)]
|
||||
self.log("\n".join(lines))
|
||||
for k, v in [
|
||||
["compression", self.s.compression()],
|
||||
["ALPN proto", self.s.selected_alpn_protocol()],
|
||||
@@ -191,7 +167,11 @@ class HttpConn(object):
|
||||
except Exception as ex:
|
||||
em = str(ex)
|
||||
|
||||
if "ALERT_CERTIFICATE_UNKNOWN" in em:
|
||||
if "ALERT_BAD_CERTIFICATE" in em:
|
||||
# firefox-linux if there is no exception yet
|
||||
self.log("client rejected our certificate (nice)")
|
||||
|
||||
elif "ALERT_CERTIFICATE_UNKNOWN" in em:
|
||||
# android-chrome keeps doing this
|
||||
pass
|
||||
|
||||
@@ -201,9 +181,14 @@ class HttpConn(object):
|
||||
return
|
||||
|
||||
if not self.sr:
|
||||
self.sr = Util.Unrecv(self.s, self.log)
|
||||
self.sr = Unrecv(self.s)
|
||||
|
||||
while not self.stopping:
|
||||
if self.is_mp:
|
||||
self.workload += 50
|
||||
if self.workload >= 2 ** 31:
|
||||
self.workload = 100
|
||||
|
||||
self.nreq += 1
|
||||
cli = HttpCli(self)
|
||||
if not cli.run():
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import base64
|
||||
import math
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import queue
|
||||
import base64
|
||||
import struct
|
||||
import socket
|
||||
import threading
|
||||
|
||||
try:
|
||||
import jinja2
|
||||
@@ -28,27 +26,8 @@ except ImportError:
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
from .__init__ import MACOS, TYPE_CHECKING, EnvParams
|
||||
from .bos import bos
|
||||
from .__init__ import E, MACOS
|
||||
from .httpconn import HttpConn
|
||||
from .util import (
|
||||
FHC,
|
||||
Garda,
|
||||
Magician,
|
||||
min_ex,
|
||||
shut_socket,
|
||||
spack,
|
||||
start_log_thrs,
|
||||
start_stackmon,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .broker_util import BrokerCli
|
||||
|
||||
try:
|
||||
from typing import Any, Optional
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class HttpSrv(object):
|
||||
@@ -57,282 +36,89 @@ class HttpSrv(object):
|
||||
relying on MpSrv for performance (HttpSrv is just plain threads)
|
||||
"""
|
||||
|
||||
def __init__(self, broker: "BrokerCli", nid: Optional[int]) -> None:
|
||||
def __init__(self, broker, is_mp=False):
|
||||
self.broker = broker
|
||||
self.nid = nid
|
||||
self.is_mp = is_mp
|
||||
self.args = broker.args
|
||||
self.E: EnvParams = self.args.E
|
||||
self.log = broker.log
|
||||
self.asrv = broker.asrv
|
||||
|
||||
# redefine in case of multiprocessing
|
||||
socket.setdefaulttimeout(120)
|
||||
|
||||
nsuf = "-n{}-i{:x}".format(nid, os.getpid()) if nid else ""
|
||||
self.magician = Magician()
|
||||
self.bans: dict[str, int] = {}
|
||||
self.gpwd = Garda(self.args.ban_pw)
|
||||
self.g404 = Garda(self.args.ban_404)
|
||||
|
||||
self.name = "hsrv" + nsuf
|
||||
self.disconnect_func = None
|
||||
self.mutex = threading.Lock()
|
||||
self.stopping = False
|
||||
|
||||
self.tp_nthr = 0 # actual
|
||||
self.tp_ncli = 0 # fading
|
||||
self.tp_time = 0.0 # latest worker collect
|
||||
self.tp_q: Optional[queue.LifoQueue[Any]] = (
|
||||
None if self.args.no_htp else queue.LifoQueue()
|
||||
)
|
||||
self.t_periodic: Optional[threading.Thread] = None
|
||||
|
||||
self.u2fh = FHC()
|
||||
self.srvs: list[socket.socket] = []
|
||||
self.ncli = 0 # exact
|
||||
self.clients: set[HttpConn] = set() # laggy
|
||||
self.nclimax = 0
|
||||
self.cb_ts = 0.0
|
||||
self.cb_v = ""
|
||||
self.clients = {}
|
||||
self.workload = 0
|
||||
self.workload_thr_alive = False
|
||||
self.cb_ts = 0
|
||||
self.cb_v = 0
|
||||
|
||||
env = jinja2.Environment()
|
||||
env.loader = jinja2.FileSystemLoader(os.path.join(self.E.mod, "web"))
|
||||
env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
|
||||
self.j2 = {
|
||||
x: env.get_template(x + ".html")
|
||||
for x in ["splash", "browser", "browser2", "msg", "md", "mde", "cf"]
|
||||
for x in ["splash", "browser", "browser2", "msg", "md", "mde"]
|
||||
}
|
||||
zs = os.path.join(self.E.mod, "web", "deps", "prism.js.gz")
|
||||
self.prism = os.path.exists(zs)
|
||||
|
||||
cert_path = os.path.join(self.E.cfg, "cert.pem")
|
||||
if bos.path.exists(cert_path):
|
||||
cert_path = os.path.join(E.cfg, "cert.pem")
|
||||
if os.path.exists(cert_path):
|
||||
self.cert_path = cert_path
|
||||
else:
|
||||
self.cert_path = ""
|
||||
self.cert_path = None
|
||||
|
||||
if self.tp_q:
|
||||
self.start_threads(4)
|
||||
|
||||
if nid:
|
||||
if self.args.stackmon:
|
||||
start_stackmon(self.args.stackmon, nid)
|
||||
|
||||
if self.args.log_thrs:
|
||||
start_log_thrs(self.log, self.args.log_thrs, nid)
|
||||
|
||||
self.th_cfg: dict[str, Any] = {}
|
||||
t = threading.Thread(target=self.post_init, name="hsrv-init2")
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
def post_init(self) -> None:
|
||||
try:
|
||||
x = self.broker.ask("thumbsrv.getcfg")
|
||||
self.th_cfg = x.get()
|
||||
except:
|
||||
pass
|
||||
|
||||
def start_threads(self, n: int) -> None:
|
||||
self.tp_nthr += n
|
||||
if self.args.log_htp:
|
||||
self.log(self.name, "workers += {} = {}".format(n, self.tp_nthr), 6)
|
||||
|
||||
for _ in range(n):
|
||||
thr = threading.Thread(
|
||||
target=self.thr_poolw,
|
||||
name=self.name + "-poolw",
|
||||
)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
def stop_threads(self, n: int) -> None:
|
||||
self.tp_nthr -= n
|
||||
if self.args.log_htp:
|
||||
self.log(self.name, "workers -= {} = {}".format(n, self.tp_nthr), 6)
|
||||
|
||||
assert self.tp_q
|
||||
for _ in range(n):
|
||||
self.tp_q.put(None)
|
||||
|
||||
def periodic(self) -> None:
|
||||
while True:
|
||||
time.sleep(2 if self.tp_ncli or self.ncli else 10)
|
||||
with self.mutex:
|
||||
self.u2fh.clean()
|
||||
if self.tp_q:
|
||||
self.tp_ncli = max(self.ncli, self.tp_ncli - 2)
|
||||
if self.tp_nthr > self.tp_ncli + 8:
|
||||
self.stop_threads(4)
|
||||
|
||||
if not self.ncli and not self.u2fh.cache and self.tp_nthr <= 8:
|
||||
self.t_periodic = None
|
||||
return
|
||||
|
||||
def listen(self, sck: socket.socket, nlisteners: int) -> None:
|
||||
if self.args.j != 1:
|
||||
# lost in the pickle; redefine
|
||||
sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
sck.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
sck.settimeout(None) # < does not inherit, ^ does
|
||||
|
||||
ip, port = sck.getsockname()
|
||||
self.srvs.append(sck)
|
||||
self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
|
||||
t = threading.Thread(
|
||||
target=self.thr_listen,
|
||||
args=(sck,),
|
||||
name="httpsrv-n{}-listen-{}-{}".format(self.nid or "0", ip, port),
|
||||
)
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
def thr_listen(self, srv_sck: socket.socket) -> None:
|
||||
"""listens on a shared tcp server"""
|
||||
ip, port = srv_sck.getsockname()
|
||||
fno = srv_sck.fileno()
|
||||
msg = "subscribed @ {}:{} f{} p{}".format(ip, port, fno, os.getpid())
|
||||
self.log(self.name, msg)
|
||||
|
||||
def fun() -> None:
|
||||
self.broker.say("cb_httpsrv_up")
|
||||
|
||||
threading.Thread(target=fun, name="sig-hsrv-up1").start()
|
||||
|
||||
while not self.stopping:
|
||||
if self.args.log_conn:
|
||||
self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="1;30")
|
||||
|
||||
if self.ncli >= self.nclimax:
|
||||
self.log(self.name, "at connection limit; waiting", 3)
|
||||
while self.ncli >= self.nclimax:
|
||||
time.sleep(0.1)
|
||||
|
||||
if self.args.log_conn:
|
||||
self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="1;30")
|
||||
|
||||
try:
|
||||
sck, addr = srv_sck.accept()
|
||||
except (OSError, socket.error) as ex:
|
||||
self.log(self.name, "accept({}): {}".format(fno, ex), c=6)
|
||||
time.sleep(0.02)
|
||||
continue
|
||||
|
||||
if self.args.log_conn:
|
||||
t = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
|
||||
"-" * 3, ip, port % 8, port
|
||||
)
|
||||
self.log("%s %s" % addr, t, c="1;30")
|
||||
|
||||
self.accept(sck, addr)
|
||||
|
||||
def accept(self, sck: socket.socket, addr: tuple[str, int]) -> None:
|
||||
def accept(self, sck, addr):
|
||||
"""takes an incoming tcp connection and creates a thread to handle it"""
|
||||
now = time.time()
|
||||
|
||||
if now - (self.tp_time or now) > 300:
|
||||
t = "httpserver threadpool died: tpt {:.2f}, now {:.2f}, nthr {}, ncli {}"
|
||||
self.log(self.name, t.format(self.tp_time, now, self.tp_nthr, self.ncli), 1)
|
||||
self.tp_time = 0
|
||||
self.tp_q = None
|
||||
|
||||
with self.mutex:
|
||||
self.ncli += 1
|
||||
if not self.t_periodic:
|
||||
name = "hsrv-pt"
|
||||
if self.nid:
|
||||
name += "-{}".format(self.nid)
|
||||
|
||||
thr = threading.Thread(target=self.periodic, name=name)
|
||||
self.t_periodic = thr
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
if self.tp_q:
|
||||
self.tp_time = self.tp_time or now
|
||||
self.tp_ncli = max(self.tp_ncli, self.ncli)
|
||||
if self.tp_nthr < self.ncli + 4:
|
||||
self.start_threads(8)
|
||||
|
||||
self.tp_q.put((sck, addr))
|
||||
return
|
||||
|
||||
if not self.args.no_htp:
|
||||
t = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
|
||||
self.log(self.name, t, 1)
|
||||
if self.args.log_conn:
|
||||
self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
|
||||
|
||||
thr = threading.Thread(
|
||||
target=self.thr_client,
|
||||
args=(sck, addr),
|
||||
name="httpconn-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
|
||||
name="httpsrv-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
|
||||
)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
def thr_poolw(self) -> None:
|
||||
assert self.tp_q
|
||||
while True:
|
||||
task = self.tp_q.get()
|
||||
if not task:
|
||||
break
|
||||
|
||||
def num_clients(self):
|
||||
with self.mutex:
|
||||
self.tp_time = 0
|
||||
return len(self.clients)
|
||||
|
||||
def shutdown(self):
|
||||
clients = list(self.clients.keys())
|
||||
for cli in clients:
|
||||
try:
|
||||
sck, addr = task
|
||||
me = threading.current_thread()
|
||||
me.name = "httpconn-{}-{}".format(
|
||||
addr[0].split(".", 2)[-1][-6:], addr[1]
|
||||
)
|
||||
self.thr_client(sck, addr)
|
||||
me.name = self.name + "-poolw"
|
||||
except Exception as ex:
|
||||
if str(ex).startswith("client d/c "):
|
||||
self.log(self.name, "thr_client: " + str(ex), 6)
|
||||
else:
|
||||
self.log(self.name, "thr_client: " + min_ex(), 3)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.stopping = True
|
||||
for srv in self.srvs:
|
||||
try:
|
||||
srv.close()
|
||||
cli.shutdown()
|
||||
except:
|
||||
pass
|
||||
|
||||
thrs = []
|
||||
clients = list(self.clients)
|
||||
for cli in clients:
|
||||
t = threading.Thread(target=cli.shutdown)
|
||||
thrs.append(t)
|
||||
t.start()
|
||||
self.log("httpsrv-n", "ok bye")
|
||||
|
||||
if self.tp_q:
|
||||
self.stop_threads(self.tp_nthr)
|
||||
for _ in range(10):
|
||||
time.sleep(0.05)
|
||||
if self.tp_q.empty():
|
||||
break
|
||||
|
||||
for t in thrs:
|
||||
t.join()
|
||||
|
||||
self.log(self.name, "ok bye")
|
||||
|
||||
def thr_client(self, sck: socket.socket, addr: tuple[str, int]) -> None:
|
||||
def thr_client(self, sck, addr):
|
||||
"""thread managing one tcp client"""
|
||||
sck.settimeout(120)
|
||||
|
||||
cli = HttpConn(sck, addr, self)
|
||||
with self.mutex:
|
||||
self.clients.add(cli)
|
||||
self.clients[cli] = 0
|
||||
|
||||
if self.is_mp:
|
||||
self.workload += 50
|
||||
if not self.workload_thr_alive:
|
||||
self.workload_thr_alive = True
|
||||
thr = threading.Thread(
|
||||
target=self.thr_workload, name="httpsrv-workload"
|
||||
)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
fno = sck.fileno()
|
||||
try:
|
||||
if self.args.log_conn:
|
||||
self.log("%s %s" % addr, "|%sC-crun" % ("-" * 4,), c="1;30")
|
||||
self.log("%s %s" % addr, "|%sC-crun" % ("-" * 6,), c="1;30")
|
||||
|
||||
cli.run()
|
||||
|
||||
except (OSError, socket.error) as ex:
|
||||
if ex.errno not in [10038, 10054, 107, 57, 49, 9]:
|
||||
if ex.errno not in [10038, 10054, 107, 57, 9]:
|
||||
self.log(
|
||||
"%s %s" % addr,
|
||||
"run({}): {}".format(fno, ex),
|
||||
@@ -342,11 +128,12 @@ class HttpSrv(object):
|
||||
finally:
|
||||
sck = cli.s
|
||||
if self.args.log_conn:
|
||||
self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 5,), c="1;30")
|
||||
self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
|
||||
|
||||
try:
|
||||
fno = sck.fileno()
|
||||
shut_socket(cli.log, sck)
|
||||
sck.shutdown(socket.SHUT_RDWR)
|
||||
sck.close()
|
||||
except (OSError, socket.error) as ex:
|
||||
if not MACOS:
|
||||
self.log(
|
||||
@@ -364,10 +151,38 @@ class HttpSrv(object):
|
||||
raise
|
||||
finally:
|
||||
with self.mutex:
|
||||
self.clients.remove(cli)
|
||||
self.ncli -= 1
|
||||
del self.clients[cli]
|
||||
|
||||
def cachebuster(self) -> str:
|
||||
if self.disconnect_func:
|
||||
self.disconnect_func(addr) # pylint: disable=not-callable
|
||||
|
||||
def thr_workload(self):
|
||||
"""indicates the python interpreter workload caused by this HttpSrv"""
|
||||
# avoid locking in extract_filedata by tracking difference here
|
||||
while True:
|
||||
time.sleep(0.2)
|
||||
with self.mutex:
|
||||
if not self.clients:
|
||||
# no clients rn, termiante thread
|
||||
self.workload_thr_alive = False
|
||||
self.workload = 0
|
||||
return
|
||||
|
||||
total = 0
|
||||
with self.mutex:
|
||||
for cli in self.clients.keys():
|
||||
now = cli.workload
|
||||
delta = now - self.clients[cli]
|
||||
if delta < 0:
|
||||
# was reset in HttpCli to prevent overflow
|
||||
delta = now
|
||||
|
||||
total += delta
|
||||
self.clients[cli] = now
|
||||
|
||||
self.workload = total
|
||||
|
||||
def cachebuster(self):
|
||||
if time.time() - self.cb_ts < 1:
|
||||
return self.cb_v
|
||||
|
||||
@@ -375,16 +190,16 @@ class HttpSrv(object):
|
||||
if time.time() - self.cb_ts < 1:
|
||||
return self.cb_v
|
||||
|
||||
v = self.E.t0
|
||||
v = E.t0
|
||||
try:
|
||||
with os.scandir(os.path.join(self.E.mod, "web")) as dh:
|
||||
with os.scandir(os.path.join(E.mod, "web")) as dh:
|
||||
for fh in dh:
|
||||
inf = fh.stat()
|
||||
inf = fh.stat(follow_symlinks=False)
|
||||
v = max(v, inf.st_mtime)
|
||||
except:
|
||||
pass
|
||||
|
||||
v = base64.urlsafe_b64encode(spack(b">xxL", int(v)))
|
||||
v = base64.urlsafe_b64encode(struct.pack(">xxL", int(v)))
|
||||
self.cb_v = v.decode("ascii")[-4:]
|
||||
self.cb_ts = time.time()
|
||||
return self.cb_v
|
||||
|
||||
@@ -1,69 +1,33 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import argparse # typechk
|
||||
import colorsys
|
||||
import hashlib
|
||||
import colorsys
|
||||
|
||||
from .__init__ import PY2
|
||||
from .th_srv import HAVE_PIL
|
||||
from .util import BytesIO
|
||||
|
||||
|
||||
class Ico(object):
|
||||
def __init__(self, args: argparse.Namespace) -> None:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
|
||||
def get(self, ext: str, as_thumb: bool, chrome: bool) -> tuple[str, bytes]:
|
||||
def get(self, ext, as_thumb):
|
||||
"""placeholder to make thumbnails not break"""
|
||||
|
||||
zb = hashlib.sha1(ext.encode("utf-8")).digest()[2:4]
|
||||
h = hashlib.md5(ext.encode("utf-8")).digest()[:2]
|
||||
if PY2:
|
||||
zb = [ord(x) for x in zb]
|
||||
h = [ord(x) for x in h]
|
||||
|
||||
c1 = colorsys.hsv_to_rgb(zb[0] / 256.0, 1, 0.3)
|
||||
c2 = colorsys.hsv_to_rgb(zb[0] / 256.0, 1, 1)
|
||||
ci = [int(x * 255) for x in list(c1) + list(c2)]
|
||||
c = "".join(["{:02x}".format(x) for x in ci])
|
||||
c1 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 0.3)
|
||||
c2 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 1)
|
||||
c = list(c1) + list(c2)
|
||||
c = [int(x * 255) for x in c]
|
||||
c = "".join(["{:02x}".format(x) for x in c])
|
||||
|
||||
w = 100
|
||||
h = 30
|
||||
if not self.args.th_no_crop and as_thumb:
|
||||
sw, sh = self.args.th_size.split("x")
|
||||
h = int(100 / (float(sw) / float(sh)))
|
||||
w = 100
|
||||
|
||||
if chrome and as_thumb:
|
||||
# cannot handle more than ~2000 unique SVGs
|
||||
if HAVE_PIL:
|
||||
# svg: 3s, cache: 6s, this: 8s
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
h = int(64 * h / w)
|
||||
w = 64
|
||||
img = Image.new("RGB", (w, h), "#" + c[:6])
|
||||
pb = ImageDraw.Draw(img)
|
||||
tw, th = pb.textsize(ext)
|
||||
pb.text(((w - tw) // 2, (h - th) // 2), ext, fill="#" + c[6:])
|
||||
img = img.resize((w * 3, h * 3), Image.NEAREST)
|
||||
|
||||
buf = BytesIO()
|
||||
img.save(buf, format="PNG", compress_level=1)
|
||||
return "image/png", buf.getvalue()
|
||||
|
||||
elif False:
|
||||
# 48s, too slow
|
||||
import pyvips
|
||||
|
||||
h = int(192 * h / w)
|
||||
w = 192
|
||||
img = pyvips.Image.text(
|
||||
ext, width=w, height=h, dpi=192, align=pyvips.Align.CENTRE
|
||||
)
|
||||
img = img.ifthenelse(ci[3:], ci[:3], blend=True)
|
||||
# i = i.resize(3, kernel=pyvips.Kernel.NEAREST)
|
||||
buf = img.write_to_buffer(".png[compression=1]")
|
||||
return "image/png", buf
|
||||
w, h = self.args.th_size.split("x")
|
||||
h = int(100 / (float(w) / float(h)))
|
||||
|
||||
svg = """\
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
@@ -73,6 +37,6 @@ class Ico(object):
|
||||
fill="#{}" font-family="monospace" font-size="14px" style="letter-spacing:.5px">{}</text>
|
||||
</g></svg>
|
||||
"""
|
||||
svg = svg.format(h, c[:6], c[6:], ext)
|
||||
svg = svg.format(h, c[:6], c[6:], ext).encode("utf-8")
|
||||
|
||||
return "image/svg+xml", svg.encode("utf-8")
|
||||
return ["image/svg+xml", svg]
|
||||
|
||||
@@ -1,26 +1,20 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import shutil
|
||||
import subprocess as sp
|
||||
import sys
|
||||
|
||||
from .__init__ import PY2, WINDOWS, E, unicode
|
||||
from .bos import bos
|
||||
from .util import REKOBO_LKEY, fsenc, min_ex, retchk, runcmd, uncyg
|
||||
from .__init__ import PY2, WINDOWS
|
||||
from .util import fsenc, fsdec, uncyg, REKOBO_LKEY
|
||||
|
||||
try:
|
||||
from typing import Any, Union
|
||||
|
||||
from .util import RootLogger
|
||||
except:
|
||||
pass
|
||||
if not PY2:
|
||||
unicode = str
|
||||
|
||||
|
||||
def have_ff(cmd: str) -> bool:
|
||||
def have_ff(cmd):
|
||||
if PY2:
|
||||
print("# checking {}".format(cmd))
|
||||
cmd = (cmd + " -version").encode("ascii").split(b" ")
|
||||
@@ -38,16 +32,13 @@ HAVE_FFPROBE = have_ff("ffprobe")
|
||||
|
||||
|
||||
class MParser(object):
|
||||
def __init__(self, cmdline: str) -> None:
|
||||
def __init__(self, cmdline):
|
||||
self.tag, args = cmdline.split("=", 1)
|
||||
self.tags = self.tag.split(",")
|
||||
|
||||
self.timeout = 60
|
||||
self.timeout = 30
|
||||
self.force = False
|
||||
self.kill = "t" # tree; all children recursively
|
||||
self.capture = 3 # outputs to consume
|
||||
self.audio = "y"
|
||||
self.pri = 0 # priority; higher = later
|
||||
self.ext = []
|
||||
|
||||
while True:
|
||||
@@ -56,7 +47,7 @@ class MParser(object):
|
||||
if WINDOWS:
|
||||
bp = uncyg(bp)
|
||||
|
||||
if bos.path.exists(bp):
|
||||
if os.path.exists(bp):
|
||||
self.bin = bp
|
||||
return
|
||||
except:
|
||||
@@ -69,14 +60,6 @@ class MParser(object):
|
||||
self.audio = arg[1:] # [r]equire [n]ot [d]ontcare
|
||||
continue
|
||||
|
||||
if arg.startswith("k"):
|
||||
self.kill = arg[1:] # [t]ree [m]ain [n]one
|
||||
continue
|
||||
|
||||
if arg.startswith("c"):
|
||||
self.capture = int(arg[1:]) # 0=none 1=stdout 2=stderr 3=both
|
||||
continue
|
||||
|
||||
if arg == "f":
|
||||
self.force = True
|
||||
continue
|
||||
@@ -89,16 +72,10 @@ class MParser(object):
|
||||
self.ext.append(arg[1:])
|
||||
continue
|
||||
|
||||
if arg.startswith("p"):
|
||||
self.pri = int(arg[1:] or "1")
|
||||
continue
|
||||
|
||||
raise Exception()
|
||||
|
||||
|
||||
def ffprobe(
|
||||
abspath: str, timeout: int = 60
|
||||
) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
|
||||
def ffprobe(abspath):
|
||||
cmd = [
|
||||
b"ffprobe",
|
||||
b"-hide_banner",
|
||||
@@ -107,20 +84,21 @@ def ffprobe(
|
||||
b"--",
|
||||
fsenc(abspath),
|
||||
]
|
||||
rc, so, se = runcmd(cmd, timeout=timeout)
|
||||
retchk(rc, cmd, se)
|
||||
return parse_ffprobe(so)
|
||||
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
|
||||
r = p.communicate()
|
||||
txt = r[0].decode("utf-8", "replace")
|
||||
return parse_ffprobe(txt)
|
||||
|
||||
|
||||
def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
|
||||
def parse_ffprobe(txt):
|
||||
"""ffprobe -show_format -show_streams"""
|
||||
streams = []
|
||||
fmt = {}
|
||||
g = {}
|
||||
g = None
|
||||
for ln in [x.rstrip("\r") for x in txt.split("\n")]:
|
||||
try:
|
||||
sk, sv = ln.split("=", 1)
|
||||
g[sk] = sv
|
||||
k, v = ln.split("=", 1)
|
||||
g[k] = v
|
||||
continue
|
||||
except:
|
||||
pass
|
||||
@@ -134,8 +112,8 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
|
||||
fmt = g
|
||||
|
||||
streams = [fmt] + streams
|
||||
ret: dict[str, Any] = {} # processed
|
||||
md: dict[str, list[Any]] = {} # raw tags
|
||||
ret = {} # processed
|
||||
md = {} # raw tags
|
||||
|
||||
is_audio = fmt.get("format_name") in ["mp3", "ogg", "flac", "wav"]
|
||||
if fmt.get("filename", "").split(".")[-1].lower() in ["m4a", "aac"]:
|
||||
@@ -183,55 +161,52 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
|
||||
]
|
||||
|
||||
if typ == "format":
|
||||
kvm = [["duration", ".dur"], ["bit_rate", ".q"], ["format_name", "fmt"]]
|
||||
kvm = [["duration", ".dur"], ["bit_rate", ".q"]]
|
||||
|
||||
for sk, rk in kvm:
|
||||
v1 = strm.get(sk)
|
||||
if v1 is None:
|
||||
v = strm.get(sk)
|
||||
if v is None:
|
||||
continue
|
||||
|
||||
if rk.startswith("."):
|
||||
try:
|
||||
zf = float(v1)
|
||||
v = float(v)
|
||||
v2 = ret.get(rk)
|
||||
if v2 is None or zf > v2:
|
||||
ret[rk] = zf
|
||||
if v2 is None or v > v2:
|
||||
ret[rk] = v
|
||||
except:
|
||||
# sqlite doesnt care but the code below does
|
||||
if v1 not in ["N/A"]:
|
||||
ret[rk] = v1
|
||||
if v not in ["N/A"]:
|
||||
ret[rk] = v
|
||||
else:
|
||||
ret[rk] = v1
|
||||
ret[rk] = v
|
||||
|
||||
if ret.get("vc") == "ansi": # shellscript
|
||||
return {}, {}
|
||||
|
||||
for strm in streams:
|
||||
for sk, sv in strm.items():
|
||||
if not sk.startswith("TAG:"):
|
||||
for k, v in strm.items():
|
||||
if not k.startswith("TAG:"):
|
||||
continue
|
||||
|
||||
sk = sk[4:].strip()
|
||||
sv = sv.strip()
|
||||
if sk and sv and sk not in md:
|
||||
md[sk] = [sv]
|
||||
k = k[4:].strip()
|
||||
v = v.strip()
|
||||
if k and v and k not in md:
|
||||
md[k] = [v]
|
||||
|
||||
for sk in [".q", ".vq", ".aq"]:
|
||||
if sk in ret:
|
||||
ret[sk] /= 1000 # bit_rate=320000
|
||||
for k in [".q", ".vq", ".aq"]:
|
||||
if k in ret:
|
||||
ret[k] /= 1000 # bit_rate=320000
|
||||
|
||||
for sk in [".q", ".vq", ".aq", ".resw", ".resh"]:
|
||||
if sk in ret:
|
||||
ret[sk] = int(ret[sk])
|
||||
for k in [".q", ".vq", ".aq", ".resw", ".resh"]:
|
||||
if k in ret:
|
||||
ret[k] = int(ret[k])
|
||||
|
||||
if ".fps" in ret:
|
||||
fps = ret[".fps"]
|
||||
if "/" in fps:
|
||||
fa, fb = fps.split("/")
|
||||
try:
|
||||
fps = int(fa) * 1.0 / int(fb)
|
||||
except:
|
||||
fps = 9001
|
||||
|
||||
if fps < 1000 and fmt.get("format_name") not in ["image2", "png_pipe"]:
|
||||
ret[".fps"] = round(fps, 3)
|
||||
@@ -244,52 +219,48 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
|
||||
if ".q" in ret:
|
||||
del ret[".q"]
|
||||
|
||||
if "fmt" in ret:
|
||||
ret["fmt"] = ret["fmt"].split(",")[0]
|
||||
|
||||
if ".resw" in ret and ".resh" in ret:
|
||||
ret["res"] = "{}x{}".format(ret[".resw"], ret[".resh"])
|
||||
|
||||
zd = {k: (0, v) for k, v in ret.items()}
|
||||
ret = {k: [0, v] for k, v in ret.items()}
|
||||
|
||||
return zd, md
|
||||
return ret, md
|
||||
|
||||
|
||||
class MTag(object):
|
||||
def __init__(self, log_func: "RootLogger", args: argparse.Namespace) -> None:
|
||||
def __init__(self, log_func, args):
|
||||
self.log_func = log_func
|
||||
self.args = args
|
||||
self.usable = True
|
||||
self.prefer_mt = not args.no_mtag_ff
|
||||
self.backend = "ffprobe" if args.no_mutagen else "mutagen"
|
||||
self.can_ffprobe = HAVE_FFPROBE and not args.no_mtag_ff
|
||||
self.prefer_mt = False
|
||||
mappings = args.mtm
|
||||
or_ffprobe = " or FFprobe"
|
||||
self.backend = "ffprobe" if args.no_mutagen else "mutagen"
|
||||
or_ffprobe = " or ffprobe"
|
||||
|
||||
if self.backend == "mutagen":
|
||||
self.get = self.get_mutagen
|
||||
try:
|
||||
import mutagen # noqa: F401 # pylint: disable=unused-import,import-outside-toplevel
|
||||
import mutagen
|
||||
except:
|
||||
self.log("could not load Mutagen, trying FFprobe instead", c=3)
|
||||
self.log("could not load mutagen, trying ffprobe instead", c=3)
|
||||
self.backend = "ffprobe"
|
||||
|
||||
if self.backend == "ffprobe":
|
||||
self.usable = self.can_ffprobe
|
||||
self.get = self.get_ffprobe
|
||||
self.prefer_mt = True
|
||||
# about 20x slower
|
||||
self.usable = HAVE_FFPROBE
|
||||
|
||||
if not HAVE_FFPROBE:
|
||||
pass
|
||||
|
||||
elif args.no_mtag_ff:
|
||||
msg = "found FFprobe but it was disabled by --no-mtag-ff"
|
||||
self.log(msg, c=3)
|
||||
if self.usable and WINDOWS and sys.version_info < (3, 8):
|
||||
self.usable = False
|
||||
or_ffprobe = " or python >= 3.8"
|
||||
msg = "found ffprobe but your python is too old; need 3.8 or newer"
|
||||
self.log(msg, c=1)
|
||||
|
||||
if not self.usable:
|
||||
msg = "need Mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
|
||||
pybin = os.path.basename(sys.executable)
|
||||
self.log(msg.format(or_ffprobe, " " * 37, pybin), c=1)
|
||||
msg = "need mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
|
||||
self.log(
|
||||
msg.format(or_ffprobe, " " * 37, os.path.basename(sys.executable)), c=1
|
||||
)
|
||||
return
|
||||
|
||||
# https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
|
||||
@@ -361,33 +332,31 @@ class MTag(object):
|
||||
}
|
||||
# self.get = self.compare
|
||||
|
||||
def log(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
def log(self, msg, c=0):
|
||||
self.log_func("mtag", msg, c)
|
||||
|
||||
def normalize_tags(
|
||||
self, parser_output: dict[str, tuple[int, Any]], md: dict[str, list[Any]]
|
||||
) -> dict[str, Union[str, float]]:
|
||||
for sk, tv in dict(md).items():
|
||||
if not tv:
|
||||
def normalize_tags(self, ret, md):
|
||||
for k, v in dict(md).items():
|
||||
if not v:
|
||||
continue
|
||||
|
||||
sk = sk.lower().split("::")[0].strip()
|
||||
key_mapping = self.rmap.get(sk)
|
||||
if not key_mapping:
|
||||
k = k.lower().split("::")[0].strip()
|
||||
mk = self.rmap.get(k)
|
||||
if not mk:
|
||||
continue
|
||||
|
||||
priority, alias = key_mapping
|
||||
if alias not in parser_output or parser_output[alias][0] > priority:
|
||||
parser_output[alias] = (priority, tv[0])
|
||||
pref, mk = mk
|
||||
if mk not in ret or ret[mk][0] > pref:
|
||||
ret[mk] = [pref, v[0]]
|
||||
|
||||
# take first value (lowest priority / most preferred)
|
||||
ret = {sk: unicode(tv[1]).strip() for sk, tv in parser_output.items()}
|
||||
# take first value
|
||||
ret = {k: unicode(v[1]).strip() for k, v in ret.items()}
|
||||
|
||||
# track 3/7 => track 3
|
||||
for sk, tv in ret.items():
|
||||
if sk[0] == ".":
|
||||
sv = str(tv).split("/")[0].strip().lstrip("0")
|
||||
ret[sk] = sv or 0
|
||||
for k, v in ret.items():
|
||||
if k[0] == ".":
|
||||
v = v.split("/")[0].strip().lstrip("0")
|
||||
ret[k] = v or 0
|
||||
|
||||
# normalize key notation to rkeobo
|
||||
okey = ret.get("key")
|
||||
@@ -397,7 +366,7 @@ class MTag(object):
|
||||
|
||||
return ret
|
||||
|
||||
def compare(self, abspath: str) -> dict[str, Union[str, float]]:
|
||||
def compare(self, abspath):
|
||||
if abspath.endswith(".au"):
|
||||
return {}
|
||||
|
||||
@@ -421,7 +390,7 @@ class MTag(object):
|
||||
v2 = r2.get(k)
|
||||
if v1 == v2:
|
||||
print(" ", k, v1)
|
||||
elif v1 != "0000": # FFprobe date=0
|
||||
elif v1 != "0000": # ffprobe date=0
|
||||
diffs.append(k)
|
||||
print(" 1", k, v1)
|
||||
print(" 2", k, v2)
|
||||
@@ -435,124 +404,65 @@ class MTag(object):
|
||||
|
||||
return r1
|
||||
|
||||
def get_mutagen(self, abspath: str) -> dict[str, Union[str, float]]:
|
||||
ret: dict[str, tuple[int, Any]] = {}
|
||||
|
||||
if not bos.path.isfile(abspath):
|
||||
return {}
|
||||
|
||||
def get_mutagen(self, abspath):
|
||||
import mutagen
|
||||
|
||||
try:
|
||||
md = mutagen.File(fsenc(abspath), easy=True)
|
||||
if not md.info.length and not md.info.codec:
|
||||
raise Exception()
|
||||
except:
|
||||
return self.get_ffprobe(abspath) if self.can_ffprobe else {}
|
||||
x = md.info.length
|
||||
except Exception as ex:
|
||||
return {}
|
||||
|
||||
sz = bos.path.getsize(abspath)
|
||||
ret = {}
|
||||
try:
|
||||
ret[".q"] = (0, int((sz / md.info.length) / 128))
|
||||
dur = int(md.info.length)
|
||||
try:
|
||||
q = int(md.info.bitrate / 1024)
|
||||
except:
|
||||
q = int((os.path.getsize(fsenc(abspath)) / dur) / 128)
|
||||
|
||||
ret[".dur"] = [0, dur]
|
||||
ret[".q"] = [0, q]
|
||||
except:
|
||||
pass
|
||||
|
||||
for attr, k, norm in [
|
||||
["codec", "ac", unicode],
|
||||
["channels", "chs", int],
|
||||
["sample_rate", ".hz", int],
|
||||
["bitrate", ".aq", int],
|
||||
["length", ".dur", int],
|
||||
]:
|
||||
try:
|
||||
v = getattr(md.info, attr)
|
||||
except:
|
||||
if k != "ac":
|
||||
continue
|
||||
|
||||
try:
|
||||
v = str(md.info).split(".")[1]
|
||||
if v.startswith("ogg"):
|
||||
v = v[3:]
|
||||
except:
|
||||
continue
|
||||
|
||||
if not v:
|
||||
continue
|
||||
|
||||
if k == ".aq":
|
||||
v /= 1000
|
||||
|
||||
if k == "ac" and v.startswith("mp4a.40."):
|
||||
v = "aac"
|
||||
|
||||
ret[k] = (0, norm(v))
|
||||
|
||||
return self.normalize_tags(ret, md)
|
||||
|
||||
def get_ffprobe(self, abspath: str) -> dict[str, Union[str, float]]:
|
||||
if not bos.path.isfile(abspath):
|
||||
return {}
|
||||
|
||||
ret, md = ffprobe(abspath, self.args.mtag_to)
|
||||
def get_ffprobe(self, abspath):
|
||||
ret, md = ffprobe(abspath)
|
||||
return self.normalize_tags(ret, md)
|
||||
|
||||
def get_bin(
|
||||
self, parsers: dict[str, MParser], abspath: str, oth_tags: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
if not bos.path.isfile(abspath):
|
||||
return {}
|
||||
|
||||
env = os.environ.copy()
|
||||
try:
|
||||
def get_bin(self, parsers, abspath):
|
||||
pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
|
||||
zsl = [str(pypath)] + [str(x) for x in sys.path if x]
|
||||
pypath = str(os.pathsep.join(zsl))
|
||||
pypath = [str(pypath)] + [str(x) for x in sys.path if x]
|
||||
pypath = str(os.pathsep.join(pypath))
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = pypath
|
||||
except:
|
||||
if not E.ox:
|
||||
raise
|
||||
|
||||
ret: dict[str, Any] = {}
|
||||
for tagname, parser in sorted(parsers.items(), key=lambda x: (x[1].pri, x[0])):
|
||||
ret = {}
|
||||
for tagname, mp in parsers.items():
|
||||
try:
|
||||
cmd = [parser.bin, abspath]
|
||||
if parser.bin.endswith(".py"):
|
||||
cmd = [sys.executable] + cmd
|
||||
|
||||
args = {
|
||||
"env": env,
|
||||
"timeout": parser.timeout,
|
||||
"kill": parser.kill,
|
||||
"capture": parser.capture,
|
||||
}
|
||||
|
||||
if parser.pri:
|
||||
zd = oth_tags.copy()
|
||||
zd.update(ret)
|
||||
args["sin"] = json.dumps(zd).encode("utf-8", "replace")
|
||||
cmd = [sys.executable, mp.bin, abspath]
|
||||
args = {"env": env, "timeout": mp.timeout}
|
||||
|
||||
if WINDOWS:
|
||||
args["creationflags"] = 0x4000
|
||||
else:
|
||||
cmd = ["nice"] + cmd
|
||||
|
||||
bcmd = [fsenc(x) for x in cmd]
|
||||
rc, v, err = runcmd(bcmd, **args) # type: ignore
|
||||
retchk(rc, bcmd, err, self.log, 5, self.args.mtag_v)
|
||||
v = v.strip()
|
||||
cmd = [fsenc(x) for x in cmd]
|
||||
v = sp.check_output(cmd, **args).strip()
|
||||
if not v:
|
||||
continue
|
||||
|
||||
if "," not in tagname:
|
||||
ret[tagname] = v
|
||||
ret[tagname] = v.decode("utf-8")
|
||||
else:
|
||||
zj = json.loads(v)
|
||||
v = json.loads(v)
|
||||
for tag in tagname.split(","):
|
||||
if tag and tag in zj:
|
||||
ret[tag] = zj[tag]
|
||||
if tag and tag in v:
|
||||
ret[tag] = v[tag]
|
||||
except:
|
||||
if self.args.mtag_v:
|
||||
t = "mtag error: tagname {}, parser {}, file {} => {}"
|
||||
self.log(t.format(tagname, parser.bin, abspath, min_ex()))
|
||||
pass
|
||||
|
||||
return ret
|
||||
|
||||
@@ -1,32 +1,23 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import tarfile
|
||||
import threading
|
||||
|
||||
from queue import Queue
|
||||
|
||||
from .bos import bos
|
||||
from .sutil import StreamArc, errdesc
|
||||
from .util import fsenc, min_ex
|
||||
|
||||
try:
|
||||
from typing import Any, Generator, Optional
|
||||
|
||||
from .util import NamedLogger
|
||||
except:
|
||||
pass
|
||||
from .sutil import errdesc
|
||||
from .util import Queue, fsenc
|
||||
|
||||
|
||||
class QFile(object): # inherit io.StringIO for painful typing
|
||||
class QFile(object):
|
||||
"""file-like object which buffers writes into a queue"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.q: Queue[Optional[bytes]] = Queue(64)
|
||||
self.bq: list[bytes] = []
|
||||
def __init__(self):
|
||||
self.q = Queue(64)
|
||||
self.bq = []
|
||||
self.nq = 0
|
||||
|
||||
def write(self, buf: Optional[bytes]) -> None:
|
||||
def write(self, buf):
|
||||
if buf is None or self.nq >= 240 * 1024:
|
||||
self.q.put(b"".join(self.bq))
|
||||
self.bq = []
|
||||
@@ -39,33 +30,26 @@ class QFile(object): # inherit io.StringIO for painful typing
|
||||
self.nq += len(buf)
|
||||
|
||||
|
||||
class StreamTar(StreamArc):
|
||||
class StreamTar(object):
|
||||
"""construct in-memory tar file from the given path"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
log: "NamedLogger",
|
||||
fgen: Generator[dict[str, Any], None, None],
|
||||
**kwargs: Any
|
||||
):
|
||||
super(StreamTar, self).__init__(log, fgen)
|
||||
|
||||
def __init__(self, fgen, **kwargs):
|
||||
self.ci = 0
|
||||
self.co = 0
|
||||
self.qfile = QFile()
|
||||
self.errf: dict[str, Any] = {}
|
||||
self.fgen = fgen
|
||||
self.errf = None
|
||||
|
||||
# python 3.8 changed to PAX_FORMAT as default,
|
||||
# waste of space and don't care about the new features
|
||||
fmt = tarfile.GNU_FORMAT
|
||||
self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt) # type: ignore
|
||||
self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
|
||||
|
||||
w = threading.Thread(target=self._gen, name="star-gen")
|
||||
w.daemon = True
|
||||
w.start()
|
||||
|
||||
def gen(self) -> Generator[Optional[bytes], None, None]:
|
||||
try:
|
||||
def gen(self):
|
||||
while True:
|
||||
buf = self.qfile.q.get()
|
||||
if not buf:
|
||||
@@ -75,11 +59,10 @@ class StreamTar(StreamArc):
|
||||
yield buf
|
||||
|
||||
yield None
|
||||
finally:
|
||||
if self.errf:
|
||||
bos.unlink(self.errf["ap"])
|
||||
os.unlink(self.errf["ap"])
|
||||
|
||||
def ser(self, f: dict[str, Any]) -> None:
|
||||
def ser(self, f):
|
||||
name = f["vp"]
|
||||
src = f["ap"]
|
||||
fsi = f["st"]
|
||||
@@ -92,25 +75,23 @@ class StreamTar(StreamArc):
|
||||
inf.gid = 0
|
||||
|
||||
self.ci += inf.size
|
||||
with open(fsenc(src), "rb", 512 * 1024) as fo:
|
||||
self.tar.addfile(inf, fo)
|
||||
with open(fsenc(src), "rb", 512 * 1024) as f:
|
||||
self.tar.addfile(inf, f)
|
||||
|
||||
def _gen(self) -> None:
|
||||
def _gen(self):
|
||||
errors = []
|
||||
for f in self.fgen:
|
||||
if "err" in f:
|
||||
errors.append((f["vp"], f["err"]))
|
||||
errors.append([f["vp"], f["err"]])
|
||||
continue
|
||||
|
||||
try:
|
||||
self.ser(f)
|
||||
except:
|
||||
ex = min_ex(5, True).replace("\n", "\n-- ")
|
||||
errors.append((f["vp"], ex))
|
||||
except Exception as ex:
|
||||
errors.append([f["vp"], repr(ex)])
|
||||
|
||||
if errors:
|
||||
self.errf, txt = errdesc(errors)
|
||||
self.log("\n".join(([repr(self.errf)] + txt[1:])))
|
||||
self.errf = errdesc(errors)
|
||||
self.ser(self.errf)
|
||||
|
||||
self.tar.close()
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
# coding: utf-8
|
||||
|
||||
"""
|
||||
This is Victor Stinner's pure-Python implementation of PEP 383: the "surrogateescape" error
|
||||
handler of Python 3.
|
||||
@@ -12,28 +10,23 @@ Original source: misc/python/surrogateescape.py in https://bitbucket.org/haypo/m
|
||||
|
||||
# This code is released under the Python license and the BSD 2-clause license
|
||||
|
||||
import codecs
|
||||
import platform
|
||||
import codecs
|
||||
import sys
|
||||
|
||||
PY3 = sys.version_info[0] > 2
|
||||
WINDOWS = platform.system() == "Windows"
|
||||
FS_ERRORS = "surrogateescape"
|
||||
|
||||
try:
|
||||
from typing import Any
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def u(text: Any) -> str:
|
||||
def u(text):
|
||||
if PY3:
|
||||
return text
|
||||
else:
|
||||
return text.decode("unicode_escape")
|
||||
|
||||
|
||||
def b(data: Any) -> bytes:
|
||||
def b(data):
|
||||
if PY3:
|
||||
return data.encode("latin1")
|
||||
else:
|
||||
@@ -48,7 +41,7 @@ else:
|
||||
bytes_chr = chr
|
||||
|
||||
|
||||
def surrogateescape_handler(exc: Any) -> tuple[str, int]:
|
||||
def surrogateescape_handler(exc):
|
||||
"""
|
||||
Pure Python implementation of the PEP 383: the "surrogateescape" error
|
||||
handler of Python 3. Undecodable bytes will be replaced by a Unicode
|
||||
@@ -79,7 +72,7 @@ class NotASurrogateError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def replace_surrogate_encode(mystring: str) -> str:
|
||||
def replace_surrogate_encode(mystring):
|
||||
"""
|
||||
Returns a (unicode) string, not the more logical bytes, because the codecs
|
||||
register_error functionality expects this.
|
||||
@@ -105,7 +98,7 @@ def replace_surrogate_encode(mystring: str) -> str:
|
||||
return str().join(decoded)
|
||||
|
||||
|
||||
def replace_surrogate_decode(mybytes: bytes) -> str:
|
||||
def replace_surrogate_decode(mybytes):
|
||||
"""
|
||||
Returns a (unicode) string
|
||||
"""
|
||||
@@ -126,7 +119,7 @@ def replace_surrogate_decode(mybytes: bytes) -> str:
|
||||
return str().join(decoded)
|
||||
|
||||
|
||||
def encodefilename(fn: str) -> bytes:
|
||||
def encodefilename(fn):
|
||||
if FS_ENCODING == "ascii":
|
||||
# ASCII encoder of Python 2 expects that the error handler returns a
|
||||
# Unicode string encodable to ASCII, whereas our surrogateescape error
|
||||
@@ -166,7 +159,7 @@ def encodefilename(fn: str) -> bytes:
|
||||
return fn.encode(FS_ENCODING, FS_ERRORS)
|
||||
|
||||
|
||||
def decodefilename(fn: bytes) -> str:
|
||||
def decodefilename(fn):
|
||||
return fn.decode(FS_ENCODING, FS_ERRORS)
|
||||
|
||||
|
||||
@@ -178,7 +171,7 @@ FS_ENCODING = sys.getfilesystemencoding()
|
||||
|
||||
if WINDOWS and not PY3:
|
||||
# py2 thinks win* is mbcs, probably a bug? anyways this works
|
||||
FS_ENCODING = "utf-8"
|
||||
FS_ENCODING = 'utf-8'
|
||||
|
||||
|
||||
# normalize the filesystem encoding name.
|
||||
@@ -186,7 +179,7 @@ if WINDOWS and not PY3:
|
||||
FS_ENCODING = codecs.lookup(FS_ENCODING).name
|
||||
|
||||
|
||||
def register_surrogateescape() -> None:
|
||||
def register_surrogateescape():
|
||||
"""
|
||||
Registers the surrogateescape error handler on Python 2 (only)
|
||||
"""
|
||||
|
||||
@@ -1,34 +1,13 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import time
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
|
||||
from .bos import bos
|
||||
|
||||
try:
|
||||
from typing import Any, Generator, Optional
|
||||
|
||||
from .util import NamedLogger
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class StreamArc(object):
|
||||
def __init__(
|
||||
self,
|
||||
log: "NamedLogger",
|
||||
fgen: Generator[dict[str, Any], None, None],
|
||||
**kwargs: Any
|
||||
):
|
||||
self.log = log
|
||||
self.fgen = fgen
|
||||
|
||||
def gen(self) -> Generator[Optional[bytes], None, None]:
|
||||
pass
|
||||
|
||||
|
||||
def errdesc(errors: list[tuple[str, str]]) -> tuple[dict[str, Any], list[str]]:
|
||||
def errdesc(errors):
|
||||
report = ["copyparty failed to add the following files to the archive:", ""]
|
||||
|
||||
for fn, err in errors:
|
||||
@@ -38,11 +17,12 @@ def errdesc(errors: list[tuple[str, str]]) -> tuple[dict[str, Any], list[str]]:
|
||||
tf_path = tf.name
|
||||
tf.write("\r\n".join(report).encode("utf-8", "replace"))
|
||||
|
||||
dt = datetime.utcnow().strftime("%Y-%m%d-%H%M%S")
|
||||
dt = datetime.utcfromtimestamp(time.time())
|
||||
dt = dt.strftime("%Y-%m%d-%H%M%S")
|
||||
|
||||
bos.chmod(tf_path, 0o444)
|
||||
os.chmod(tf_path, 0o444)
|
||||
return {
|
||||
"vp": "archive-errors-{}.txt".format(dt),
|
||||
"ap": tf_path,
|
||||
"st": bos.stat(tf_path),
|
||||
}, report
|
||||
"st": os.stat(tf_path),
|
||||
}
|
||||
|
||||
@@ -1,388 +1,90 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import calendar
|
||||
import gzip
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import signal
|
||||
import socket
|
||||
import string
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
import calendar
|
||||
|
||||
try:
|
||||
from types import FrameType
|
||||
|
||||
import typing
|
||||
from typing import Any, Optional, Union
|
||||
except:
|
||||
pass
|
||||
|
||||
from .__init__ import ANYWIN, MACOS, PY2, VT100, WINDOWS, EnvParams, unicode
|
||||
from .__init__ import PY2, WINDOWS, MACOS, VT100
|
||||
from .util import mp
|
||||
from .authsrv import AuthSrv
|
||||
from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
|
||||
from .tcpsrv import TcpSrv
|
||||
from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
|
||||
from .up2k import Up2k
|
||||
from .util import (
|
||||
VERSIONS,
|
||||
HMaccas,
|
||||
alltrace,
|
||||
ansi_re,
|
||||
min_ex,
|
||||
mp,
|
||||
start_log_thrs,
|
||||
start_stackmon,
|
||||
)
|
||||
from .th_srv import ThumbSrv, HAVE_PIL, HAVE_WEBP
|
||||
|
||||
|
||||
class SvcHub(object):
|
||||
"""
|
||||
Hosts all services which cannot be parallelized due to reliance on monolithic resources.
|
||||
Creates a Broker which does most of the heavy stuff; hosted services can use this to perform work:
|
||||
hub.broker.<say|ask>(destination, args_list).
|
||||
hub.broker.put(want_reply, destination, args_list).
|
||||
|
||||
Either BrokerThr (plain threads) or BrokerMP (multiprocessing) is used depending on configuration.
|
||||
Nothing is returned synchronously; if you want any value returned from the call,
|
||||
put() can return a queue (if want_reply=True) which has a blocking get() with the response.
|
||||
"""
|
||||
|
||||
def __init__(self, args: argparse.Namespace, argv: list[str], printed: str) -> None:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.argv = argv
|
||||
self.E: EnvParams = args.E
|
||||
self.logf: Optional[typing.TextIO] = None
|
||||
self.logf_base_fn = ""
|
||||
self.stop_req = False
|
||||
self.stopping = False
|
||||
self.stopped = False
|
||||
self.reload_req = False
|
||||
self.reloading = False
|
||||
self.stop_cond = threading.Condition()
|
||||
self.nsigs = 3
|
||||
self.retcode = 0
|
||||
self.httpsrv_up = 0
|
||||
|
||||
self.ansi_re = re.compile("\033\\[[^m]*m")
|
||||
self.log_mutex = threading.Lock()
|
||||
self.next_day = 0
|
||||
self.tstack = 0.0
|
||||
|
||||
self.iphash = HMaccas(os.path.join(self.E.cfg, "iphash"), 8)
|
||||
|
||||
if args.sss or args.s >= 3:
|
||||
args.ss = True
|
||||
args.lo = args.lo or "cpp-%Y-%m%d-%H%M%S.txt.xz"
|
||||
args.ls = args.ls or "**,*,ln,p,r"
|
||||
|
||||
if args.ss or args.s >= 2:
|
||||
args.s = True
|
||||
args.no_logues = True
|
||||
args.no_readme = True
|
||||
args.unpost = 0
|
||||
args.no_del = True
|
||||
args.no_mv = True
|
||||
args.hardlink = True
|
||||
args.vague_403 = True
|
||||
args.ban_404 = "50,60,1440"
|
||||
args.nih = True
|
||||
|
||||
if args.s:
|
||||
args.dotpart = True
|
||||
args.no_thumb = True
|
||||
args.no_mtag_ff = True
|
||||
args.no_robots = True
|
||||
args.force_js = True
|
||||
|
||||
self.log = self._log_disabled if args.q else self._log_enabled
|
||||
if args.lo:
|
||||
self._setup_logfile(printed)
|
||||
|
||||
if args.stackmon:
|
||||
start_stackmon(args.stackmon, 0)
|
||||
|
||||
if args.log_thrs:
|
||||
start_log_thrs(self.log, args.log_thrs, 0)
|
||||
|
||||
if not args.use_fpool and args.j != 1:
|
||||
args.no_fpool = True
|
||||
t = "multithreading enabled with -j {}, so disabling fpool -- this can reduce upload performance on some filesystems"
|
||||
self.log("root", t.format(args.j))
|
||||
|
||||
if not args.no_fpool and args.j != 1:
|
||||
t = "WARNING: --use-fpool combined with multithreading is untested and can probably cause undefined behavior"
|
||||
if ANYWIN:
|
||||
t = 'windows cannot do multithreading without --no-fpool, so enabling that -- note that upload performance will suffer if you have microsoft defender "real-time protection" enabled, so you probably want to use -j 1 instead'
|
||||
args.no_fpool = True
|
||||
|
||||
self.log("root", t, c=3)
|
||||
|
||||
bri = "zy"[args.theme % 2 :][:1]
|
||||
ch = "abcdefghijklmnopqrstuvwx"[int(args.theme / 2)]
|
||||
args.theme = "{0}{1} {0} {1}".format(ch, bri)
|
||||
|
||||
if not args.hardlink and args.never_symlink:
|
||||
args.no_dedup = True
|
||||
|
||||
if args.log_fk:
|
||||
args.log_fk = re.compile(args.log_fk)
|
||||
|
||||
# initiate all services to manage
|
||||
self.asrv = AuthSrv(self.args, self.log)
|
||||
self.asrv = AuthSrv(self.args, self.log, False)
|
||||
if args.ls:
|
||||
self.asrv.dbg_ls()
|
||||
|
||||
self.tcpsrv = TcpSrv(self)
|
||||
self.up2k = Up2k(self)
|
||||
|
||||
decs = {k: 1 for k in self.args.th_dec.split(",")}
|
||||
if not HAVE_VIPS:
|
||||
decs.pop("vips", None)
|
||||
if not HAVE_PIL:
|
||||
decs.pop("pil", None)
|
||||
if not HAVE_FFMPEG or not HAVE_FFPROBE:
|
||||
decs.pop("ff", None)
|
||||
|
||||
self.args.th_dec = list(decs.keys())
|
||||
self.thumbsrv = None
|
||||
if not args.no_thumb:
|
||||
t = ", ".join(self.args.th_dec) or "(None available)"
|
||||
self.log("thumb", "decoder preference: {}".format(t))
|
||||
|
||||
if "pil" in self.args.th_dec and not HAVE_WEBP:
|
||||
msg = "disabling webp thumbnails because either libwebp is not available or your Pillow is too old"
|
||||
if HAVE_PIL:
|
||||
if not HAVE_WEBP:
|
||||
args.th_no_webp = True
|
||||
msg = "setting --th-no-webp because either libwebp is not available or your Pillow is too old"
|
||||
self.log("thumb", msg, c=3)
|
||||
|
||||
if self.args.th_dec:
|
||||
self.thumbsrv = ThumbSrv(self)
|
||||
else:
|
||||
msg = "need either Pillow, pyvips, or FFmpeg to create thumbnails; for example:\n{0}{1} -m pip install --user Pillow\n{0}{1} -m pip install --user pyvips\n{0}apt install ffmpeg"
|
||||
msg = msg.format(" " * 37, os.path.basename(sys.executable))
|
||||
self.log("thumb", msg, c=3)
|
||||
|
||||
if not args.no_acode and args.no_thumb:
|
||||
msg = "setting --no-acode because --no-thumb (sorry)"
|
||||
self.log("thumb", msg, c=6)
|
||||
args.no_acode = True
|
||||
|
||||
if not args.no_acode and (not HAVE_FFMPEG or not HAVE_FFPROBE):
|
||||
msg = "setting --no-acode because either FFmpeg or FFprobe is not available"
|
||||
self.log("thumb", msg, c=6)
|
||||
args.no_acode = True
|
||||
|
||||
args.th_poke = min(args.th_poke, args.th_maxage, args.ac_maxage)
|
||||
|
||||
if args.ftp or args.ftps:
|
||||
from .ftpd import Ftpd
|
||||
|
||||
self.ftpd = Ftpd(self)
|
||||
msg = "need Pillow to create thumbnails; for example:\n{}{} -m pip install --user Pillow\n"
|
||||
self.log(
|
||||
"thumb", msg.format(" " * 37, os.path.basename(sys.executable)), c=3
|
||||
)
|
||||
|
||||
# decide which worker impl to use
|
||||
if self.check_mp_enable():
|
||||
from .broker_mp import BrokerMp as Broker
|
||||
else:
|
||||
from .broker_thr import BrokerThr as Broker # type: ignore
|
||||
self.log("root", "cannot efficiently use multiple CPU cores")
|
||||
from .broker_thr import BrokerThr as Broker
|
||||
|
||||
self.broker = Broker(self)
|
||||
|
||||
def thr_httpsrv_up(self) -> None:
|
||||
time.sleep(1 if self.args.ign_ebind_all else 5)
|
||||
expected = self.broker.num_workers * self.tcpsrv.nsrv
|
||||
failed = expected - self.httpsrv_up
|
||||
if not failed:
|
||||
return
|
||||
|
||||
if self.args.ign_ebind_all:
|
||||
if not self.tcpsrv.srv:
|
||||
for _ in range(self.broker.num_workers):
|
||||
self.broker.say("cb_httpsrv_up")
|
||||
return
|
||||
|
||||
if self.args.ign_ebind and self.tcpsrv.srv:
|
||||
return
|
||||
|
||||
t = "{}/{} workers failed to start"
|
||||
t = t.format(failed, expected)
|
||||
self.log("root", t, 1)
|
||||
|
||||
self.retcode = 1
|
||||
self.sigterm()
|
||||
|
||||
def sigterm(self) -> None:
|
||||
os.kill(os.getpid(), signal.SIGTERM)
|
||||
|
||||
def cb_httpsrv_up(self) -> None:
|
||||
self.httpsrv_up += 1
|
||||
if self.httpsrv_up != self.broker.num_workers:
|
||||
return
|
||||
|
||||
time.sleep(0.1) # purely cosmetic dw
|
||||
self.log("root", "workers OK\n")
|
||||
self.up2k.init_vols()
|
||||
|
||||
thr = threading.Thread(target=self.sd_notify, name="sd-notify")
|
||||
def run(self):
|
||||
thr = threading.Thread(target=self.tcpsrv.run, name="svchub-main")
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
def _logname(self) -> str:
|
||||
dt = datetime.utcnow()
|
||||
fn = str(self.args.lo)
|
||||
for fs in "YmdHMS":
|
||||
fs = "%" + fs
|
||||
if fs in fn:
|
||||
fn = fn.replace(fs, dt.strftime(fs))
|
||||
|
||||
return fn
|
||||
|
||||
def _setup_logfile(self, printed: str) -> None:
|
||||
base_fn = fn = sel_fn = self._logname()
|
||||
if fn != self.args.lo:
|
||||
ctr = 0
|
||||
# yup this is a race; if started sufficiently concurrently, two
|
||||
# copyparties can grab the same logfile (considered and ignored)
|
||||
while os.path.exists(sel_fn):
|
||||
ctr += 1
|
||||
sel_fn = "{}.{}".format(fn, ctr)
|
||||
|
||||
fn = sel_fn
|
||||
|
||||
# winxp/py2.7 support: thr.join() kills signals
|
||||
try:
|
||||
import lzma
|
||||
while True:
|
||||
time.sleep(9001)
|
||||
|
||||
lh = lzma.open(fn, "wt", encoding="utf-8", errors="replace", preset=0)
|
||||
except KeyboardInterrupt:
|
||||
with self.log_mutex:
|
||||
print("OPYTHAT")
|
||||
|
||||
except:
|
||||
import codecs
|
||||
|
||||
lh = codecs.open(fn, "w", encoding="utf-8", errors="replace")
|
||||
|
||||
argv = [sys.executable] + self.argv
|
||||
if hasattr(shlex, "quote"):
|
||||
argv = [shlex.quote(x) for x in argv]
|
||||
else:
|
||||
argv = ['"{}"'.format(x) for x in argv]
|
||||
|
||||
msg = "[+] opened logfile [{}]\n".format(fn)
|
||||
printed += msg
|
||||
lh.write("t0: {:.3f}\nargv: {}\n\n{}".format(self.E.t0, " ".join(argv), printed))
|
||||
self.logf = lh
|
||||
self.logf_base_fn = base_fn
|
||||
print(msg, end="")
|
||||
|
||||
def run(self) -> None:
|
||||
self.tcpsrv.run()
|
||||
|
||||
thr = threading.Thread(target=self.thr_httpsrv_up, name="sig-hsrv-up2")
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
sigs = [signal.SIGINT, signal.SIGTERM]
|
||||
if not ANYWIN:
|
||||
sigs.append(signal.SIGUSR1)
|
||||
|
||||
for sig in sigs:
|
||||
signal.signal(sig, self.signal_handler)
|
||||
|
||||
# macos hangs after shutdown on sigterm with while-sleep,
|
||||
# windows cannot ^c stop_cond (and win10 does the macos thing but winxp is fine??)
|
||||
# linux is fine with both,
|
||||
# never lucky
|
||||
if ANYWIN:
|
||||
# msys-python probably fine but >msys-python
|
||||
thr = threading.Thread(target=self.stop_thr, name="svchub-sig")
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
try:
|
||||
while not self.stop_req:
|
||||
time.sleep(1)
|
||||
except:
|
||||
pass
|
||||
|
||||
self.shutdown()
|
||||
# cant join; eats signals on win10
|
||||
while not self.stopped:
|
||||
time.sleep(0.1)
|
||||
else:
|
||||
self.stop_thr()
|
||||
|
||||
def reload(self) -> str:
|
||||
if self.reloading:
|
||||
return "cannot reload; already in progress"
|
||||
|
||||
self.reloading = True
|
||||
t = threading.Thread(target=self._reload, name="reloading")
|
||||
t.daemon = True
|
||||
t.start()
|
||||
return "reload initiated"
|
||||
|
||||
def _reload(self) -> None:
|
||||
self.log("root", "reload scheduled")
|
||||
with self.up2k.mutex:
|
||||
self.asrv.reload()
|
||||
self.up2k.reload()
|
||||
self.broker.reload()
|
||||
|
||||
self.reloading = False
|
||||
|
||||
def stop_thr(self) -> None:
|
||||
while not self.stop_req:
|
||||
with self.stop_cond:
|
||||
self.stop_cond.wait(9001)
|
||||
|
||||
if self.reload_req:
|
||||
self.reload_req = False
|
||||
self.reload()
|
||||
|
||||
self.shutdown()
|
||||
|
||||
def signal_handler(self, sig: int, frame: Optional[FrameType]) -> None:
|
||||
if self.stopping:
|
||||
if self.nsigs <= 0:
|
||||
try:
|
||||
threading.Thread(target=self.pr, args=("OMBO BREAKER",)).start()
|
||||
time.sleep(0.1)
|
||||
except:
|
||||
pass
|
||||
|
||||
if ANYWIN:
|
||||
os.system("taskkill /f /pid {}".format(os.getpid()))
|
||||
else:
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
else:
|
||||
self.nsigs -= 1
|
||||
return
|
||||
|
||||
if not ANYWIN and sig == signal.SIGUSR1:
|
||||
self.reload_req = True
|
||||
else:
|
||||
self.stop_req = True
|
||||
|
||||
with self.stop_cond:
|
||||
self.stop_cond.notify_all()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.stopping:
|
||||
return
|
||||
|
||||
# start_log_thrs(print, 0.1, 1)
|
||||
|
||||
self.stopping = True
|
||||
self.stop_req = True
|
||||
with self.stop_cond:
|
||||
self.stop_cond.notify_all()
|
||||
|
||||
ret = 1
|
||||
try:
|
||||
self.pr("OPYTHAT")
|
||||
self.tcpsrv.shutdown()
|
||||
self.broker.shutdown()
|
||||
self.up2k.shutdown()
|
||||
if self.thumbsrv:
|
||||
self.thumbsrv.shutdown()
|
||||
|
||||
@@ -392,43 +94,22 @@ class SvcHub(object):
|
||||
break
|
||||
|
||||
if n == 3:
|
||||
self.pr("waiting for thumbsrv (10sec)...")
|
||||
print("waiting for thumbsrv (10sec)...")
|
||||
|
||||
self.pr("nailed it", end="")
|
||||
ret = self.retcode
|
||||
except:
|
||||
self.pr("\033[31m[ error during shutdown ]\n{}\033[0m".format(min_ex()))
|
||||
raise
|
||||
print("nailed it", end="")
|
||||
finally:
|
||||
if self.args.wintitle:
|
||||
print("\033]0;\033\\", file=sys.stderr, end="")
|
||||
sys.stderr.flush()
|
||||
print("\033[0m")
|
||||
|
||||
self.pr("\033[0m")
|
||||
if self.logf:
|
||||
self.logf.close()
|
||||
|
||||
self.stopped = True
|
||||
sys.exit(ret)
|
||||
|
||||
def _log_disabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
|
||||
if not self.logf:
|
||||
return
|
||||
def _log_disabled(self, src, msg, c=0):
|
||||
pass
|
||||
|
||||
def _log_enabled(self, src, msg, c=0):
|
||||
"""handles logging from all components"""
|
||||
with self.log_mutex:
|
||||
ts = datetime.utcnow().strftime("%Y-%m%d-%H%M%S.%f")[:-3]
|
||||
self.logf.write("@{} [{}] {}\n".format(ts, src, msg))
|
||||
|
||||
now = time.time()
|
||||
if now >= self.next_day:
|
||||
self._set_next_day()
|
||||
|
||||
def _set_next_day(self) -> None:
|
||||
if self.next_day and self.logf and self.logf_base_fn != self._logname():
|
||||
self.logf.close()
|
||||
self._setup_logfile("")
|
||||
|
||||
dt = datetime.utcnow()
|
||||
dt = datetime.utcfromtimestamp(now)
|
||||
print("\033[36m{}\033[0m\n".format(dt.strftime("%Y-%m-%d")), end="")
|
||||
|
||||
# unix timestamp of next 00:00:00 (leap-seconds safe)
|
||||
day_now = dt.day
|
||||
@@ -438,25 +119,16 @@ class SvcHub(object):
|
||||
dt = dt.replace(hour=0, minute=0, second=0)
|
||||
self.next_day = calendar.timegm(dt.utctimetuple())
|
||||
|
||||
def _log_enabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
|
||||
"""handles logging from all components"""
|
||||
with self.log_mutex:
|
||||
now = time.time()
|
||||
if now >= self.next_day:
|
||||
dt = datetime.utcfromtimestamp(now)
|
||||
print("\033[36m{}\033[0m\n".format(dt.strftime("%Y-%m-%d")), end="")
|
||||
self._set_next_day()
|
||||
|
||||
fmt = "\033[36m{} \033[33m{:21} \033[0m{}\n"
|
||||
if not VT100:
|
||||
fmt = "{} {:21} {}\n"
|
||||
if "\033" in msg:
|
||||
msg = ansi_re.sub("", msg)
|
||||
msg = self.ansi_re.sub("", msg)
|
||||
if "\033" in src:
|
||||
src = ansi_re.sub("", src)
|
||||
src = self.ansi_re.sub("", src)
|
||||
elif c:
|
||||
if isinstance(c, int):
|
||||
msg = "\033[3{}m{}\033[0m".format(c, msg)
|
||||
msg = "\033[3{}m{}".format(c, msg)
|
||||
elif "\033" not in c:
|
||||
msg = "\033[{}m{}\033[0m".format(c, msg)
|
||||
else:
|
||||
@@ -472,45 +144,38 @@ class SvcHub(object):
|
||||
except:
|
||||
print(msg.encode("ascii", "replace").decode(), end="")
|
||||
|
||||
if self.logf:
|
||||
self.logf.write(msg)
|
||||
|
||||
def pr(self, *a: Any, **ka: Any) -> None:
|
||||
with self.log_mutex:
|
||||
print(*a, **ka)
|
||||
|
||||
def check_mp_support(self) -> str:
|
||||
def check_mp_support(self):
|
||||
vmin = sys.version_info[1]
|
||||
if WINDOWS:
|
||||
msg = "need python 3.3 or newer for multiprocessing;"
|
||||
if PY2 or vmin < 3:
|
||||
if PY2:
|
||||
# py2 pickler doesn't support winsock
|
||||
return msg
|
||||
elif vmin < 3:
|
||||
return msg
|
||||
elif MACOS:
|
||||
return "multiprocessing is wonky on mac osx;"
|
||||
else:
|
||||
msg = "need python 3.3+ for multiprocessing;"
|
||||
if PY2 or vmin < 3:
|
||||
msg = "need python 2.7 or 3.3+ for multiprocessing;"
|
||||
if not PY2 and vmin < 3:
|
||||
return msg
|
||||
|
||||
try:
|
||||
x: mp.Queue[tuple[str, str]] = mp.Queue(1)
|
||||
x.put(("foo", "bar"))
|
||||
x = mp.Queue(1)
|
||||
x.put(["foo", "bar"])
|
||||
if x.get()[0] != "foo":
|
||||
raise Exception()
|
||||
except:
|
||||
return "multiprocessing is not supported on your platform;"
|
||||
|
||||
return ""
|
||||
return None
|
||||
|
||||
def check_mp_enable(self) -> bool:
|
||||
def check_mp_enable(self):
|
||||
if self.args.j == 1:
|
||||
self.log("root", "multiprocessing disabled by argument -j 1;")
|
||||
return False
|
||||
|
||||
try:
|
||||
if mp.cpu_count() <= 1:
|
||||
raise Exception()
|
||||
except:
|
||||
self.log("svchub", "only one CPU detected; multiprocessing disabled")
|
||||
return False
|
||||
|
||||
try:
|
||||
@@ -524,38 +189,5 @@ class SvcHub(object):
|
||||
if not err:
|
||||
return True
|
||||
else:
|
||||
self.log("svchub", err)
|
||||
self.log("svchub", "cannot efficiently use multiple CPU cores")
|
||||
self.log("root", err)
|
||||
return False
|
||||
|
||||
def sd_notify(self) -> None:
|
||||
try:
|
||||
zb = os.getenv("NOTIFY_SOCKET")
|
||||
if not zb:
|
||||
return
|
||||
|
||||
addr = unicode(zb)
|
||||
if addr.startswith("@"):
|
||||
addr = "\0" + addr[1:]
|
||||
|
||||
t = "".join(x for x in addr if x in string.printable)
|
||||
self.log("sd_notify", t)
|
||||
|
||||
sck = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
sck.connect(addr)
|
||||
sck.sendall(b"READY=1")
|
||||
except:
|
||||
self.log("sd_notify", min_ex())
|
||||
|
||||
def log_stacks(self) -> None:
|
||||
td = time.time() - self.tstack
|
||||
if td < 300:
|
||||
self.log("stacks", "cooldown {}".format(td))
|
||||
return
|
||||
|
||||
self.tstack = time.time()
|
||||
zs = "{}\n{}".format(VERSIONS, alltrace())
|
||||
zb = zs.encode("utf-8", "replace")
|
||||
zb = gzip.compress(zb)
|
||||
zs = base64.b64encode(zb).decode("ascii")
|
||||
self.log("stacks", zs)
|
||||
|
||||
@@ -1,24 +1,18 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import calendar
|
||||
import os
|
||||
import time
|
||||
import zlib
|
||||
import struct
|
||||
from datetime import datetime
|
||||
|
||||
from .bos import bos
|
||||
from .sutil import StreamArc, errdesc
|
||||
from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile
|
||||
|
||||
try:
|
||||
from typing import Any, Generator, Optional
|
||||
|
||||
from .util import NamedLogger
|
||||
except:
|
||||
pass
|
||||
from .sutil import errdesc
|
||||
from .util import yieldfile, sanitize_fn
|
||||
|
||||
|
||||
def dostime2unix(buf: bytes) -> int:
|
||||
t, d = sunpack(b"<HH", buf)
|
||||
def dostime2unix(buf):
|
||||
t, d = struct.unpack("<HH", buf)
|
||||
|
||||
ts = (t & 0x1F) * 2
|
||||
tm = (t >> 5) & 0x3F
|
||||
@@ -32,38 +26,27 @@ def dostime2unix(buf: bytes) -> int:
|
||||
tf = "{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}"
|
||||
iso = tf.format(*tt)
|
||||
|
||||
dt = time.strptime(iso, "%Y-%m-%d %H:%M:%S")
|
||||
return int(calendar.timegm(dt))
|
||||
dt = datetime.strptime(iso, "%Y-%m-%d %H:%M:%S")
|
||||
return int(dt.timestamp())
|
||||
|
||||
|
||||
def unixtime2dos(ts: int) -> bytes:
|
||||
tt = time.gmtime(ts + 1)
|
||||
def unixtime2dos(ts):
|
||||
tt = time.gmtime(ts)
|
||||
dy, dm, dd, th, tm, ts = list(tt)[:6]
|
||||
|
||||
bd = ((dy - 1980) << 9) + (dm << 5) + dd
|
||||
bt = (th << 11) + (tm << 5) + ts // 2
|
||||
try:
|
||||
return spack(b"<HH", bt, bd)
|
||||
except:
|
||||
return b"\x00\x00\x21\x00"
|
||||
return struct.pack("<HH", bt, bd)
|
||||
|
||||
|
||||
def gen_fdesc(sz: int, crc32: int, z64: bool) -> bytes:
|
||||
def gen_fdesc(sz, crc32, z64):
|
||||
ret = b"\x50\x4b\x07\x08"
|
||||
fmt = b"<LQQ" if z64 else b"<LLL"
|
||||
ret += spack(fmt, crc32, sz, sz)
|
||||
fmt = "<LQQ" if z64 else "<LLL"
|
||||
ret += struct.pack(fmt, crc32, sz, sz)
|
||||
return ret
|
||||
|
||||
|
||||
def gen_hdr(
|
||||
h_pos: Optional[int],
|
||||
fn: str,
|
||||
sz: int,
|
||||
lastmod: int,
|
||||
utf8: bool,
|
||||
icrc32: int,
|
||||
pre_crc: bool,
|
||||
) -> bytes:
|
||||
def gen_hdr(h_pos, fn, sz, lastmod, utf8, crc32, pre_crc):
|
||||
"""
|
||||
does regular file headers
|
||||
and the central directory meme if h_pos is set
|
||||
@@ -82,8 +65,8 @@ def gen_hdr(
|
||||
# confusingly this doesn't bump if h_pos
|
||||
req_ver = b"\x2d\x00" if z64 else b"\x0a\x00"
|
||||
|
||||
if icrc32:
|
||||
crc32 = spack(b"<L", icrc32)
|
||||
if crc32:
|
||||
crc32 = struct.pack("<L", crc32)
|
||||
else:
|
||||
crc32 = b"\x00" * 4
|
||||
|
||||
@@ -91,7 +74,7 @@ def gen_hdr(
|
||||
# 4b magic, 2b min-ver
|
||||
ret = b"\x50\x4b\x03\x04" + req_ver
|
||||
else:
|
||||
# 4b magic, 2b spec-ver (1b compat, 1b os (00 dos, 03 unix)), 2b min-ver
|
||||
# 4b magic, 2b spec-ver, 2b min-ver
|
||||
ret = b"\x50\x4b\x01\x02\x1e\x03" + req_ver
|
||||
|
||||
ret += b"\x00" if pre_crc else b"\x08" # streaming
|
||||
@@ -104,49 +87,36 @@ def gen_hdr(
|
||||
# however infozip does actual sz and it even works on winxp
|
||||
# (same reasning for z64 extradata later)
|
||||
vsz = 0xFFFFFFFF if z64 else sz
|
||||
ret += spack(b"<LL", vsz, vsz)
|
||||
ret += struct.pack("<LL", vsz, vsz)
|
||||
|
||||
# windows support (the "?" replace below too)
|
||||
fn = sanitize_fn(fn, "/", [])
|
||||
fn = sanitize_fn(fn, ok="/")
|
||||
bfn = fn.encode("utf-8" if utf8 else "cp437", "replace").replace(b"?", b"_")
|
||||
|
||||
# add ntfs (0x24) and/or unix (0x10) extrafields for utc, add z64 if requested
|
||||
z64_len = len(z64v) * 8 + 4 if z64v else 0
|
||||
ret += spack(b"<HH", len(bfn), 0x10 + z64_len)
|
||||
ret += struct.pack("<HH", len(bfn), z64_len)
|
||||
|
||||
if h_pos is not None:
|
||||
# 2b comment, 2b diskno
|
||||
ret += b"\x00" * 4
|
||||
|
||||
# 2b internal.attr, 4b external.attr
|
||||
# infozip-macos: 0100 0000 a481 (spec-ver 1e03) file:644
|
||||
# infozip-macos: 0100 0100 0080 (spec-ver 1e03) file:000
|
||||
# win10-zip: 0000 2000 0000 (spec-ver xx00) FILE_ATTRIBUTE_ARCHIVE
|
||||
ret += b"\x00\x00\x00\x00\xa4\x81" # unx
|
||||
# ret += b"\x00\x00\x20\x00\x00\x00" # fat
|
||||
# infozip-macos: 0100 0000 a481 file:644
|
||||
# infozip-macos: 0100 0100 0080 file:000
|
||||
ret += b"\x01\x00\x00\x00\xa4\x81"
|
||||
|
||||
# 4b local-header-ofs
|
||||
ret += spack(b"<L", min(h_pos, 0xFFFFFFFF))
|
||||
ret += struct.pack("<L", min(h_pos, 0xFFFFFFFF))
|
||||
|
||||
ret += bfn
|
||||
|
||||
# ntfs: type 0a, size 20, rsvd, attr1, len 18, mtime, atime, ctime
|
||||
# b"\xa3\x2f\x82\x41\x55\x68\xd8\x01" 1652616838.798941100 ~5.861518 132970904387989411 ~58615181
|
||||
# nt = int((lastmod + 11644473600) * 10000000)
|
||||
# ret += spack(b"<HHLHHQQQ", 0xA, 0x20, 0, 1, 0x18, nt, nt, nt)
|
||||
|
||||
# unix: type 0d, size 0c, atime, mtime, uid, gid
|
||||
ret += spack(b"<HHLLHH", 0xD, 0xC, int(lastmod), int(lastmod), 1000, 1000)
|
||||
|
||||
if z64v:
|
||||
ret += spack(b"<HH" + b"Q" * len(z64v), 1, len(z64v) * 8, *z64v)
|
||||
ret += struct.pack("<HH" + "Q" * len(z64v), 1, len(z64v) * 8, *z64v)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def gen_ecdr(
|
||||
items: list[tuple[str, int, int, int, int]], cdir_pos: int, cdir_end: int
|
||||
) -> tuple[bytes, bool]:
|
||||
def gen_ecdr(items, cdir_pos, cdir_end):
|
||||
"""
|
||||
summary of all file headers,
|
||||
usually the zipfile footer unless something clamps
|
||||
@@ -166,17 +136,15 @@ def gen_ecdr(
|
||||
need_64 = nitems == 0xFFFF or 0xFFFFFFFF in [csz, cpos]
|
||||
|
||||
# 2b tnfiles, 2b dnfiles, 4b dir sz, 4b dir pos
|
||||
ret += spack(b"<HHLL", nitems, nitems, csz, cpos)
|
||||
ret += struct.pack("<HHLL", nitems, nitems, csz, cpos)
|
||||
|
||||
# 2b comment length
|
||||
ret += b"\x00\x00"
|
||||
|
||||
return ret, need_64
|
||||
return [ret, need_64]
|
||||
|
||||
|
||||
def gen_ecdr64(
|
||||
items: list[tuple[str, int, int, int, int]], cdir_pos: int, cdir_end: int
|
||||
) -> bytes:
|
||||
def gen_ecdr64(items, cdir_pos, cdir_end):
|
||||
"""
|
||||
z64 end of central directory
|
||||
added when numfiles or a headerptr clamps
|
||||
@@ -195,12 +163,12 @@ def gen_ecdr64(
|
||||
|
||||
# 8b tnfiles, 8b dnfiles, 8b dir sz, 8b dir pos
|
||||
cdir_sz = cdir_end - cdir_pos
|
||||
ret += spack(b"<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
|
||||
ret += struct.pack("<QQQQ", len(items), len(items), cdir_sz, cdir_pos)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def gen_ecdr64_loc(ecdr64_pos: int) -> bytes:
|
||||
def gen_ecdr64_loc(ecdr64_pos):
|
||||
"""
|
||||
z64 end of central directory locator
|
||||
points to ecdr64
|
||||
@@ -210,41 +178,35 @@ def gen_ecdr64_loc(ecdr64_pos: int) -> bytes:
|
||||
ret = b"\x50\x4b\x06\x07"
|
||||
|
||||
# 4b cdisk, 8b start of ecdr64, 4b ndisks
|
||||
ret += spack(b"<LQL", 0, ecdr64_pos, 1)
|
||||
ret += struct.pack("<LQL", 0, ecdr64_pos, 1)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
class StreamZip(StreamArc):
|
||||
def __init__(
|
||||
self,
|
||||
log: "NamedLogger",
|
||||
fgen: Generator[dict[str, Any], None, None],
|
||||
utf8: bool = False,
|
||||
pre_crc: bool = False,
|
||||
) -> None:
|
||||
super(StreamZip, self).__init__(log, fgen)
|
||||
|
||||
class StreamZip(object):
|
||||
def __init__(self, fgen, utf8=False, pre_crc=False):
|
||||
self.fgen = fgen
|
||||
self.utf8 = utf8
|
||||
self.pre_crc = pre_crc
|
||||
|
||||
self.pos = 0
|
||||
self.items: list[tuple[str, int, int, int, int]] = []
|
||||
self.items = []
|
||||
|
||||
def _ct(self, buf: bytes) -> bytes:
|
||||
def _ct(self, buf):
|
||||
self.pos += len(buf)
|
||||
return buf
|
||||
|
||||
def ser(self, f: dict[str, Any]) -> Generator[bytes, None, None]:
|
||||
def ser(self, f):
|
||||
name = f["vp"]
|
||||
src = f["ap"]
|
||||
st = f["st"]
|
||||
|
||||
sz = st.st_size
|
||||
ts = st.st_mtime
|
||||
ts = st.st_mtime + 1
|
||||
|
||||
crc = 0
|
||||
crc = None
|
||||
if self.pre_crc:
|
||||
crc = 0
|
||||
for buf in yieldfile(src):
|
||||
crc = zlib.crc32(buf, crc)
|
||||
|
||||
@@ -254,6 +216,7 @@ class StreamZip(StreamArc):
|
||||
buf = gen_hdr(None, name, sz, ts, self.utf8, crc, self.pre_crc)
|
||||
yield self._ct(buf)
|
||||
|
||||
crc = crc or 0
|
||||
for buf in yieldfile(src):
|
||||
if not self.pre_crc:
|
||||
crc = zlib.crc32(buf, crc)
|
||||
@@ -262,7 +225,7 @@ class StreamZip(StreamArc):
|
||||
|
||||
crc &= 0xFFFFFFFF
|
||||
|
||||
self.items.append((name, sz, ts, crc, h_pos))
|
||||
self.items.append([name, sz, ts, crc, h_pos])
|
||||
|
||||
z64 = sz >= 4 * 1024 * 1024 * 1024
|
||||
|
||||
@@ -270,26 +233,22 @@ class StreamZip(StreamArc):
|
||||
buf = gen_fdesc(sz, crc, z64)
|
||||
yield self._ct(buf)
|
||||
|
||||
def gen(self) -> Generator[bytes, None, None]:
|
||||
def gen(self):
|
||||
errors = []
|
||||
try:
|
||||
for f in self.fgen:
|
||||
if "err" in f:
|
||||
errors.append((f["vp"], f["err"]))
|
||||
errors.append([f["vp"], f["err"]])
|
||||
continue
|
||||
|
||||
try:
|
||||
for x in self.ser(f):
|
||||
yield x
|
||||
except GeneratorExit:
|
||||
raise
|
||||
except:
|
||||
ex = min_ex(5, True).replace("\n", "\n-- ")
|
||||
errors.append((f["vp"], ex))
|
||||
except Exception as ex:
|
||||
errors.append([f["vp"], repr(ex)])
|
||||
|
||||
if errors:
|
||||
errf, txt = errdesc(errors)
|
||||
self.log("\n".join(([repr(errf)] + txt[1:])))
|
||||
errf = errdesc(errors)
|
||||
print(repr(errf))
|
||||
for x in self.ser(errf):
|
||||
yield x
|
||||
|
||||
@@ -310,6 +269,6 @@ class StreamZip(StreamArc):
|
||||
|
||||
ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
|
||||
yield self._ct(ecdr)
|
||||
finally:
|
||||
|
||||
if errors:
|
||||
bos.unlink(errf["ap"])
|
||||
os.unlink(errf["ap"])
|
||||
|
||||
@@ -1,16 +1,12 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import socket
|
||||
import sys
|
||||
import select
|
||||
|
||||
from .__init__ import ANYWIN, MACOS, TYPE_CHECKING, unicode
|
||||
from .util import chkcmd
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
from .util import chkcmd, Counter
|
||||
|
||||
|
||||
class TcpSrv(object):
|
||||
@@ -19,37 +15,13 @@ class TcpSrv(object):
|
||||
which then uses the least busy HttpSrv to handle it
|
||||
"""
|
||||
|
||||
def __init__(self, hub: "SvcHub"):
|
||||
def __init__(self, hub):
|
||||
self.hub = hub
|
||||
self.args = hub.args
|
||||
self.log = hub.log
|
||||
|
||||
# mp-safe since issue6056
|
||||
socket.setdefaulttimeout(120)
|
||||
|
||||
self.num_clients = Counter()
|
||||
self.stopping = False
|
||||
self.srv: list[socket.socket] = []
|
||||
self.nsrv = 0
|
||||
ok: dict[str, list[int]] = {}
|
||||
for ip in self.args.i:
|
||||
ok[ip] = []
|
||||
for port in self.args.p:
|
||||
self.nsrv += 1
|
||||
try:
|
||||
self._listen(ip, port)
|
||||
ok[ip].append(port)
|
||||
except Exception as ex:
|
||||
if self.args.ign_ebind or self.args.ign_ebind_all:
|
||||
t = "could not listen on {}:{}: {}"
|
||||
self.log("tcpsrv", t.format(ip, port, ex), c=3)
|
||||
else:
|
||||
raise
|
||||
|
||||
if not self.srv and not self.args.ign_ebind_all:
|
||||
raise Exception("could not listen on any of the given interfaces")
|
||||
|
||||
if self.nsrv != len(self.srv):
|
||||
self.log("tcpsrv", "")
|
||||
|
||||
ip = "127.0.0.1"
|
||||
eps = {ip: "local only"}
|
||||
@@ -60,64 +32,27 @@ class TcpSrv(object):
|
||||
for x in nonlocals:
|
||||
eps[x] = "external"
|
||||
|
||||
msgs = []
|
||||
title_tab: dict[str, dict[str, int]] = {}
|
||||
title_vars = [x[1:] for x in self.args.wintitle.split(" ") if x.startswith("$")]
|
||||
t = "available @ {}://{}:{}/ (\033[33m{}\033[0m)"
|
||||
for ip, desc in sorted(eps.items(), key=lambda x: x[1]):
|
||||
for port in sorted(self.args.p):
|
||||
if port not in ok.get(ip, ok.get("0.0.0.0", [])):
|
||||
continue
|
||||
self.log(
|
||||
"tcpsrv",
|
||||
"available @ http://{}:{}/ (\033[33m{}\033[0m)".format(
|
||||
ip, port, desc
|
||||
),
|
||||
)
|
||||
|
||||
proto = " http"
|
||||
if self.args.http_only:
|
||||
pass
|
||||
elif self.args.https_only or port == 443:
|
||||
proto = "https"
|
||||
self.srv = []
|
||||
for ip in self.args.i:
|
||||
for port in self.args.p:
|
||||
self.srv.append(self._listen(ip, port))
|
||||
|
||||
msgs.append(t.format(proto, ip, port, desc))
|
||||
|
||||
if not self.args.wintitle:
|
||||
continue
|
||||
|
||||
if port in [80, 443]:
|
||||
ep = ip
|
||||
else:
|
||||
ep = "{}:{}".format(ip, port)
|
||||
|
||||
hits = []
|
||||
if "pub" in title_vars and "external" in unicode(desc):
|
||||
hits.append(("pub", ep))
|
||||
|
||||
if "pub" in title_vars or "all" in title_vars:
|
||||
hits.append(("all", ep))
|
||||
|
||||
for var in title_vars:
|
||||
if var.startswith("ip-") and ep.startswith(var[3:]):
|
||||
hits.append((var, ep))
|
||||
|
||||
for tk, tv in hits:
|
||||
try:
|
||||
title_tab[tk][tv] = 1
|
||||
except:
|
||||
title_tab[tk] = {tv: 1}
|
||||
|
||||
if msgs:
|
||||
msgs[-1] += "\n"
|
||||
for t in msgs:
|
||||
self.log("tcpsrv", t)
|
||||
|
||||
if self.args.wintitle:
|
||||
self._set_wintitle(title_tab)
|
||||
|
||||
def _listen(self, ip: str, port: int) -> None:
|
||||
def _listen(self, ip, port):
|
||||
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
srv.settimeout(None) # < does not inherit, ^ does
|
||||
try:
|
||||
srv.bind((ip, port))
|
||||
self.srv.append(srv)
|
||||
return srv
|
||||
except (OSError, socket.error) as ex:
|
||||
if ex.errno in [98, 48]:
|
||||
e = "\033[1;31mport {} is busy on interface {}\033[0m".format(port, ip)
|
||||
@@ -127,19 +62,50 @@ class TcpSrv(object):
|
||||
raise
|
||||
raise Exception(e)
|
||||
|
||||
def run(self) -> None:
|
||||
def run(self):
|
||||
for srv in self.srv:
|
||||
srv.listen(self.args.nc)
|
||||
ip, port = srv.getsockname()
|
||||
fno = srv.fileno()
|
||||
msg = "listening @ {}:{} f{} p{}".format(ip, port, fno, os.getpid())
|
||||
self.log("tcpsrv", msg)
|
||||
if self.args.q:
|
||||
print(msg)
|
||||
self.log("tcpsrv", "listening @ {0}:{1}".format(ip, port))
|
||||
|
||||
self.hub.broker.say("listen", srv)
|
||||
while not self.stopping:
|
||||
if self.args.log_conn:
|
||||
self.log("tcpsrv", "|%sC-ncli" % ("-" * 1,), c="1;30")
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.num_clients.v >= self.args.nc:
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
|
||||
if self.args.log_conn:
|
||||
self.log("tcpsrv", "|%sC-acc1" % ("-" * 2,), c="1;30")
|
||||
|
||||
try:
|
||||
# macos throws bad-fd
|
||||
ready, _, _ = select.select(self.srv, [], [])
|
||||
except:
|
||||
ready = []
|
||||
if not self.stopping:
|
||||
raise
|
||||
|
||||
for srv in ready:
|
||||
if self.stopping:
|
||||
break
|
||||
|
||||
sck, addr = srv.accept()
|
||||
sip, sport = srv.getsockname()
|
||||
if self.args.log_conn:
|
||||
self.log(
|
||||
"%s %s" % addr,
|
||||
"|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
|
||||
"-" * 3, sip, sport % 8, sport
|
||||
),
|
||||
c="1;30",
|
||||
)
|
||||
|
||||
self.num_clients.add()
|
||||
self.hub.broker.put(False, "httpconn", sck, addr)
|
||||
|
||||
def shutdown(self):
|
||||
self.stopping = True
|
||||
try:
|
||||
for srv in self.srv:
|
||||
@@ -149,149 +115,26 @@ class TcpSrv(object):
|
||||
|
||||
self.log("tcpsrv", "ok bye")
|
||||
|
||||
def ips_linux_ifconfig(self) -> dict[str, str]:
|
||||
# for termux
|
||||
def detect_interfaces(self, listen_ips):
|
||||
eps = {}
|
||||
|
||||
# get all ips and their interfaces
|
||||
try:
|
||||
txt, _ = chkcmd(["ifconfig"])
|
||||
ip_addr, _ = chkcmd("ip", "addr")
|
||||
except:
|
||||
return {}
|
||||
|
||||
eps: dict[str, str] = {}
|
||||
dev = None
|
||||
ip = None
|
||||
up = None
|
||||
for ln in (txt + "\n").split("\n"):
|
||||
if not ln.strip() and dev and ip:
|
||||
eps[ip] = dev + ("" if up else ", \033[31mLINK-DOWN")
|
||||
dev = ip = up = None
|
||||
continue
|
||||
|
||||
if ln == ln.lstrip():
|
||||
dev = re.split(r"[: ]", ln)[0]
|
||||
|
||||
if "UP" in re.split(r"[<>, \t]", ln):
|
||||
up = True
|
||||
|
||||
m = re.match(r"^\s+inet\s+([^ ]+)", ln)
|
||||
if m:
|
||||
ip = m.group(1)
|
||||
|
||||
return eps
|
||||
|
||||
def ips_linux(self) -> dict[str, str]:
|
||||
try:
|
||||
txt, _ = chkcmd(["ip", "addr"])
|
||||
except:
|
||||
return self.ips_linux_ifconfig()
|
||||
ip_addr = None
|
||||
|
||||
if ip_addr:
|
||||
r = re.compile(r"^\s+inet ([^ ]+)/.* (.*)")
|
||||
ri = re.compile(r"^\s*[0-9]+\s*:.*")
|
||||
up = False
|
||||
eps: dict[str, str] = {}
|
||||
for ln in txt.split("\n"):
|
||||
if ri.match(ln):
|
||||
up = "UP" in re.split("[>,< ]", ln)
|
||||
|
||||
for ln in ip_addr.split("\n"):
|
||||
try:
|
||||
ip, dev = r.match(ln.rstrip()).groups() # type: ignore
|
||||
eps[ip] = dev + ("" if up else ", \033[31mLINK-DOWN")
|
||||
ip, dev = r.match(ln.rstrip()).groups()
|
||||
for lip in listen_ips:
|
||||
if lip in ["0.0.0.0", ip]:
|
||||
eps[ip] = dev
|
||||
except:
|
||||
pass
|
||||
|
||||
return eps
|
||||
|
||||
def ips_macos(self) -> dict[str, str]:
|
||||
eps: dict[str, str] = {}
|
||||
try:
|
||||
txt, _ = chkcmd(["ifconfig"])
|
||||
except:
|
||||
return eps
|
||||
|
||||
rdev = re.compile(r"^([^ ]+):")
|
||||
rip = re.compile(r"^\tinet ([0-9\.]+) ")
|
||||
dev = "UNKNOWN"
|
||||
for ln in txt.split("\n"):
|
||||
m = rdev.match(ln)
|
||||
if m:
|
||||
dev = m.group(1)
|
||||
|
||||
m = rip.match(ln)
|
||||
if m:
|
||||
eps[m.group(1)] = dev
|
||||
dev = "UNKNOWN"
|
||||
|
||||
return eps
|
||||
|
||||
def ips_windows_ipconfig(self) -> tuple[dict[str, str], set[str]]:
|
||||
eps: dict[str, str] = {}
|
||||
offs: set[str] = set()
|
||||
try:
|
||||
txt, _ = chkcmd(["ipconfig"])
|
||||
except:
|
||||
return eps, offs
|
||||
|
||||
rdev = re.compile(r"(^[^ ].*):$")
|
||||
rip = re.compile(r"^ +IPv?4? [^:]+: *([0-9\.]{7,15})$")
|
||||
roff = re.compile(r".*: Media disconnected$")
|
||||
dev = None
|
||||
for ln in txt.replace("\r", "").split("\n"):
|
||||
m = rdev.match(ln)
|
||||
if m:
|
||||
if dev and dev not in eps.values():
|
||||
offs.add(dev)
|
||||
|
||||
dev = m.group(1).split(" adapter ", 1)[-1]
|
||||
|
||||
if dev and roff.match(ln):
|
||||
offs.add(dev)
|
||||
dev = None
|
||||
|
||||
m = rip.match(ln)
|
||||
if m and dev:
|
||||
eps[m.group(1)] = dev
|
||||
dev = None
|
||||
|
||||
if dev and dev not in eps.values():
|
||||
offs.add(dev)
|
||||
|
||||
return eps, offs
|
||||
|
||||
def ips_windows_netsh(self) -> dict[str, str]:
|
||||
eps: dict[str, str] = {}
|
||||
try:
|
||||
txt, _ = chkcmd("netsh interface ip show address".split())
|
||||
except:
|
||||
return eps
|
||||
|
||||
rdev = re.compile(r'.* "([^"]+)"$')
|
||||
rip = re.compile(r".* IP\b.*: +([0-9\.]{7,15})$")
|
||||
dev = None
|
||||
for ln in txt.replace("\r", "").split("\n"):
|
||||
m = rdev.match(ln)
|
||||
if m:
|
||||
dev = m.group(1)
|
||||
|
||||
m = rip.match(ln)
|
||||
if m and dev:
|
||||
eps[m.group(1)] = dev
|
||||
|
||||
return eps
|
||||
|
||||
def detect_interfaces(self, listen_ips: list[str]) -> dict[str, str]:
|
||||
if MACOS:
|
||||
eps = self.ips_macos()
|
||||
elif ANYWIN:
|
||||
eps, off = self.ips_windows_ipconfig() # sees more interfaces + link state
|
||||
eps.update(self.ips_windows_netsh()) # has better names
|
||||
for k, v in eps.items():
|
||||
if v in off:
|
||||
eps[k] += ", \033[31mLINK-DOWN"
|
||||
else:
|
||||
eps = self.ips_linux()
|
||||
|
||||
if "0.0.0.0" not in listen_ips:
|
||||
eps = {k: v for k, v in eps.items() if k in listen_ips}
|
||||
|
||||
default_route = None
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
for ip in [
|
||||
@@ -304,6 +147,7 @@ class TcpSrv(object):
|
||||
]:
|
||||
try:
|
||||
s.connect((ip, 1))
|
||||
# raise OSError(13, "a")
|
||||
default_route = s.getsockname()[0]
|
||||
break
|
||||
except (OSError, socket.error) as ex:
|
||||
@@ -323,26 +167,3 @@ class TcpSrv(object):
|
||||
eps[default_route] = desc
|
||||
|
||||
return eps
|
||||
|
||||
def _set_wintitle(self, vs: dict[str, dict[str, int]]) -> None:
|
||||
vs["all"] = vs.get("all", {"Local-Only": 1})
|
||||
vs["pub"] = vs.get("pub", vs["all"])
|
||||
|
||||
vs2 = {}
|
||||
for k, eps in vs.items():
|
||||
vs2[k] = {
|
||||
ep: 1
|
||||
for ep in eps.keys()
|
||||
if ":" not in ep or ep.split(":")[0] not in eps
|
||||
}
|
||||
|
||||
title = ""
|
||||
vs = vs2
|
||||
for p in self.args.wintitle.split(" "):
|
||||
if p.startswith("$"):
|
||||
p = " and ".join(sorted(vs.get(p[1:], {"(None)": 1}).keys()))
|
||||
|
||||
title += "{} ".format(p)
|
||||
|
||||
print("\033]0;{}\033\\".format(title), file=sys.stderr, end="")
|
||||
sys.stderr.flush()
|
||||
|
||||
@@ -3,133 +3,53 @@ from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
|
||||
from .__init__ import TYPE_CHECKING
|
||||
from .authsrv import VFS
|
||||
from .bos import bos
|
||||
from .th_srv import HAVE_WEBP, thumb_path
|
||||
from .util import Cooldown
|
||||
|
||||
try:
|
||||
from typing import Optional, Union
|
||||
except:
|
||||
pass
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .httpsrv import HttpSrv
|
||||
from .th_srv import thumb_path, THUMBABLE, FMT_FF
|
||||
|
||||
|
||||
class ThumbCli(object):
|
||||
def __init__(self, hsrv: "HttpSrv") -> None:
|
||||
self.broker = hsrv.broker
|
||||
self.log_func = hsrv.log
|
||||
self.args = hsrv.args
|
||||
self.asrv = hsrv.asrv
|
||||
def __init__(self, broker):
|
||||
self.broker = broker
|
||||
self.args = broker.args
|
||||
self.asrv = broker.asrv
|
||||
|
||||
# cache on both sides for less broker spam
|
||||
self.cooldown = Cooldown(self.args.th_poke)
|
||||
|
||||
try:
|
||||
c = hsrv.th_cfg
|
||||
except:
|
||||
c = {k: {} for k in ["thumbable", "pil", "vips", "ffi", "ffv", "ffa"]}
|
||||
|
||||
self.thumbable = c["thumbable"]
|
||||
self.fmt_pil = c["pil"]
|
||||
self.fmt_vips = c["vips"]
|
||||
self.fmt_ffi = c["ffi"]
|
||||
self.fmt_ffv = c["ffv"]
|
||||
self.fmt_ffa = c["ffa"]
|
||||
|
||||
# defer args.th_ff_jpg, can change at runtime
|
||||
d = next((x for x in self.args.th_dec if x in ("vips", "pil")), None)
|
||||
self.can_webp = HAVE_WEBP or d == "vips"
|
||||
|
||||
def log(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
self.log_func("thumbcli", msg, c)
|
||||
|
||||
def get(self, dbv: VFS, rem: str, mtime: float, fmt: str) -> Optional[str]:
|
||||
ptop = dbv.realpath
|
||||
def get(self, ptop, rem, mtime, fmt):
|
||||
ext = rem.rsplit(".")[-1].lower()
|
||||
if ext not in self.thumbable or "dthumb" in dbv.flags:
|
||||
if ext not in THUMBABLE:
|
||||
return None
|
||||
|
||||
is_vid = ext in self.fmt_ffv
|
||||
if is_vid and "dvthumb" in dbv.flags:
|
||||
is_vid = ext in FMT_FF
|
||||
if is_vid and self.args.no_vthumb:
|
||||
return None
|
||||
|
||||
want_opus = fmt in ("opus", "caf")
|
||||
is_au = ext in self.fmt_ffa
|
||||
if is_au:
|
||||
if want_opus:
|
||||
if self.args.no_acode:
|
||||
return None
|
||||
else:
|
||||
if "dathumb" in dbv.flags:
|
||||
return None
|
||||
elif want_opus:
|
||||
return None
|
||||
|
||||
is_img = not is_vid and not is_au
|
||||
if is_img and "dithumb" in dbv.flags:
|
||||
return None
|
||||
|
||||
preferred = self.args.th_dec[0] if self.args.th_dec else ""
|
||||
|
||||
if rem.startswith(".hist/th/") and rem.split(".")[-1] in ["webp", "jpg", "png"]:
|
||||
return os.path.join(ptop, rem)
|
||||
|
||||
if fmt == "j" and self.args.th_no_jpg:
|
||||
fmt = "w"
|
||||
|
||||
if fmt == "w":
|
||||
if (
|
||||
self.args.th_no_webp
|
||||
or (is_img and not self.can_webp)
|
||||
or (self.args.th_ff_jpg and (not is_img or preferred == "ff"))
|
||||
):
|
||||
if self.args.th_no_webp or (is_vid and self.args.th_ff_jpg):
|
||||
fmt = "j"
|
||||
|
||||
histpath = self.asrv.vfs.histtab.get(ptop)
|
||||
if not histpath:
|
||||
self.log("no histpath for [{}]".format(ptop))
|
||||
return None
|
||||
|
||||
histpath = self.asrv.vfs.histtab[ptop]
|
||||
tpath = thumb_path(histpath, rem, mtime, fmt)
|
||||
tpaths = [tpath]
|
||||
if fmt == "w":
|
||||
# also check for jpg (maybe webp is unavailable)
|
||||
tpaths.append(tpath.rsplit(".", 1)[0] + ".jpg")
|
||||
|
||||
ret = None
|
||||
abort = False
|
||||
for tp in tpaths:
|
||||
try:
|
||||
st = bos.stat(tp)
|
||||
st = os.stat(tpath)
|
||||
if st.st_size:
|
||||
ret = tpath = tp
|
||||
fmt = ret.rsplit(".")[1]
|
||||
ret = tpath
|
||||
else:
|
||||
abort = True
|
||||
return None
|
||||
except:
|
||||
pass
|
||||
|
||||
if ret:
|
||||
tdir = os.path.dirname(tpath)
|
||||
if self.cooldown.poke(tdir):
|
||||
self.broker.say("thumbsrv.poke", tdir)
|
||||
|
||||
if want_opus:
|
||||
# audio files expire individually
|
||||
if self.cooldown.poke(tpath):
|
||||
self.broker.say("thumbsrv.poke", tpath)
|
||||
self.broker.put(False, "thumbsrv.poke", tdir)
|
||||
|
||||
return ret
|
||||
|
||||
if abort:
|
||||
return None
|
||||
|
||||
if not bos.path.getsize(os.path.join(ptop, rem)):
|
||||
return None
|
||||
|
||||
x = self.broker.ask("thumbsrv.get", ptop, rem, mtime, fmt)
|
||||
return x.get() # type: ignore
|
||||
x = self.broker.put(True, "thumbsrv.get", ptop, rem, mtime, fmt)
|
||||
return x.get()
|
||||
|
||||
@@ -1,28 +1,22 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import time
|
||||
import shutil
|
||||
import base64
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import subprocess as sp
|
||||
import threading
|
||||
import time
|
||||
import subprocess as sp
|
||||
|
||||
from queue import Queue
|
||||
|
||||
from .__init__ import TYPE_CHECKING
|
||||
from .bos import bos
|
||||
from .__init__ import PY2
|
||||
from .util import fsenc, runcmd, Queue, Cooldown, BytesIO, min_ex
|
||||
from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
|
||||
from .util import BytesIO, Cooldown, Pebkac, fsenc, min_ex, runcmd, statdir, vsplit
|
||||
|
||||
try:
|
||||
from typing import Optional, Union
|
||||
except:
|
||||
pass
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .svchub import SvcHub
|
||||
if not PY2:
|
||||
unicode = str
|
||||
|
||||
|
||||
HAVE_PIL = False
|
||||
HAVE_HEIF = False
|
||||
@@ -30,7 +24,7 @@ HAVE_AVIF = False
|
||||
HAVE_WEBP = False
|
||||
|
||||
try:
|
||||
from PIL import ExifTags, Image, ImageOps
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
HAVE_PIL = True
|
||||
try:
|
||||
@@ -48,7 +42,7 @@ try:
|
||||
pass
|
||||
|
||||
try:
|
||||
import pillow_avif # noqa: F401 # pylint: disable=unused-import
|
||||
import pillow_avif
|
||||
|
||||
HAVE_AVIF = True
|
||||
except:
|
||||
@@ -56,18 +50,39 @@ try:
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
HAVE_VIPS = True
|
||||
import pyvips
|
||||
except:
|
||||
HAVE_VIPS = False
|
||||
# https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
|
||||
# ffmpeg -formats
|
||||
FMT_PIL = "bmp dib gif icns ico jpg jpeg jp2 jpx pcx png pbm pgm ppm pnm sgi tga tif tiff webp xbm dds xpm"
|
||||
FMT_FF = "av1 asf avi flv m4v mkv mjpeg mjpg mpg mpeg mpg2 mpeg2 h264 avc h265 hevc mov 3gp mp4 ts mpegts nut ogv ogm rm vob webm wmv"
|
||||
|
||||
if HAVE_HEIF:
|
||||
FMT_PIL += " heif heifs heic heics"
|
||||
|
||||
if HAVE_AVIF:
|
||||
FMT_PIL += " avif avifs"
|
||||
|
||||
FMT_PIL, FMT_FF = [{x: True for x in y.split(" ") if x} for y in [FMT_PIL, FMT_FF]]
|
||||
|
||||
|
||||
def thumb_path(histpath: str, rem: str, mtime: float, fmt: str) -> str:
|
||||
THUMBABLE = {}
|
||||
|
||||
if HAVE_PIL:
|
||||
THUMBABLE.update(FMT_PIL)
|
||||
|
||||
if HAVE_FFMPEG and HAVE_FFPROBE:
|
||||
THUMBABLE.update(FMT_FF)
|
||||
|
||||
|
||||
def thumb_path(histpath, rem, mtime, fmt):
|
||||
# base16 = 16 = 256
|
||||
# b64-lc = 38 = 1444
|
||||
# base64 = 64 = 4096
|
||||
rd, fn = vsplit(rem)
|
||||
try:
|
||||
rd, fn = rem.rsplit("/", 1)
|
||||
except:
|
||||
rd = ""
|
||||
fn = rem
|
||||
|
||||
if rd:
|
||||
h = hashlib.sha512(fsenc(rd)).digest()
|
||||
b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24]
|
||||
@@ -79,17 +94,13 @@ def thumb_path(histpath: str, rem: str, mtime: float, fmt: str) -> str:
|
||||
h = hashlib.sha512(fsenc(fn)).digest()
|
||||
fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
|
||||
|
||||
if fmt in ("opus", "caf"):
|
||||
cat = "ac"
|
||||
else:
|
||||
fmt = "webp" if fmt == "w" else "png" if fmt == "p" else "jpg"
|
||||
cat = "th"
|
||||
|
||||
return "{}/{}/{}/{}.{:x}.{}".format(histpath, cat, rd, fn, int(mtime), fmt)
|
||||
return "{}/th/{}/{}.{:x}.{}".format(
|
||||
histpath, rd, fn, int(mtime), "webp" if fmt == "w" else "jpg"
|
||||
)
|
||||
|
||||
|
||||
class ThumbSrv(object):
|
||||
def __init__(self, hub: "SvcHub") -> None:
|
||||
def __init__(self, hub):
|
||||
self.hub = hub
|
||||
self.asrv = hub.asrv
|
||||
self.args = hub.args
|
||||
@@ -100,85 +111,47 @@ class ThumbSrv(object):
|
||||
self.poke_cd = Cooldown(self.args.th_poke)
|
||||
|
||||
self.mutex = threading.Lock()
|
||||
self.busy: dict[str, list[threading.Condition]] = {}
|
||||
self.busy = {}
|
||||
self.stopping = False
|
||||
self.nthr = max(1, self.args.th_mt)
|
||||
|
||||
self.q: Queue[Optional[tuple[str, str]]] = Queue(self.nthr * 4)
|
||||
self.nthr = os.cpu_count() if hasattr(os, "cpu_count") else 4
|
||||
self.q = Queue(self.nthr * 4)
|
||||
for n in range(self.nthr):
|
||||
thr = threading.Thread(
|
||||
t = threading.Thread(
|
||||
target=self.worker, name="thumb-{}-{}".format(n, self.nthr)
|
||||
)
|
||||
thr.daemon = True
|
||||
thr.start()
|
||||
|
||||
want_ff = not self.args.no_vthumb or not self.args.no_athumb
|
||||
if want_ff and (not HAVE_FFMPEG or not HAVE_FFPROBE):
|
||||
missing = []
|
||||
if not HAVE_FFMPEG:
|
||||
missing.append("FFmpeg")
|
||||
|
||||
if not HAVE_FFPROBE:
|
||||
missing.append("FFprobe")
|
||||
|
||||
msg = "cannot create audio/video thumbnails because some of the required programs are not available: "
|
||||
msg += ", ".join(missing)
|
||||
self.log(msg, c=3)
|
||||
|
||||
if self.args.th_clean:
|
||||
t = threading.Thread(target=self.cleaner, name="thumb.cln")
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
self.fmt_pil, self.fmt_vips, self.fmt_ffi, self.fmt_ffv, self.fmt_ffa = [
|
||||
set(y.split(","))
|
||||
for y in [
|
||||
self.args.th_r_pil,
|
||||
self.args.th_r_vips,
|
||||
self.args.th_r_ffi,
|
||||
self.args.th_r_ffv,
|
||||
self.args.th_r_ffa,
|
||||
]
|
||||
]
|
||||
if not self.args.no_vthumb and (not HAVE_FFMPEG or not HAVE_FFPROBE):
|
||||
missing = []
|
||||
if not HAVE_FFMPEG:
|
||||
missing.append("ffmpeg")
|
||||
|
||||
if not HAVE_HEIF:
|
||||
for f in "heif heifs heic heics".split(" "):
|
||||
self.fmt_pil.discard(f)
|
||||
if not HAVE_FFPROBE:
|
||||
missing.append("ffprobe")
|
||||
|
||||
if not HAVE_AVIF:
|
||||
for f in "avif avifs".split(" "):
|
||||
self.fmt_pil.discard(f)
|
||||
msg = "cannot create video thumbnails because some of the required programs are not available: "
|
||||
msg += ", ".join(missing)
|
||||
self.log(msg, c=3)
|
||||
|
||||
self.thumbable: set[str] = set()
|
||||
t = threading.Thread(target=self.cleaner, name="thumb-cleaner")
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
if "pil" in self.args.th_dec:
|
||||
self.thumbable |= self.fmt_pil
|
||||
|
||||
if "vips" in self.args.th_dec:
|
||||
self.thumbable |= self.fmt_vips
|
||||
|
||||
if "ff" in self.args.th_dec:
|
||||
for zss in [self.fmt_ffi, self.fmt_ffv, self.fmt_ffa]:
|
||||
self.thumbable |= zss
|
||||
|
||||
def log(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
def log(self, msg, c=0):
|
||||
self.log_func("thumb", msg, c)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
def shutdown(self):
|
||||
self.stopping = True
|
||||
for _ in range(self.nthr):
|
||||
self.q.put(None)
|
||||
|
||||
def stopped(self) -> bool:
|
||||
def stopped(self):
|
||||
with self.mutex:
|
||||
return not self.nthr
|
||||
|
||||
def get(self, ptop: str, rem: str, mtime: float, fmt: str) -> Optional[str]:
|
||||
histpath = self.asrv.vfs.histtab.get(ptop)
|
||||
if not histpath:
|
||||
self.log("no histpath for [{}]".format(ptop))
|
||||
return None
|
||||
|
||||
def get(self, ptop, rem, mtime, fmt):
|
||||
histpath = self.asrv.vfs.histtab[ptop]
|
||||
tpath = thumb_path(histpath, rem, mtime, fmt)
|
||||
abspath = os.path.join(ptop, rem)
|
||||
cond = threading.Condition(self.mutex)
|
||||
@@ -189,10 +162,13 @@ class ThumbSrv(object):
|
||||
self.log("wait {}".format(tpath))
|
||||
except:
|
||||
thdir = os.path.dirname(tpath)
|
||||
bos.makedirs(thdir)
|
||||
try:
|
||||
os.makedirs(thdir)
|
||||
except:
|
||||
pass
|
||||
|
||||
inf_path = os.path.join(thdir, "dir.txt")
|
||||
if not bos.path.exists(inf_path):
|
||||
if not os.path.exists(inf_path):
|
||||
with open(inf_path, "wb") as f:
|
||||
f.write(fsenc(os.path.dirname(abspath)))
|
||||
|
||||
@@ -200,7 +176,7 @@ class ThumbSrv(object):
|
||||
do_conv = True
|
||||
|
||||
if do_conv:
|
||||
self.q.put((abspath, tpath))
|
||||
self.q.put([abspath, tpath])
|
||||
self.log("conv {} \033[0m{}".format(tpath, abspath), c=6)
|
||||
|
||||
while not self.stopping:
|
||||
@@ -212,26 +188,15 @@ class ThumbSrv(object):
|
||||
cond.wait(3)
|
||||
|
||||
try:
|
||||
st = bos.stat(tpath)
|
||||
st = os.stat(tpath)
|
||||
if st.st_size:
|
||||
self.poke(tpath)
|
||||
return tpath
|
||||
except:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def getcfg(self) -> dict[str, set[str]]:
|
||||
return {
|
||||
"thumbable": self.thumbable,
|
||||
"pil": self.fmt_pil,
|
||||
"vips": self.fmt_vips,
|
||||
"ffi": self.fmt_ffi,
|
||||
"ffv": self.fmt_ffv,
|
||||
"ffa": self.fmt_ffa,
|
||||
}
|
||||
|
||||
def worker(self) -> None:
|
||||
def worker(self):
|
||||
while not self.stopping:
|
||||
task = self.q.get()
|
||||
if not task:
|
||||
@@ -239,46 +204,20 @@ class ThumbSrv(object):
|
||||
|
||||
abspath, tpath = task
|
||||
ext = abspath.split(".")[-1].lower()
|
||||
png_ok = False
|
||||
fun = None
|
||||
if not bos.path.exists(tpath):
|
||||
for lib in self.args.th_dec:
|
||||
if fun:
|
||||
break
|
||||
elif lib == "pil" and ext in self.fmt_pil:
|
||||
if not os.path.exists(tpath):
|
||||
if ext in FMT_PIL:
|
||||
fun = self.conv_pil
|
||||
elif lib == "vips" and ext in self.fmt_vips:
|
||||
fun = self.conv_vips
|
||||
elif lib == "ff" and ext in self.fmt_ffi or ext in self.fmt_ffv:
|
||||
elif ext in FMT_FF:
|
||||
fun = self.conv_ffmpeg
|
||||
elif lib == "ff" and ext in self.fmt_ffa:
|
||||
if tpath.endswith(".opus") or tpath.endswith(".caf"):
|
||||
fun = self.conv_opus
|
||||
elif tpath.endswith(".png"):
|
||||
fun = self.conv_waves
|
||||
png_ok = True
|
||||
else:
|
||||
fun = self.conv_spec
|
||||
|
||||
if not png_ok and tpath.endswith(".png"):
|
||||
raise Pebkac(400, "png only allowed for waveforms")
|
||||
|
||||
if fun:
|
||||
try:
|
||||
fun(abspath, tpath)
|
||||
except Exception as ex:
|
||||
msg = "{} could not create thumbnail of {}\n{}"
|
||||
msg = msg.format(fun.__name__, abspath, min_ex())
|
||||
c: Union[str, int] = 1 if "<Signals.SIG" in msg else "1;30"
|
||||
self.log(msg, c)
|
||||
if getattr(ex, "returncode", 0) != 321:
|
||||
with open(tpath, "wb") as _:
|
||||
pass
|
||||
else:
|
||||
# ffmpeg may spawn empty files on windows
|
||||
try:
|
||||
os.unlink(tpath)
|
||||
except:
|
||||
msg = "{} failed on {}\n{}"
|
||||
self.log(msg.format(fun.__name__, abspath, min_ex()), 3)
|
||||
with open(tpath, "wb") as _:
|
||||
pass
|
||||
|
||||
with self.mutex:
|
||||
@@ -292,38 +231,21 @@ class ThumbSrv(object):
|
||||
with self.mutex:
|
||||
self.nthr -= 1
|
||||
|
||||
def fancy_pillow(self, im: "Image.Image") -> "Image.Image":
|
||||
# exif_transpose is expensive (loads full image + unconditional copy)
|
||||
r = max(*self.res) * 2
|
||||
im.thumbnail((r, r), resample=Image.LANCZOS)
|
||||
def conv_pil(self, abspath, tpath):
|
||||
with Image.open(fsenc(abspath)) as im:
|
||||
crop = not self.args.th_no_crop
|
||||
res2 = self.res
|
||||
if crop:
|
||||
res2 = (res2[0] * 2, res2[1] * 2)
|
||||
|
||||
try:
|
||||
k = next(k for k, v in ExifTags.TAGS.items() if v == "Orientation")
|
||||
exif = im.getexif()
|
||||
rot = int(exif[k])
|
||||
del exif[k]
|
||||
except:
|
||||
rot = 1
|
||||
|
||||
rots = {8: Image.ROTATE_90, 3: Image.ROTATE_180, 6: Image.ROTATE_270}
|
||||
if rot in rots:
|
||||
im = im.transpose(rots[rot])
|
||||
|
||||
if self.args.th_no_crop:
|
||||
im.thumbnail(self.res, resample=Image.LANCZOS)
|
||||
else:
|
||||
im.thumbnail(res2, resample=Image.LANCZOS)
|
||||
if crop:
|
||||
iw, ih = im.size
|
||||
dw, dh = self.res
|
||||
res = (min(iw, dw), min(ih, dh))
|
||||
im = ImageOps.fit(im, res, method=Image.LANCZOS)
|
||||
|
||||
return im
|
||||
|
||||
def conv_pil(self, abspath: str, tpath: str) -> None:
|
||||
with Image.open(fsenc(abspath)) as im:
|
||||
try:
|
||||
im = self.fancy_pillow(im)
|
||||
except Exception as ex:
|
||||
self.log("fancy_pillow {}".format(ex), "1;30")
|
||||
except:
|
||||
im.thumbnail(self.res)
|
||||
|
||||
fmts = ["RGB", "L"]
|
||||
@@ -338,44 +260,24 @@ class ThumbSrv(object):
|
||||
fmts += ["RGBA", "LA"]
|
||||
args["method"] = 6
|
||||
else:
|
||||
# default q = 75
|
||||
args["progressive"] = True
|
||||
pass # default q = 75
|
||||
|
||||
if im.mode not in fmts:
|
||||
# print("conv {}".format(im.mode))
|
||||
print("conv {}".format(im.mode))
|
||||
im = im.convert("RGB")
|
||||
|
||||
im.save(tpath, **args)
|
||||
im.save(tpath, quality=40, method=6)
|
||||
|
||||
def conv_vips(self, abspath: str, tpath: str) -> None:
|
||||
crops = ["centre", "none"]
|
||||
if self.args.th_no_crop:
|
||||
crops = ["none"]
|
||||
def conv_ffmpeg(self, abspath, tpath):
|
||||
ret, _ = ffprobe(abspath)
|
||||
|
||||
w, h = self.res
|
||||
kw = {"height": h, "size": "down", "intent": "relative"}
|
||||
|
||||
for c in crops:
|
||||
try:
|
||||
kw["crop"] = c
|
||||
img = pyvips.Image.thumbnail(abspath, w, **kw)
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
||||
img.write_to_file(tpath, Q=40)
|
||||
|
||||
def conv_ffmpeg(self, abspath: str, tpath: str) -> None:
|
||||
ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
|
||||
if not ret:
|
||||
return
|
||||
|
||||
ext = abspath.rsplit(".")[-1].lower()
|
||||
if ext in ["h264", "h265"] or ext in self.fmt_ffi:
|
||||
seek: list[bytes] = []
|
||||
ext = abspath.rsplit(".")[-1]
|
||||
if ext in ["h264", "h265"]:
|
||||
seek = []
|
||||
else:
|
||||
dur = ret[".dur"][1] if ".dur" in ret else 4
|
||||
seek = [b"-ss", "{:.0f}".format(dur / 3).encode("utf-8")]
|
||||
seek = "{:.0f}".format(dur / 3)
|
||||
seek = [b"-ss", seek.encode("utf-8")]
|
||||
|
||||
scale = "scale={0}:{1}:force_original_aspect_ratio="
|
||||
if self.args.th_no_crop:
|
||||
@@ -383,7 +285,7 @@ class ThumbSrv(object):
|
||||
else:
|
||||
scale += "increase,crop={0}:{1},setsar=1:1"
|
||||
|
||||
bscale = scale.format(*list(self.res)).encode("utf-8")
|
||||
scale = scale.format(*list(self.res)).encode("utf-8")
|
||||
# fmt: off
|
||||
cmd = [
|
||||
b"ffmpeg",
|
||||
@@ -394,10 +296,8 @@ class ThumbSrv(object):
|
||||
cmd += seek
|
||||
cmd += [
|
||||
b"-i", fsenc(abspath),
|
||||
b"-map", b"0:v:0",
|
||||
b"-vf", bscale,
|
||||
b"-frames:v", b"1",
|
||||
b"-metadata:s:v:0", b"rotate=0",
|
||||
b"-vf", scale,
|
||||
b"-vframes", b"1",
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
@@ -415,185 +315,27 @@ class ThumbSrv(object):
|
||||
]
|
||||
|
||||
cmd += [fsenc(tpath)]
|
||||
self._run_ff(cmd)
|
||||
|
||||
def _run_ff(self, cmd: list[bytes]) -> None:
|
||||
# self.log((b" ".join(cmd)).decode("utf-8"))
|
||||
ret, _, serr = runcmd(cmd, timeout=self.args.th_convt)
|
||||
if not ret:
|
||||
return
|
||||
|
||||
c: Union[str, int] = "1;30"
|
||||
t = "FFmpeg failed (probably a corrupt video file):\n"
|
||||
if (
|
||||
(not self.args.th_ff_jpg or time.time() - int(self.args.th_ff_jpg) < 60)
|
||||
and cmd[-1].lower().endswith(b".webp")
|
||||
and (
|
||||
"Error selecting an encoder" in serr
|
||||
or "Automatic encoder selection failed" in serr
|
||||
or "Default encoder for format webp" in serr
|
||||
or "Please choose an encoder manually" in serr
|
||||
)
|
||||
):
|
||||
self.args.th_ff_jpg = time.time()
|
||||
t = "FFmpeg failed because it was compiled without libwebp; enabling --th-ff-jpg to force jpeg output:\n"
|
||||
ret = 321
|
||||
c = 1
|
||||
|
||||
if (
|
||||
not self.args.th_ff_swr or time.time() - int(self.args.th_ff_swr) < 60
|
||||
) and (
|
||||
"Requested resampling engine is unavailable" in serr
|
||||
or "output pad on Parsed_aresample_" in serr
|
||||
):
|
||||
self.args.th_ff_swr = time.time()
|
||||
t = "FFmpeg failed because it was compiled without libsox; enabling --th-ff-swr to force swr resampling:\n"
|
||||
ret = 321
|
||||
c = 1
|
||||
|
||||
lines = serr.strip("\n").split("\n")
|
||||
if len(lines) > 50:
|
||||
lines = lines[:25] + ["[...]"] + lines[-25:]
|
||||
|
||||
txt = "\n".join(["ff: " + str(x) for x in lines])
|
||||
if len(txt) > 5000:
|
||||
txt = txt[:2500] + "...\nff: [...]\nff: ..." + txt[-2500:]
|
||||
|
||||
self.log(t + txt, c=c)
|
||||
ret, sout, serr = runcmd(*cmd)
|
||||
if ret != 0:
|
||||
msg = ["ff: {}".format(x) for x in serr.split("\n")]
|
||||
self.log("FFmpeg failed:\n" + "\n".join(msg), c="1;30")
|
||||
raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
|
||||
|
||||
def conv_waves(self, abspath: str, tpath: str) -> None:
|
||||
ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
|
||||
if "ac" not in ret:
|
||||
raise Exception("not audio")
|
||||
|
||||
flt = (
|
||||
b"[0:a:0]"
|
||||
b"compand=.3|.3:1|1:-90/-60|-60/-40|-40/-30|-20/-20:6:0:-90:0.2"
|
||||
b",volume=2"
|
||||
b",showwavespic=s=2048x64:colors=white"
|
||||
b",convolution=1 1 1 1 1 1 1 1 1:1 1 1 1 1 1 1 1 1:1 1 1 1 1 1 1 1 1:1 -1 1 -1 5 -1 1 -1 1" # idk what im doing but it looks ok
|
||||
)
|
||||
|
||||
# fmt: off
|
||||
cmd = [
|
||||
b"ffmpeg",
|
||||
b"-nostdin",
|
||||
b"-v", b"error",
|
||||
b"-hide_banner",
|
||||
b"-i", fsenc(abspath),
|
||||
b"-filter_complex", flt,
|
||||
b"-frames:v", b"1",
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
cmd += [fsenc(tpath)]
|
||||
self._run_ff(cmd)
|
||||
|
||||
def conv_spec(self, abspath: str, tpath: str) -> None:
|
||||
ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
|
||||
if "ac" not in ret:
|
||||
raise Exception("not audio")
|
||||
|
||||
fc = "[0:a:0]aresample=48000{},showspectrumpic=s=640x512,crop=780:544:70:50[o]"
|
||||
|
||||
if self.args.th_ff_swr:
|
||||
fco = ":filter_size=128:cutoff=0.877"
|
||||
else:
|
||||
fco = ":resampler=soxr"
|
||||
|
||||
fc = fc.format(fco)
|
||||
|
||||
# fmt: off
|
||||
cmd = [
|
||||
b"ffmpeg",
|
||||
b"-nostdin",
|
||||
b"-v", b"error",
|
||||
b"-hide_banner",
|
||||
b"-i", fsenc(abspath),
|
||||
b"-filter_complex", fc.encode("utf-8"),
|
||||
b"-map", b"[o]",
|
||||
b"-frames:v", b"1",
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
if tpath.endswith(".jpg"):
|
||||
cmd += [
|
||||
b"-q:v",
|
||||
b"6", # default=??
|
||||
]
|
||||
else:
|
||||
cmd += [
|
||||
b"-q:v",
|
||||
b"50", # default=75
|
||||
b"-compression_level:v",
|
||||
b"6", # default=4, 0=fast, 6=max
|
||||
]
|
||||
|
||||
cmd += [fsenc(tpath)]
|
||||
self._run_ff(cmd)
|
||||
|
||||
def conv_opus(self, abspath: str, tpath: str) -> None:
|
||||
if self.args.no_acode:
|
||||
raise Exception("disabled in server config")
|
||||
|
||||
ret, _ = ffprobe(abspath, int(self.args.th_convt / 2))
|
||||
if "ac" not in ret:
|
||||
raise Exception("not audio")
|
||||
|
||||
src_opus = abspath.lower().endswith(".opus") or ret["ac"][1] == "opus"
|
||||
want_caf = tpath.endswith(".caf")
|
||||
tmp_opus = tpath
|
||||
if want_caf:
|
||||
tmp_opus = tpath.rsplit(".", 1)[0] + ".opus"
|
||||
|
||||
if not want_caf or (not src_opus and not bos.path.isfile(tmp_opus)):
|
||||
# fmt: off
|
||||
cmd = [
|
||||
b"ffmpeg",
|
||||
b"-nostdin",
|
||||
b"-v", b"error",
|
||||
b"-hide_banner",
|
||||
b"-i", fsenc(abspath),
|
||||
b"-map_metadata", b"-1",
|
||||
b"-map", b"0:a:0",
|
||||
b"-c:a", b"libopus",
|
||||
b"-b:a", b"128k",
|
||||
fsenc(tmp_opus)
|
||||
]
|
||||
# fmt: on
|
||||
self._run_ff(cmd)
|
||||
|
||||
if want_caf:
|
||||
# fmt: off
|
||||
cmd = [
|
||||
b"ffmpeg",
|
||||
b"-nostdin",
|
||||
b"-v", b"error",
|
||||
b"-hide_banner",
|
||||
b"-i", fsenc(abspath if src_opus else tmp_opus),
|
||||
b"-map_metadata", b"-1",
|
||||
b"-map", b"0:a:0",
|
||||
b"-c:a", b"copy",
|
||||
b"-f", b"caf",
|
||||
fsenc(tpath)
|
||||
]
|
||||
# fmt: on
|
||||
self._run_ff(cmd)
|
||||
|
||||
def poke(self, tdir: str) -> None:
|
||||
def poke(self, tdir):
|
||||
if not self.poke_cd.poke(tdir):
|
||||
return
|
||||
|
||||
ts = int(time.time())
|
||||
try:
|
||||
for _ in range(4):
|
||||
bos.utime(tdir, (ts, ts))
|
||||
tdir = os.path.dirname(tdir)
|
||||
p1 = os.path.dirname(tdir)
|
||||
p2 = os.path.dirname(p1)
|
||||
for dp in [tdir, p1, p2]:
|
||||
os.utime(fsenc(dp), (ts, ts))
|
||||
except:
|
||||
pass
|
||||
|
||||
def cleaner(self) -> None:
|
||||
def cleaner(self):
|
||||
interval = self.args.th_clean
|
||||
while True:
|
||||
time.sleep(interval)
|
||||
@@ -608,42 +350,30 @@ class ThumbSrv(object):
|
||||
|
||||
self.log("\033[Jcln ok; rm {} dirs".format(ndirs))
|
||||
|
||||
def clean(self, histpath: str) -> int:
|
||||
ret = 0
|
||||
for cat in ["th", "ac"]:
|
||||
top = os.path.join(histpath, cat)
|
||||
if not bos.path.isdir(top):
|
||||
continue
|
||||
|
||||
ret += self._clean(cat, top)
|
||||
|
||||
return ret
|
||||
|
||||
def _clean(self, cat: str, thumbpath: str) -> int:
|
||||
def clean(self, histpath):
|
||||
thumbpath = os.path.join(histpath, "th")
|
||||
# self.log("cln {}".format(thumbpath))
|
||||
exts = ["jpg", "webp"] if cat == "th" else ["opus", "caf"]
|
||||
maxage = getattr(self.args, cat + "_maxage")
|
||||
maxage = self.args.th_maxage
|
||||
now = time.time()
|
||||
prev_b64 = None
|
||||
prev_fp = ""
|
||||
prev_fp = None
|
||||
try:
|
||||
t1 = statdir(self.log_func, not self.args.no_scandir, False, thumbpath)
|
||||
ents = sorted(list(t1))
|
||||
ents = os.listdir(thumbpath)
|
||||
except:
|
||||
return 0
|
||||
|
||||
ndirs = 0
|
||||
for f, inf in ents:
|
||||
for f in sorted(ents):
|
||||
fp = os.path.join(thumbpath, f)
|
||||
cmp = fp.lower().replace("\\", "/")
|
||||
|
||||
# "top" or b64 prefix/full (a folder)
|
||||
if len(f) <= 3 or len(f) == 24:
|
||||
age = now - inf.st_mtime
|
||||
age = now - os.path.getmtime(fp)
|
||||
if age > maxage:
|
||||
with self.mutex:
|
||||
safe = True
|
||||
for k in self.busy:
|
||||
for k in self.busy.keys():
|
||||
if k.lower().replace("\\", "/").startswith(cmp):
|
||||
safe = False
|
||||
break
|
||||
@@ -653,15 +383,16 @@ class ThumbSrv(object):
|
||||
self.log("rm -rf [{}]".format(fp))
|
||||
shutil.rmtree(fp, ignore_errors=True)
|
||||
else:
|
||||
ndirs += self._clean(cat, fp)
|
||||
|
||||
ndirs += self.clean(fp)
|
||||
continue
|
||||
|
||||
# thumb file
|
||||
try:
|
||||
b64, ts, ext = f.split(".")
|
||||
if len(b64) != 24 or len(ts) != 8 or ext not in exts:
|
||||
if len(b64) != 24 or len(ts) != 8 or ext not in ["jpg", "webp"]:
|
||||
raise Exception()
|
||||
|
||||
ts = int(ts, 16)
|
||||
except:
|
||||
if f != "dir.txt":
|
||||
self.log("foreign file in thumbs dir: [{}]".format(fp), 1)
|
||||
@@ -670,11 +401,7 @@ class ThumbSrv(object):
|
||||
|
||||
if b64 == prev_b64:
|
||||
self.log("rm replaced [{}]".format(fp))
|
||||
bos.unlink(prev_fp)
|
||||
|
||||
if cat != "th" and inf.st_mtime + maxage < now:
|
||||
self.log("rm expired [{}]".format(fp))
|
||||
bos.unlink(fp)
|
||||
os.unlink(prev_fp)
|
||||
|
||||
prev_b64 = b64
|
||||
prev_fp = fp
|
||||
|
||||
@@ -1,37 +1,25 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import calendar
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import os
|
||||
import time
|
||||
from operator import itemgetter
|
||||
import threading
|
||||
from datetime import datetime
|
||||
|
||||
from .__init__ import ANYWIN, TYPE_CHECKING, unicode
|
||||
from .bos import bos
|
||||
from .util import s3dec, Pebkac, min_ex
|
||||
from .up2k import up2k_wark_from_hashlist
|
||||
from .util import HAVE_SQLITE3, Pebkac, absreal, gen_filekey, min_ex, quotep, s3dec
|
||||
|
||||
if HAVE_SQLITE3:
|
||||
|
||||
try:
|
||||
HAVE_SQLITE3 = True
|
||||
import sqlite3
|
||||
|
||||
try:
|
||||
from pathlib import Path
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
from typing import Any, Optional, Union
|
||||
except:
|
||||
pass
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .httpconn import HttpConn
|
||||
HAVE_SQLITE3 = False
|
||||
|
||||
|
||||
class U2idx(object):
|
||||
def __init__(self, conn: "HttpConn") -> None:
|
||||
def __init__(self, conn):
|
||||
self.log_func = conn.log_func
|
||||
self.asrv = conn.asrv
|
||||
self.args = conn.args
|
||||
@@ -41,21 +29,17 @@ class U2idx(object):
|
||||
self.log("your python does not have sqlite3; searching will be disabled")
|
||||
return
|
||||
|
||||
self.active_id = ""
|
||||
self.active_cur: Optional["sqlite3.Cursor"] = None
|
||||
self.cur: dict[str, "sqlite3.Cursor"] = {}
|
||||
self.mem_cur = sqlite3.connect(":memory:").cursor()
|
||||
self.cur = {}
|
||||
self.mem_cur = sqlite3.connect(":memory:")
|
||||
self.mem_cur.execute(r"create table a (b text)")
|
||||
|
||||
self.p_end = 0.0
|
||||
self.p_dur = 0.0
|
||||
self.p_end = None
|
||||
self.p_dur = 0
|
||||
|
||||
def log(self, msg: str, c: Union[int, str] = 0) -> None:
|
||||
def log(self, msg, c=0):
|
||||
self.log_func("u2idx", msg, c)
|
||||
|
||||
def fsearch(
|
||||
self, vols: list[tuple[str, str, dict[str, Any]]], body: dict[str, Any]
|
||||
) -> list[dict[str, Any]]:
|
||||
def fsearch(self, vols, body):
|
||||
"""search by up2k hashlist"""
|
||||
if not HAVE_SQLITE3:
|
||||
return []
|
||||
@@ -64,15 +48,15 @@ class U2idx(object):
|
||||
fhash = body["hash"]
|
||||
wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
|
||||
|
||||
uq = "substr(w,1,16) = ? and w = ?"
|
||||
uv: list[Union[str, int]] = [wark[:16], wark]
|
||||
uq = "where substr(w,1,16) = ? and w = ?"
|
||||
uv = [wark[:16], wark]
|
||||
|
||||
try:
|
||||
return self.run_query(vols, uq, uv, True, False, 99999)[0]
|
||||
return self.run_query(vols, uq, uv)[0]
|
||||
except:
|
||||
raise Pebkac(500, min_ex())
|
||||
|
||||
def get_cur(self, ptop: str) -> Optional["sqlite3.Cursor"]:
|
||||
def get_cur(self, ptop):
|
||||
if not HAVE_SQLITE3:
|
||||
return None
|
||||
|
||||
@@ -80,55 +64,32 @@ class U2idx(object):
|
||||
if cur:
|
||||
return cur
|
||||
|
||||
histpath = self.asrv.vfs.histtab.get(ptop)
|
||||
if not histpath:
|
||||
self.log("no histpath for [{}]".format(ptop))
|
||||
return None
|
||||
|
||||
histpath = self.asrv.vfs.histtab[ptop]
|
||||
db_path = os.path.join(histpath, "up2k.db")
|
||||
if not bos.path.exists(db_path):
|
||||
if not os.path.exists(db_path):
|
||||
return None
|
||||
|
||||
cur = None
|
||||
if ANYWIN:
|
||||
uri = ""
|
||||
try:
|
||||
uri = "{}?mode=ro&nolock=1".format(Path(db_path).as_uri())
|
||||
cur = sqlite3.connect(uri, 2, uri=True).cursor()
|
||||
self.log("ro: {}".format(db_path))
|
||||
except:
|
||||
self.log("could not open read-only: {}\n{}".format(uri, min_ex()))
|
||||
|
||||
if not cur:
|
||||
# on windows, this steals the write-lock from up2k.deferred_init --
|
||||
# seen on win 10.0.17763.2686, py 3.10.4, sqlite 3.37.2
|
||||
cur = sqlite3.connect(db_path, 2).cursor()
|
||||
self.log("opened {}".format(db_path))
|
||||
|
||||
self.cur[ptop] = cur
|
||||
return cur
|
||||
|
||||
def search(
|
||||
self, vols: list[tuple[str, str, dict[str, Any]]], uq: str, lim: int
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
def search(self, vols, uq):
|
||||
"""search by query params"""
|
||||
if not HAVE_SQLITE3:
|
||||
return [], []
|
||||
return []
|
||||
|
||||
q = ""
|
||||
v: Union[str, int] = ""
|
||||
va: list[Union[str, int]] = []
|
||||
have_up = False # query has up.* operands
|
||||
have_mt = False
|
||||
va = []
|
||||
joins = ""
|
||||
is_key = True
|
||||
is_size = False
|
||||
is_date = False
|
||||
field_end = "" # closing parenthesis or whatever
|
||||
kw_key = ["(", ")", "and ", "or ", "not "]
|
||||
kw_val = ["==", "=", "!=", ">", ">=", "<", "<=", "like "]
|
||||
ptn_mt = re.compile(r"^\.?[a-z_-]+$")
|
||||
ptn_lc = re.compile(r" (mt\.v) ([=<!>]+) \? \) $")
|
||||
ptn_lcv = re.compile(r"[a-zA-Z]")
|
||||
ptn_mt = re.compile(r"^\.?[a-z]+$")
|
||||
mt_ctr = 0
|
||||
mt_keycmp = "substr(up.w,1,16)"
|
||||
mt_keycmp2 = None
|
||||
|
||||
while True:
|
||||
uq = uq.strip()
|
||||
@@ -147,47 +108,35 @@ class U2idx(object):
|
||||
if ok:
|
||||
continue
|
||||
|
||||
if uq.startswith('"'):
|
||||
v, uq = uq[1:].split('"', 1)
|
||||
while v.endswith("\\"):
|
||||
v2, uq = uq.split('"', 1)
|
||||
v = v[:-1] + '"' + v2
|
||||
uq = uq.strip()
|
||||
else:
|
||||
v, uq = (uq + " ").split(" ", 1)
|
||||
v = v.replace('\\"', '"')
|
||||
|
||||
if is_key:
|
||||
is_key = False
|
||||
|
||||
if v == "size":
|
||||
v = "up.sz"
|
||||
is_size = True
|
||||
have_up = True
|
||||
|
||||
elif v == "date":
|
||||
v = "up.mt"
|
||||
is_date = True
|
||||
have_up = True
|
||||
|
||||
elif v == "path":
|
||||
v = "trim(?||up.rd,'/')"
|
||||
va.append("\nrd")
|
||||
have_up = True
|
||||
v = "up.rd"
|
||||
|
||||
elif v == "name":
|
||||
v = "up.fn"
|
||||
have_up = True
|
||||
|
||||
elif v == "tags" or ptn_mt.match(v):
|
||||
have_mt = True
|
||||
field_end = ") "
|
||||
mt_ctr += 1
|
||||
mt_keycmp2 = "mt{}.w".format(mt_ctr)
|
||||
joins += "inner join mt mt{} on {} = {} ".format(
|
||||
mt_ctr, mt_keycmp, mt_keycmp2
|
||||
)
|
||||
mt_keycmp = mt_keycmp2
|
||||
if v == "tags":
|
||||
vq = "mt.v"
|
||||
v = "mt{0}.v".format(mt_ctr)
|
||||
else:
|
||||
vq = "+mt.k = '{}' and mt.v".format(v)
|
||||
|
||||
v = "exists(select 1 from mt where mt.w = mtw and " + vq
|
||||
v = "+mt{0}.k = '{1}' and mt{0}.v".format(mt_ctr, v)
|
||||
|
||||
else:
|
||||
raise Pebkac(400, "invalid key [" + v + "]")
|
||||
@@ -200,17 +149,18 @@ class U2idx(object):
|
||||
|
||||
if is_date:
|
||||
is_date = False
|
||||
v = re.sub(r"[tzTZ, ]+", " ", v).strip()
|
||||
v = v.upper().rstrip("Z").replace(",", " ").replace("T", " ")
|
||||
while " " in v:
|
||||
v = v.replace(" ", " ")
|
||||
|
||||
for fmt in [
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%d %H:%M",
|
||||
"%Y-%m-%d %H",
|
||||
"%Y-%m-%d",
|
||||
"%Y-%m",
|
||||
"%Y",
|
||||
]:
|
||||
try:
|
||||
v = calendar.timegm(time.strptime(str(v), fmt))
|
||||
v = datetime.strptime(v, fmt).timestamp()
|
||||
break
|
||||
except:
|
||||
pass
|
||||
@@ -232,41 +182,13 @@ class U2idx(object):
|
||||
va.append(v)
|
||||
is_key = True
|
||||
|
||||
if field_end:
|
||||
q += field_end
|
||||
field_end = ""
|
||||
|
||||
# lowercase tag searches
|
||||
m = ptn_lc.search(q)
|
||||
zs = unicode(v)
|
||||
if not m or not ptn_lcv.search(zs):
|
||||
continue
|
||||
|
||||
va.pop()
|
||||
va.append(zs.lower())
|
||||
q = q[: m.start()]
|
||||
|
||||
field, oper = m.groups()
|
||||
if oper in ["=", "=="]:
|
||||
q += " {} like ? ) ".format(field)
|
||||
else:
|
||||
q += " lower({}) {} ? ) ".format(field, oper)
|
||||
|
||||
try:
|
||||
return self.run_query(vols, q, va, have_up, have_mt, lim)
|
||||
return self.run_query(vols, joins + "where " + q, va)
|
||||
except Exception as ex:
|
||||
raise Pebkac(500, repr(ex))
|
||||
|
||||
def run_query(
|
||||
self,
|
||||
vols: list[tuple[str, str, dict[str, Any]]],
|
||||
uq: str,
|
||||
uv: list[Union[str, int]],
|
||||
have_up: bool,
|
||||
have_mt: bool,
|
||||
lim: int,
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
done_flag: list[bool] = []
|
||||
def run_query(self, vols, uq, uv):
|
||||
done_flag = []
|
||||
self.active_id = "{:.6f}_{}".format(
|
||||
time.time(), threading.current_thread().ident
|
||||
)
|
||||
@@ -282,17 +204,16 @@ class U2idx(object):
|
||||
thr.start()
|
||||
|
||||
if not uq or not uv:
|
||||
uq = "select * from up"
|
||||
uv = []
|
||||
elif have_mt:
|
||||
uq = "select up.*, substr(up.w,1,16) mtw from up where " + uq
|
||||
q = "select * from up"
|
||||
v = ()
|
||||
else:
|
||||
uq = "select up.* from up where " + uq
|
||||
q = "select up.* from up " + uq
|
||||
v = tuple(uv)
|
||||
|
||||
self.log("qs: {!r} {!r}".format(uq, uv))
|
||||
self.log("qs: {!r} {!r}".format(q, v))
|
||||
|
||||
ret = []
|
||||
lim = min(lim, int(self.args.srch_hits))
|
||||
lim = 1000
|
||||
taglist = {}
|
||||
for (vtop, ptop, flags) in vols:
|
||||
cur = self.get_cur(ptop)
|
||||
@@ -301,49 +222,25 @@ class U2idx(object):
|
||||
|
||||
self.active_cur = cur
|
||||
|
||||
vuv = []
|
||||
for v in uv:
|
||||
if v == "\nrd":
|
||||
v = vtop + "/"
|
||||
|
||||
vuv.append(v)
|
||||
|
||||
sret = []
|
||||
fk = flags.get("fk")
|
||||
c = cur.execute(uq, tuple(vuv))
|
||||
c = cur.execute(q, v)
|
||||
for hit in c:
|
||||
w, ts, sz, rd, fn, ip, at = hit[:7]
|
||||
w, ts, sz, rd, fn = hit
|
||||
lim -= 1
|
||||
if lim < 0:
|
||||
if lim <= 0:
|
||||
break
|
||||
|
||||
if rd.startswith("//") or fn.startswith("//"):
|
||||
rd, fn = s3dec(rd, fn)
|
||||
|
||||
if not fk:
|
||||
suf = ""
|
||||
else:
|
||||
try:
|
||||
ap = absreal(os.path.join(ptop, rd, fn))
|
||||
inf = bos.stat(ap)
|
||||
except:
|
||||
continue
|
||||
|
||||
suf = (
|
||||
"?k="
|
||||
+ gen_filekey(
|
||||
self.args.fk_salt, ap, sz, 0 if ANYWIN else inf.st_ino
|
||||
)[:fk]
|
||||
)
|
||||
|
||||
rp = quotep("/".join([x for x in [vtop, rd, fn] if x])) + suf
|
||||
rp = "/".join([x for x in [vtop, rd, fn] if x])
|
||||
sret.append({"ts": int(ts), "sz": sz, "rp": rp, "w": w[:16]})
|
||||
|
||||
for hit in sret:
|
||||
w = hit["w"]
|
||||
del hit["w"]
|
||||
tags = {}
|
||||
q2 = "select k, v from mt where w = ? and +k != 'x'"
|
||||
q2 = "select k, v from mt where w = ? and k != 'x'"
|
||||
for k, v2 in cur.execute(q2, (w,)):
|
||||
taglist[k] = True
|
||||
tags[k] = v2
|
||||
@@ -354,26 +251,21 @@ class U2idx(object):
|
||||
# print("[{}] {}".format(ptop, sret))
|
||||
|
||||
done_flag.append(True)
|
||||
self.active_id = ""
|
||||
self.active_id = None
|
||||
|
||||
# undupe hits from multiple metadata keys
|
||||
if len(ret) > 1:
|
||||
ret = [ret[0]] + [
|
||||
y
|
||||
for x, y in zip(ret[:-1], ret[1:])
|
||||
if x["rp"].split("?")[0] != y["rp"].split("?")[0]
|
||||
y for x, y in zip(ret[:-1], ret[1:]) if x["rp"] != y["rp"]
|
||||
]
|
||||
|
||||
ret.sort(key=itemgetter("rp"))
|
||||
|
||||
return ret, list(taglist.keys())
|
||||
|
||||
def terminator(self, identifier: str, done_flag: list[bool]) -> None:
|
||||
def terminator(self, identifier, done_flag):
|
||||
for _ in range(self.timeout):
|
||||
time.sleep(1)
|
||||
if done_flag:
|
||||
return
|
||||
|
||||
if identifier == self.active_id:
|
||||
assert self.active_cur
|
||||
self.active_cur.connection.interrupt()
|
||||
|
||||
2502
copyparty/up2k.py
2502
copyparty/up2k.py
File diff suppressed because it is too large
Load Diff
1686
copyparty/util.py
1686
copyparty/util.py
File diff suppressed because it is too large
Load Diff
80
copyparty/vcr.py
Normal file
80
copyparty/vcr.py
Normal file
@@ -0,0 +1,80 @@
|
||||
# coding: utf-8
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import time
|
||||
import shlex
|
||||
import subprocess as sp
|
||||
|
||||
from .__init__ import PY2
|
||||
from .util import fsenc
|
||||
|
||||
|
||||
class VCR_Direct(object):
|
||||
def __init__(self, cli, fpath):
|
||||
self.cli = cli
|
||||
self.fpath = fpath
|
||||
|
||||
self.log_func = cli.log_func
|
||||
self.log_src = cli.log_src
|
||||
|
||||
def log(self, msg, c=0):
|
||||
self.log_func(self.log_src, "vcr: {}".format(msg), c)
|
||||
|
||||
def run(self):
|
||||
opts = self.cli.uparam
|
||||
|
||||
# fmt: off
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-nostdin",
|
||||
"-hide_banner",
|
||||
"-v", "warning",
|
||||
"-i", fsenc(self.fpath),
|
||||
"-vf", "scale=640:-4",
|
||||
"-c:a", "libopus",
|
||||
"-b:a", "128k",
|
||||
"-c:v", "libvpx",
|
||||
"-deadline", "realtime",
|
||||
"-row-mt", "1"
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
if "ss" in opts:
|
||||
cmd.extend(["-ss", opts["ss"]])
|
||||
|
||||
if "crf" in opts:
|
||||
cmd.extend(["-b:v", "0", "-crf", opts["crf"]])
|
||||
else:
|
||||
cmd.extend(["-b:v", "{}M".format(opts.get("mbps", 1.2))])
|
||||
|
||||
cmd.extend(["-f", "webm", "-"])
|
||||
|
||||
comp = str if not PY2 else unicode
|
||||
cmd = [x.encode("utf-8") if isinstance(x, comp) else x for x in cmd]
|
||||
|
||||
self.log(" ".join([shlex.quote(x.decode("utf-8", "replace")) for x in cmd]))
|
||||
|
||||
p = sp.Popen(cmd, stdout=sp.PIPE)
|
||||
|
||||
self.cli.send_headers(None, mime="video/webm")
|
||||
|
||||
fails = 0
|
||||
while True:
|
||||
self.log("read")
|
||||
buf = p.stdout.read(1024 * 4)
|
||||
if not buf:
|
||||
fails += 1
|
||||
if p.poll() is not None or fails > 30:
|
||||
self.log("ffmpeg exited")
|
||||
return
|
||||
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
|
||||
fails = 0
|
||||
try:
|
||||
self.cli.s.sendall(buf)
|
||||
except:
|
||||
self.log("client disconnected")
|
||||
p.kill()
|
||||
return
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -3,14 +3,13 @@
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>{{ title }}</title>
|
||||
<title>⇆🎉 {{ title }}</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=0.8">
|
||||
{{ html_head }}
|
||||
<link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
|
||||
<link rel="stylesheet" media="screen" href="/.cpr/browser.css?_={{ ts }}">
|
||||
<link rel="stylesheet" type="text/css" media="screen" href="/.cpr/browser.css?_={{ ts }}">
|
||||
<link rel="stylesheet" type="text/css" media="screen" href="/.cpr/upload.css?_={{ ts }}">
|
||||
{%- if css %}
|
||||
<link rel="stylesheet" media="screen" href="{{ css }}?_={{ ts }}">
|
||||
<link rel="stylesheet" type="text/css" media="screen" href="{{ css }}?_={{ ts }}">
|
||||
{%- endif %}
|
||||
</head>
|
||||
|
||||
@@ -19,9 +18,9 @@
|
||||
|
||||
<div id="op_search" class="opview">
|
||||
{%- if have_tags_idx %}
|
||||
<div id="srch_form" class="tags opbox"></div>
|
||||
<div id="srch_form" class="tags"></div>
|
||||
{%- else %}
|
||||
<div id="srch_form" class="opbox"></div>
|
||||
<div id="srch_form"></div>
|
||||
{%- endif %}
|
||||
<div id="srch_q"></div>
|
||||
</div>
|
||||
@@ -32,43 +31,40 @@
|
||||
<div id="u2err"></div>
|
||||
<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
|
||||
<input type="hidden" name="act" value="bput" />
|
||||
<input type="file" name="f" multiple /><br />
|
||||
<input type="file" name="f" multiple><br />
|
||||
<input type="submit" value="start upload">
|
||||
</form>
|
||||
<a id="bbsw" href="?b=u"><br />switch to basic browser</a>
|
||||
</div>
|
||||
|
||||
<div id="op_mkdir" class="opview opbox act">
|
||||
<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
|
||||
<input type="hidden" name="act" value="mkdir" />
|
||||
📂<input type="text" name="name" class="i">
|
||||
<input type="submit" value="make directory">
|
||||
<input type="text" name="name" size="30">
|
||||
<input type="submit" value="mkdir">
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div id="op_new_md" class="opview opbox">
|
||||
<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
|
||||
<input type="hidden" name="act" value="new_md" />
|
||||
📝<input type="text" name="name" class="i">
|
||||
<input type="submit" value="new markdown doc">
|
||||
<input type="text" name="name" size="30">
|
||||
<input type="submit" value="create doc">
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div id="op_msg" class="opview opbox act">
|
||||
<form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="{{ url_suf }}">
|
||||
📟<input type="text" name="msg" class="i">
|
||||
<input type="submit" value="send msg to srv log">
|
||||
<input type="text" name="msg" size="30">
|
||||
<input type="submit" value="send msg">
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div id="op_unpost" class="opview opbox"></div>
|
||||
|
||||
<div id="op_up2k" class="opview"></div>
|
||||
|
||||
<div id="op_cfg" class="opview opbox opwide"></div>
|
||||
|
||||
<h1 id="path">
|
||||
<a href="#" id="entree">🌲</a>
|
||||
<a href="#" id="entree" tt="show directory tree$NHotkey: B">🌲</a>
|
||||
{%- for n in vpnodes %}
|
||||
<a href="/{{ n[0] }}">{{ n[1] }}</a>
|
||||
{%- endfor %}
|
||||
@@ -78,12 +74,6 @@
|
||||
|
||||
<div id="wrap">
|
||||
|
||||
{%- if doc %}
|
||||
<div id="bdoc"><pre>{{ doc|e }}</pre></div>
|
||||
{%- else %}
|
||||
<div id="bdoc"></div>
|
||||
{%- endif %}
|
||||
|
||||
<div id="pro" class="logue">{{ logues[0] }}</div>
|
||||
|
||||
<table id="files">
|
||||
@@ -120,9 +110,7 @@
|
||||
|
||||
<div id="epi" class="logue">{{ logues[1] }}</div>
|
||||
|
||||
<h2><a href="/?h" id="goh">control-panel</a></h2>
|
||||
|
||||
<a href="#" id="repl">π</a>
|
||||
<h2><a href="/?h">control-panel</a></h2>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -133,39 +121,15 @@
|
||||
<div id="widget"></div>
|
||||
|
||||
<script>
|
||||
var acct = "{{ acct }}",
|
||||
perms = {{ perms }},
|
||||
themes = {{ themes }},
|
||||
dtheme = "{{ dtheme }}",
|
||||
srvinf = "{{ srv_info }}",
|
||||
lang = "{{ lang }}",
|
||||
dfavico = "{{ favico }}",
|
||||
def_hcols = {{ def_hcols|tojson }},
|
||||
var perms = {{ perms }},
|
||||
tag_order_cfg = {{ tag_order }},
|
||||
have_up2k_idx = {{ have_up2k_idx|tojson }},
|
||||
have_tags_idx = {{ have_tags_idx|tojson }},
|
||||
have_acode = {{ have_acode|tojson }},
|
||||
have_mv = {{ have_mv|tojson }},
|
||||
have_del = {{ have_del|tojson }},
|
||||
have_unpost = {{ have_unpost }},
|
||||
have_zip = {{ have_zip|tojson }},
|
||||
lifetime = {{ lifetime }},
|
||||
turbolvl = {{ turbolvl }},
|
||||
u2sort = "{{ u2sort }}",
|
||||
have_emp = {{ have_emp|tojson }},
|
||||
txt_ext = "{{ txt_ext }}",
|
||||
{% if no_prism %}no_prism = 1,{% endif %}
|
||||
readme = {{ readme|tojson }},
|
||||
ls0 = {{ ls0|tojson }};
|
||||
|
||||
document.documentElement.className = localStorage.theme || dtheme;
|
||||
have_zip = {{ have_zip|tojson }};
|
||||
</script>
|
||||
<script src="/.cpr/util.js?_={{ ts }}"></script>
|
||||
<script src="/.cpr/baguettebox.js?_={{ ts }}"></script>
|
||||
<script src="/.cpr/browser.js?_={{ ts }}"></script>
|
||||
<script src="/.cpr/up2k.js?_={{ ts }}"></script>
|
||||
{%- if js %}
|
||||
<script src="{{ js }}?_={{ ts }}"></script>
|
||||
{%- endif %}
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,7 +6,6 @@
|
||||
<title>{{ title }}</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=0.8">
|
||||
{{ html_head }}
|
||||
<style>
|
||||
html{font-family:sans-serif}
|
||||
td{border:1px solid #999;border-width:1px 1px 0 0;padding:0 5px}
|
||||
@@ -45,9 +44,7 @@
|
||||
<tr><td></td><td><a href="../{{ url_suf }}">parent folder</a></td><td>-</td><td>-</td></tr>
|
||||
|
||||
{%- for f in files %}
|
||||
<tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}{{
|
||||
'&' + url_suf[1:] if url_suf[:1] == '?' and '?' in f.href else url_suf
|
||||
}}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.dt }}</td></tr>
|
||||
<tr><td>{{ f.lead }}</td><td><a href="{{ f.href }}{{ url_suf }}">{{ f.name|e }}</a></td><td>{{ f.sz }}</td><td>{{ f.dt }}</td></tr>
|
||||
{%- endfor %}
|
||||
|
||||
</tbody>
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>{{ svcname }}</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=0.8">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div id="box" style="opacity: 0; font-family: sans-serif">
|
||||
<h3>please press F5 to reload the page</h3>
|
||||
<p>sorry for the inconvenience</p>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
setTimeout(function() {
|
||||
document.getElementById('box').style.opacity = 1;
|
||||
}, 500);
|
||||
|
||||
parent.toast.ok(30, parent.L.cf_ok);
|
||||
parent.qsr('#cf_frame');
|
||||
</script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
@@ -13,7 +13,8 @@ audio_eq.apply = function () {
|
||||
|
||||
var can = ebi('fft_can');
|
||||
if (!can) {
|
||||
can = mknod('canvas', 'fft_can');
|
||||
can = mknod('canvas');
|
||||
can.setAttribute('id', 'fft_can');
|
||||
can.style.cssText = 'position:absolute;left:0;bottom:5em;width:' + w + 'px;height:' + h + 'px;z-index:9001';
|
||||
document.body.appendChild(can);
|
||||
can.width = w;
|
||||
|
||||
@@ -1,17 +1,13 @@
|
||||
@font-face {
|
||||
font-family: 'scp';
|
||||
src: local('Source Code Pro Regular'), local('SourceCodePro-Regular'), url(/.cpr/deps/scp.woff2) format('woff2');
|
||||
}
|
||||
html, body {
|
||||
color: #333;
|
||||
background: #eee;
|
||||
font-family: sans-serif;
|
||||
line-height: 1.5em;
|
||||
}
|
||||
#repl {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
right: .5em;
|
||||
border: none;
|
||||
color: inherit;
|
||||
background: none;
|
||||
}
|
||||
#mtw {
|
||||
display: none;
|
||||
}
|
||||
@@ -19,12 +15,122 @@ html, body {
|
||||
margin: 0 auto;
|
||||
padding: 0 1.5em;
|
||||
}
|
||||
#toast {
|
||||
bottom: auto;
|
||||
top: 1.4em;
|
||||
pre, code, a {
|
||||
color: #480;
|
||||
background: #f7f7f7;
|
||||
border: .07em solid #ddd;
|
||||
border-radius: .2em;
|
||||
padding: .1em .3em;
|
||||
margin: 0 .1em;
|
||||
}
|
||||
a {
|
||||
text-decoration: none;
|
||||
code {
|
||||
font-size: .96em;
|
||||
}
|
||||
pre, code {
|
||||
font-family: 'scp', monospace, monospace;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-all;
|
||||
}
|
||||
pre {
|
||||
counter-reset: precode;
|
||||
}
|
||||
pre code {
|
||||
counter-increment: precode;
|
||||
display: inline-block;
|
||||
margin: 0 -.3em;
|
||||
padding: .4em .5em;
|
||||
border: none;
|
||||
border-bottom: 1px solid #cdc;
|
||||
min-width: calc(100% - .6em);
|
||||
line-height: 1.1em;
|
||||
}
|
||||
pre code:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
pre code::before {
|
||||
content: counter(precode);
|
||||
-webkit-user-select: none;
|
||||
-moz-user-select: none;
|
||||
-ms-user-select: none;
|
||||
user-select: none;
|
||||
display: inline-block;
|
||||
text-align: right;
|
||||
font-size: .75em;
|
||||
color: #48a;
|
||||
width: 4em;
|
||||
padding-right: 1.5em;
|
||||
margin-left: -5.5em;
|
||||
}
|
||||
pre code:hover {
|
||||
background: #fec;
|
||||
color: #360;
|
||||
}
|
||||
h1, h2 {
|
||||
line-height: 1.5em;
|
||||
}
|
||||
h1 {
|
||||
font-size: 1.7em;
|
||||
text-align: center;
|
||||
border: 1em solid #777;
|
||||
border-width: .05em 0;
|
||||
margin: 3em 0;
|
||||
}
|
||||
h2 {
|
||||
font-size: 1.5em;
|
||||
font-weight: normal;
|
||||
background: #f7f7f7;
|
||||
border-top: .07em solid #fff;
|
||||
border-bottom: .07em solid #bbb;
|
||||
border-radius: .5em .5em 0 0;
|
||||
padding-left: .4em;
|
||||
margin-top: 3em;
|
||||
}
|
||||
h3 {
|
||||
border-bottom: .1em solid #999;
|
||||
}
|
||||
h1 a, h3 a, h5 a,
|
||||
h2 a, h4 a, h6 a {
|
||||
color: inherit;
|
||||
display: block;
|
||||
background: none;
|
||||
border: none;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
}
|
||||
#mp ul,
|
||||
#mp ol {
|
||||
border-left: .3em solid #ddd;
|
||||
}
|
||||
#m>ul,
|
||||
#m>ol {
|
||||
border-color: #bbb;
|
||||
}
|
||||
#mp ul>li {
|
||||
list-style-type: disc;
|
||||
}
|
||||
#mp ul>li,
|
||||
#mp ol>li {
|
||||
margin: .7em 0;
|
||||
}
|
||||
strong {
|
||||
color: #000;
|
||||
}
|
||||
p>em,
|
||||
li>em,
|
||||
td>em {
|
||||
color: #c50;
|
||||
padding: .1em;
|
||||
border-bottom: .1em solid #bbb;
|
||||
}
|
||||
blockquote {
|
||||
font-family: serif;
|
||||
background: #f7f7f7;
|
||||
border: .07em dashed #ccc;
|
||||
padding: 0 2em;
|
||||
margin: 1em 0;
|
||||
}
|
||||
small {
|
||||
opacity: .8;
|
||||
}
|
||||
#toc {
|
||||
margin: 0 1em;
|
||||
@@ -60,7 +166,7 @@ a {
|
||||
z-index: 99;
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
font-family: 'scp', monospace, monospace;
|
||||
font-family: monospace, monospace;
|
||||
font-weight: bold;
|
||||
font-size: 1.3em;
|
||||
line-height: .1em;
|
||||
@@ -72,6 +178,14 @@ a {
|
||||
color: #6b3;
|
||||
text-shadow: .02em 0 0 #6b3;
|
||||
}
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
margin: 1em 0;
|
||||
}
|
||||
th, td {
|
||||
padding: .2em .5em;
|
||||
border: .12em solid #aaa;
|
||||
}
|
||||
blink {
|
||||
animation: blinker .7s cubic-bezier(.9, 0, .1, 1) infinite;
|
||||
}
|
||||
@@ -84,36 +198,6 @@ blink {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
.mdo pre {
|
||||
counter-reset: precode;
|
||||
}
|
||||
.mdo pre code {
|
||||
counter-increment: precode;
|
||||
display: inline-block;
|
||||
border: none;
|
||||
border-bottom: 1px solid #cdc;
|
||||
min-width: calc(100% - .6em);
|
||||
}
|
||||
.mdo pre code:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
.mdo pre code::before {
|
||||
content: counter(precode);
|
||||
-webkit-user-select: none;
|
||||
-moz-user-select: none;
|
||||
-ms-user-select: none;
|
||||
user-select: none;
|
||||
display: inline-block;
|
||||
text-align: right;
|
||||
font-size: .75em;
|
||||
color: #48a;
|
||||
width: 4em;
|
||||
padding-right: 1.5em;
|
||||
margin-left: -5.5em;
|
||||
}
|
||||
|
||||
|
||||
@media screen {
|
||||
html, body {
|
||||
margin: 0;
|
||||
@@ -130,6 +214,34 @@ blink {
|
||||
#mp {
|
||||
max-width: 52em;
|
||||
margin-bottom: 6em;
|
||||
word-break: break-word;
|
||||
overflow-wrap: break-word;
|
||||
word-wrap: break-word; /*ie*/
|
||||
}
|
||||
a {
|
||||
color: #fff;
|
||||
background: #39b;
|
||||
text-decoration: none;
|
||||
padding: 0 .3em;
|
||||
border: none;
|
||||
border-bottom: .07em solid #079;
|
||||
}
|
||||
h2 {
|
||||
color: #fff;
|
||||
background: #555;
|
||||
margin-top: 2em;
|
||||
border-bottom: .22em solid #999;
|
||||
border-top: none;
|
||||
}
|
||||
h1 {
|
||||
color: #fff;
|
||||
background: #444;
|
||||
font-weight: normal;
|
||||
border-top: .4em solid #fb0;
|
||||
border-bottom: .4em solid #777;
|
||||
border-radius: 0 1em 0 1em;
|
||||
margin: 3em 0 1em 0;
|
||||
padding: .5em 0;
|
||||
}
|
||||
#mn {
|
||||
padding: 1.3em 0 .7em 1em;
|
||||
@@ -161,7 +273,7 @@ blink {
|
||||
height: 1.05em;
|
||||
margin: -.2em .3em -.2em -.4em;
|
||||
display: inline-block;
|
||||
border: 1px solid rgba(154,154,154,0.6);
|
||||
border: 1px solid rgba(0,0,0,0.2);
|
||||
border-width: .2em .2em 0 0;
|
||||
transform: rotate(45deg);
|
||||
}
|
||||
@@ -182,8 +294,6 @@ blink {
|
||||
color: #444;
|
||||
background: none;
|
||||
text-decoration: underline;
|
||||
margin: 0 .1em;
|
||||
padding: 0 .3em;
|
||||
border: none;
|
||||
}
|
||||
#mh a:hover {
|
||||
@@ -212,52 +322,100 @@ blink {
|
||||
#toolsbox a+a {
|
||||
text-decoration: none;
|
||||
}
|
||||
#lno {
|
||||
position: absolute;
|
||||
right: 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
html.z,
|
||||
html.z body {
|
||||
html.dark,
|
||||
html.dark body {
|
||||
background: #222;
|
||||
color: #ccc;
|
||||
}
|
||||
html.z #toc a {
|
||||
html.dark #toc a {
|
||||
color: #ccc;
|
||||
border-left: .4em solid #444;
|
||||
border-bottom: .1em solid #333;
|
||||
}
|
||||
html.z #toc a.act {
|
||||
html.dark #toc a.act {
|
||||
color: #fff;
|
||||
border-left: .4em solid #3ad;
|
||||
}
|
||||
html.z #toc li {
|
||||
html.dark #toc li {
|
||||
border-width: 0;
|
||||
}
|
||||
html.z #mn a {
|
||||
html.dark #mp a {
|
||||
background: #057;
|
||||
}
|
||||
html.dark #mp h1 a, html.dark #mp h4 a,
|
||||
html.dark #mp h2 a, html.dark #mp h5 a,
|
||||
html.dark #mp h3 a, html.dark #mp h6 a {
|
||||
color: inherit;
|
||||
background: none;
|
||||
}
|
||||
html.dark pre,
|
||||
html.dark code {
|
||||
color: #8c0;
|
||||
background: #1a1a1a;
|
||||
border: .07em solid #333;
|
||||
}
|
||||
html.dark #mp ul,
|
||||
html.dark #mp ol {
|
||||
border-color: #444;
|
||||
}
|
||||
html.dark #m>ul,
|
||||
html.dark #m>ol {
|
||||
border-color: #555;
|
||||
}
|
||||
html.dark strong {
|
||||
color: #fff;
|
||||
}
|
||||
html.dark p>em,
|
||||
html.dark li>em,
|
||||
html.dark td>em {
|
||||
color: #f94;
|
||||
border-color: #666;
|
||||
}
|
||||
html.dark h1 {
|
||||
background: #383838;
|
||||
border-top: .4em solid #b80;
|
||||
border-bottom: .4em solid #4c4c4c;
|
||||
}
|
||||
html.dark h2 {
|
||||
background: #444;
|
||||
border-bottom: .22em solid #555;
|
||||
}
|
||||
html.dark td,
|
||||
html.dark th {
|
||||
border-color: #444;
|
||||
}
|
||||
html.dark blockquote {
|
||||
background: #282828;
|
||||
border: .07em dashed #444;
|
||||
}
|
||||
html.dark #mn a:not(:last-child)::after {
|
||||
border-color: rgba(255,255,255,0.3);
|
||||
}
|
||||
html.dark #mn a {
|
||||
color: #ccc;
|
||||
}
|
||||
html.z #mn {
|
||||
html.dark #mn {
|
||||
border-bottom: 1px solid #333;
|
||||
}
|
||||
html.z #mn,
|
||||
html.z #mh {
|
||||
html.dark #mn,
|
||||
html.dark #mh {
|
||||
background: #222;
|
||||
}
|
||||
html.z #mh a {
|
||||
html.dark #mh a {
|
||||
color: #ccc;
|
||||
background: none;
|
||||
}
|
||||
html.z #mh a:hover {
|
||||
html.dark #mh a:hover {
|
||||
background: #333;
|
||||
color: #fff;
|
||||
}
|
||||
html.z #toolsbox {
|
||||
html.dark #toolsbox {
|
||||
background: #222;
|
||||
}
|
||||
html.z #toolsbox.open {
|
||||
html.dark #toolsbox.open {
|
||||
box-shadow: 0 .2em .2em #069;
|
||||
border-radius: 0 0 .4em .4em;
|
||||
}
|
||||
@@ -305,23 +463,23 @@ blink {
|
||||
|
||||
|
||||
|
||||
html.z #toc {
|
||||
html.dark #toc {
|
||||
background: #282828;
|
||||
border-top: 1px solid #2c2c2c;
|
||||
box-shadow: 0 0 1em #181818;
|
||||
}
|
||||
html.z #toc,
|
||||
html.z #mw {
|
||||
html.dark #toc,
|
||||
html.dark #mw {
|
||||
scrollbar-color: #b80 #282828;
|
||||
}
|
||||
html.z #toc::-webkit-scrollbar-track {
|
||||
html.dark #toc::-webkit-scrollbar-track {
|
||||
background: #282828;
|
||||
}
|
||||
html.z #toc::-webkit-scrollbar {
|
||||
html.dark #toc::-webkit-scrollbar {
|
||||
background: #282828;
|
||||
width: .8em;
|
||||
}
|
||||
html.z #toc::-webkit-scrollbar-thumb {
|
||||
html.dark #toc::-webkit-scrollbar-thumb {
|
||||
background: #b80;
|
||||
}
|
||||
}
|
||||
@@ -338,15 +496,12 @@ blink {
|
||||
mso-footer-margin: .6in;
|
||||
mso-paper-source: 0;
|
||||
}
|
||||
.mdo a {
|
||||
a {
|
||||
color: #079;
|
||||
text-decoration: none;
|
||||
border-bottom: .07em solid #4ac;
|
||||
padding: 0 .3em;
|
||||
}
|
||||
#repl {
|
||||
display: none;
|
||||
}
|
||||
#toc>ul {
|
||||
border-left: .1em solid #84c4dd;
|
||||
}
|
||||
@@ -371,20 +526,18 @@ blink {
|
||||
a[ctr]::before {
|
||||
content: attr(ctr) '. ';
|
||||
}
|
||||
.mdo h1 {
|
||||
h1 {
|
||||
margin: 2em 0;
|
||||
}
|
||||
.mdo h2 {
|
||||
h2 {
|
||||
margin: 2em 0 0 0;
|
||||
}
|
||||
.mdo h1,
|
||||
.mdo h2,
|
||||
.mdo h3 {
|
||||
h1, h2, h3 {
|
||||
page-break-inside: avoid;
|
||||
}
|
||||
.mdo h1::after,
|
||||
.mdo h2::after,
|
||||
.mdo h3::after {
|
||||
h1::after,
|
||||
h2::after,
|
||||
h3::after {
|
||||
content: 'orz';
|
||||
color: transparent;
|
||||
display: block;
|
||||
@@ -392,20 +545,20 @@ blink {
|
||||
padding: 4em 0 0 0;
|
||||
margin: 0 0 -5em 0;
|
||||
}
|
||||
.mdo p {
|
||||
p {
|
||||
page-break-inside: avoid;
|
||||
}
|
||||
.mdo table {
|
||||
table {
|
||||
page-break-inside: auto;
|
||||
}
|
||||
.mdo tr {
|
||||
tr {
|
||||
page-break-inside: avoid;
|
||||
page-break-after: auto;
|
||||
}
|
||||
.mdo thead {
|
||||
thead {
|
||||
display: table-header-group;
|
||||
}
|
||||
.mdo tfoot {
|
||||
tfoot {
|
||||
display: table-footer-group;
|
||||
}
|
||||
#mp a.vis::after {
|
||||
@@ -413,32 +566,31 @@ blink {
|
||||
border-bottom: 1px solid #bbb;
|
||||
color: #444;
|
||||
}
|
||||
.mdo blockquote {
|
||||
blockquote {
|
||||
border-color: #555;
|
||||
}
|
||||
.mdo code {
|
||||
code {
|
||||
border-color: #bbb;
|
||||
}
|
||||
.mdo pre,
|
||||
.mdo pre code {
|
||||
pre, pre code {
|
||||
border-color: #999;
|
||||
}
|
||||
.mdo pre code::before {
|
||||
pre code::before {
|
||||
color: #058;
|
||||
}
|
||||
|
||||
|
||||
|
||||
html.z .mdo a {
|
||||
html.dark a {
|
||||
color: #000;
|
||||
}
|
||||
html.z .mdo pre,
|
||||
html.z .mdo code {
|
||||
html.dark pre,
|
||||
html.dark code {
|
||||
color: #240;
|
||||
}
|
||||
html.z .mdo p>em,
|
||||
html.z .mdo li>em,
|
||||
html.z .mdo td>em {
|
||||
html.dark p>em,
|
||||
html.dark li>em,
|
||||
html.dark td>em {
|
||||
color: #940;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,24 +1,22 @@
|
||||
<!DOCTYPE html><html><head>
|
||||
<meta charset="utf-8">
|
||||
<title>📝 {{ title }}</title>
|
||||
<title>📝🎉 {{ title }}</title> <!-- 📜 -->
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=0.7">
|
||||
{{ html_head }}
|
||||
<link rel="stylesheet" href="/.cpr/ui.css?_={{ ts }}">
|
||||
<link rel="stylesheet" href="/.cpr/md.css?_={{ ts }}">
|
||||
<link href="/.cpr/md.css?_={{ ts }}" rel="stylesheet">
|
||||
{%- if edit %}
|
||||
<link rel="stylesheet" href="/.cpr/md2.css?_={{ ts }}">
|
||||
<link href="/.cpr/md2.css?_={{ ts }}" rel="stylesheet">
|
||||
{%- endif %}
|
||||
</head>
|
||||
<body>
|
||||
<div id="mn"></div>
|
||||
<div id="mn">navbar</div>
|
||||
<div id="mh">
|
||||
<a id="lightswitch" href="#">go dark</a>
|
||||
<a id="navtoggle" href="#">hide nav</a>
|
||||
{%- if edit %}
|
||||
<a id="save" href="{{ arg_base }}edit" tt="Hotkey: ctrl-s">save</a>
|
||||
<a id="sbs" href="#" tt="editor and preview side by side">sbs</a>
|
||||
<a id="nsbs" href="#" tt="switch between editor and preview$NHotkey: ctrl-e">editor</a>
|
||||
<a id="save" href="?edit">save</a>
|
||||
<a id="sbs" href="#">sbs</a>
|
||||
<a id="nsbs" href="#">editor</a>
|
||||
<div id="toolsbox">
|
||||
<a id="tools" href="#">tools</a>
|
||||
<a id="fmt_table" href="#">prettify table (ctrl-k)</a>
|
||||
@@ -27,11 +25,10 @@
|
||||
<a id="cfg_uni" href="#">non-ascii: whitelist</a>
|
||||
<a id="help" href="#">help</a>
|
||||
</div>
|
||||
<span id="lno">L#</span>
|
||||
{%- else %}
|
||||
<a href="{{ arg_base }}edit" tt="good: higher performance$Ngood: same document width as viewer$Nbad: assumes you know markdown">edit (basic)</a>
|
||||
<a href="{{ arg_base }}edit2" tt="not in-house so probably less buggy">edit (fancy)</a>
|
||||
<a href="{{ arg_base }}raw">view raw</a>
|
||||
<a href="?edit">edit (basic)</a>
|
||||
<a href="?edit2">edit (fancy)</a>
|
||||
<a href="?raw">view raw</a>
|
||||
{%- endif %}
|
||||
</div>
|
||||
<div id="toc"></div>
|
||||
@@ -45,9 +42,8 @@
|
||||
if you're still reading this, check that javascript is allowed
|
||||
</div>
|
||||
</div>
|
||||
<div id="mp" class="mdo"></div>
|
||||
<div id="mp"></div>
|
||||
</div>
|
||||
<a href="#" id="repl">π</a>
|
||||
|
||||
{%- if edit %}
|
||||
<div id="helpbox">
|
||||
@@ -127,28 +123,26 @@ write markdown (most html is 🙆 too)
|
||||
|
||||
<script>
|
||||
|
||||
var last_modified = {{ lastmod }},
|
||||
have_emp = {{ have_emp|tojson }},
|
||||
dfavico = "{{ favico }}";
|
||||
|
||||
var last_modified = {{ lastmod }};
|
||||
var md_opt = {
|
||||
link_md_as_html: false,
|
||||
allow_plugins: {{ md_plug }},
|
||||
modpoll_freq: {{ md_chk_rate }}
|
||||
};
|
||||
|
||||
(function () {
|
||||
var l = localStorage,
|
||||
drk = l.light != 1,
|
||||
btn = document.getElementById("lightswitch"),
|
||||
f = function (e) {
|
||||
if (e) { e.preventDefault(); drk = !drk; }
|
||||
document.documentElement.className = drk? "z":"y";
|
||||
btn.innerHTML = "go " + (drk ? "light":"dark");
|
||||
l.light = drk? 0:1;
|
||||
var btn = document.getElementById("lightswitch");
|
||||
var toggle = function (e) {
|
||||
if (e) e.preventDefault();
|
||||
var dark = !document.documentElement.getAttribute("class");
|
||||
document.documentElement.setAttribute("class", dark ? "dark" : "");
|
||||
btn.innerHTML = "go " + (dark ? "light" : "dark");
|
||||
if (window.localStorage)
|
||||
localStorage.setItem('lightmode', dark ? 0 : 1);
|
||||
};
|
||||
|
||||
btn.onclick = f;
|
||||
f();
|
||||
btn.onclick = toggle;
|
||||
if (window.localStorage && localStorage.getItem('lightmode') != 1)
|
||||
toggle();
|
||||
})();
|
||||
|
||||
</script>
|
||||
|
||||
@@ -20,6 +20,27 @@ var dbg = function () { };
|
||||
// dbg = console.log
|
||||
|
||||
|
||||
// plugins
|
||||
var md_plug = {};
|
||||
|
||||
|
||||
function hesc(txt) {
|
||||
return txt.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
||||
}
|
||||
|
||||
|
||||
function cls(dom, name, add) {
|
||||
var re = new RegExp('(^| )' + name + '( |$)');
|
||||
var lst = (dom.getAttribute('class') + '').replace(re, "$1$2").replace(/ /, "");
|
||||
dom.setAttribute('class', lst + (add ? ' ' + name : ''));
|
||||
}
|
||||
|
||||
|
||||
function statify(obj) {
|
||||
return JSON.parse(JSON.stringify(obj));
|
||||
}
|
||||
|
||||
|
||||
// dodge browser issues
|
||||
(function () {
|
||||
var ua = navigator.userAgent;
|
||||
@@ -35,34 +56,20 @@ var dbg = function () { };
|
||||
|
||||
// add navbar
|
||||
(function () {
|
||||
var parts = get_evpath().split('/'), link = '', o;
|
||||
for (var a = 0, aa = parts.length - 2; a <= aa; a++) {
|
||||
link += parts[a] + (a < aa ? '/' : '');
|
||||
o = mknod('a');
|
||||
o.setAttribute('href', link);
|
||||
o.textContent = uricom_dec(parts[a]) || 'top';
|
||||
dom_nav.appendChild(o);
|
||||
var n = document.location + '';
|
||||
n = n.substr(n.indexOf('//') + 2).split('?')[0].split('/');
|
||||
n[0] = 'top';
|
||||
var loc = [];
|
||||
var nav = [];
|
||||
for (var a = 0; a < n.length; a++) {
|
||||
if (a > 0)
|
||||
loc.push(n[a]);
|
||||
|
||||
var dec = hesc(uricom_dec(n[a])[0]);
|
||||
|
||||
nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
|
||||
}
|
||||
})();
|
||||
|
||||
|
||||
// image load handler
|
||||
var img_load = (function () {
|
||||
var r = {};
|
||||
r.callbacks = [];
|
||||
|
||||
function fire() {
|
||||
for (var a = 0; a < r.callbacks.length; a++)
|
||||
r.callbacks[a]();
|
||||
}
|
||||
|
||||
var timeout = null;
|
||||
r.done = function () {
|
||||
clearTimeout(timeout);
|
||||
timeout = setTimeout(fire, 500);
|
||||
};
|
||||
|
||||
return r;
|
||||
dom_nav.innerHTML = nav.join('');
|
||||
})();
|
||||
|
||||
|
||||
@@ -81,13 +88,13 @@ function copydom(src, dst, lv) {
|
||||
|
||||
var rpl = [];
|
||||
for (var a = sc.length - 1; a >= 0; a--) {
|
||||
var st = sc[a].tagName || sc[a].nodeType,
|
||||
dt = dc[a].tagName || dc[a].nodeType;
|
||||
var st = sc[a].tagName,
|
||||
dt = dc[a].tagName;
|
||||
|
||||
if (st !== dt) {
|
||||
dbg("replace L%d (%d/%d) type %s/%s", lv, a, sc.length, st, dt);
|
||||
dst.innerHTML = src.innerHTML;
|
||||
return;
|
||||
rpl.push(a);
|
||||
continue;
|
||||
}
|
||||
|
||||
var sa = sc[a].attributes || [],
|
||||
@@ -136,11 +143,8 @@ function copydom(src, dst, lv) {
|
||||
// repl is reversed; build top-down
|
||||
var nbytes = 0;
|
||||
for (var a = rpl.length - 1; a >= 0; a--) {
|
||||
var i = rpl[a],
|
||||
prop = sc[i].nodeType == 1 ? 'outerHTML' : 'nodeValue';
|
||||
|
||||
var html = sc[i][prop];
|
||||
dc[i][prop] = html;
|
||||
var html = sc[rpl[a]].outerHTML;
|
||||
dc[rpl[a]].outerHTML = html;
|
||||
nbytes += html.length;
|
||||
}
|
||||
if (nbytes > 0)
|
||||
@@ -156,8 +160,11 @@ function copydom(src, dst, lv) {
|
||||
}
|
||||
|
||||
|
||||
md_plug_err = function (ex, js) {
|
||||
qsr('#md_errbox');
|
||||
function md_plug_err(ex, js) {
|
||||
var errbox = ebi('md_errbox');
|
||||
if (errbox)
|
||||
errbox.parentNode.removeChild(errbox);
|
||||
|
||||
if (!ex)
|
||||
return;
|
||||
|
||||
@@ -169,15 +176,16 @@ md_plug_err = function (ex, js) {
|
||||
var lns = js.split('\n');
|
||||
if (ln < lns.length) {
|
||||
o = mknod('span');
|
||||
o.style.cssText = "color:#ac2;font-size:.9em;font-family:'scp',monospace,monospace;display:block";
|
||||
o.style.cssText = 'color:#ac2;font-size:.9em;font-family:scp;display:block';
|
||||
o.textContent = lns[ln - 1];
|
||||
}
|
||||
}
|
||||
var errbox = mknod('div', 'md_errbox');
|
||||
errbox = mknod('div');
|
||||
errbox.setAttribute('id', 'md_errbox');
|
||||
errbox.style.cssText = 'position:absolute;top:0;left:0;padding:1em .5em;background:#2b2b2b;color:#fc5'
|
||||
errbox.textContent = msg;
|
||||
errbox.onclick = function () {
|
||||
modal.alert('<pre>' + esc(ex.stack) + '</pre>');
|
||||
alert('' + ex.stack);
|
||||
};
|
||||
if (o) {
|
||||
errbox.appendChild(o);
|
||||
@@ -192,12 +200,50 @@ md_plug_err = function (ex, js) {
|
||||
}
|
||||
|
||||
|
||||
function load_plug(md_text, plug_type) {
|
||||
if (!md_opt.allow_plugins)
|
||||
return md_text;
|
||||
|
||||
var find = '\n```copyparty_' + plug_type + '\n';
|
||||
var ofs = md_text.indexOf(find);
|
||||
if (ofs === -1)
|
||||
return md_text;
|
||||
|
||||
var ofs2 = md_text.indexOf('\n```', ofs + 1);
|
||||
if (ofs2 == -1)
|
||||
return md_text;
|
||||
|
||||
var js = md_text.slice(ofs + find.length, ofs2 + 1);
|
||||
var md = md_text.slice(0, ofs + 1) + md_text.slice(ofs2 + 4);
|
||||
|
||||
var old_plug = md_plug[plug_type];
|
||||
if (!old_plug || old_plug[1] != js) {
|
||||
js = 'const x = { ' + js + ' }; x;';
|
||||
try {
|
||||
var x = eval(js);
|
||||
}
|
||||
catch (ex) {
|
||||
md_plug[plug_type] = null;
|
||||
md_plug_err(ex, js);
|
||||
return md;
|
||||
}
|
||||
if (x['ctor']) {
|
||||
x['ctor']();
|
||||
delete x['ctor'];
|
||||
}
|
||||
md_plug[plug_type] = [x, js];
|
||||
}
|
||||
|
||||
return md;
|
||||
}
|
||||
|
||||
|
||||
function convert_markdown(md_text, dest_dom) {
|
||||
md_text = md_text.replace(/\r/g, '');
|
||||
|
||||
md_plug_err(null);
|
||||
md_text = load_md_plug(md_text, 'pre');
|
||||
md_text = load_md_plug(md_text, 'post');
|
||||
md_text = load_plug(md_text, 'pre');
|
||||
md_text = load_plug(md_text, 'post');
|
||||
|
||||
var marked_opts = {
|
||||
//headerPrefix: 'h-',
|
||||
@@ -205,12 +251,12 @@ function convert_markdown(md_text, dest_dom) {
|
||||
gfm: true
|
||||
};
|
||||
|
||||
var ext = md_plug.pre;
|
||||
var ext = md_plug['pre'];
|
||||
if (ext)
|
||||
Object.assign(marked_opts, ext[0]);
|
||||
|
||||
try {
|
||||
var md_html = marked.parse(md_text, marked_opts);
|
||||
var md_html = marked(md_text, marked_opts);
|
||||
}
|
||||
catch (ex) {
|
||||
if (ext)
|
||||
@@ -218,14 +264,7 @@ function convert_markdown(md_text, dest_dom) {
|
||||
|
||||
throw ex;
|
||||
}
|
||||
var md_dom = dest_dom;
|
||||
try {
|
||||
md_dom = new DOMParser().parseFromString(md_html, "text/html").body;
|
||||
}
|
||||
catch (ex) {
|
||||
md_dom.innerHTML = md_html;
|
||||
window.copydom = noop;
|
||||
}
|
||||
var md_dom = new DOMParser().parseFromString(md_html, "text/html").body;
|
||||
|
||||
var nodes = md_dom.getElementsByTagName('a');
|
||||
for (var a = nodes.length - 1; a >= 0; a--) {
|
||||
@@ -235,7 +274,7 @@ function convert_markdown(md_text, dest_dom) {
|
||||
if (!txt)
|
||||
nodes[a].textContent = href;
|
||||
else if (href !== txt)
|
||||
nodes[a].className = 'vis';
|
||||
nodes[a].setAttribute('class', 'vis');
|
||||
}
|
||||
|
||||
// todo-lists (should probably be a marked extension)
|
||||
@@ -251,7 +290,7 @@ function convert_markdown(md_text, dest_dom) {
|
||||
var clas = done ? 'done' : 'pend';
|
||||
var char = done ? 'Y' : 'N';
|
||||
|
||||
dom_li.className = 'task-list-item';
|
||||
dom_li.setAttribute('class', 'task-list-item');
|
||||
dom_li.style.listStyleType = 'none';
|
||||
var html = dom_li.innerHTML;
|
||||
dom_li.innerHTML =
|
||||
@@ -306,7 +345,7 @@ function convert_markdown(md_text, dest_dom) {
|
||||
el.innerHTML = '<a href="#' + id + '">' + el.innerHTML + '</a>';
|
||||
}
|
||||
|
||||
ext = md_plug.post;
|
||||
ext = md_plug['post'];
|
||||
if (ext && ext[0].render)
|
||||
try {
|
||||
ext[0].render(md_dom);
|
||||
@@ -317,10 +356,6 @@ function convert_markdown(md_text, dest_dom) {
|
||||
|
||||
copydom(md_dom, dest_dom, 0);
|
||||
|
||||
var imgs = dest_dom.getElementsByTagName('img');
|
||||
for (var a = 0, aa = imgs.length; a < aa; a++)
|
||||
imgs[a].onload = img_load.done;
|
||||
|
||||
if (ext && ext[0].render2)
|
||||
try {
|
||||
ext[0].render2(dest_dom);
|
||||
@@ -332,7 +367,8 @@ function convert_markdown(md_text, dest_dom) {
|
||||
|
||||
|
||||
function init_toc() {
|
||||
qsr('#ml');
|
||||
var loader = ebi('ml');
|
||||
loader.parentNode.removeChild(loader);
|
||||
|
||||
var anchors = []; // list of toc entries, complex objects
|
||||
var anchor = null; // current toc node
|
||||
@@ -395,7 +431,7 @@ function init_toc() {
|
||||
|
||||
// collect vertical position of all toc items (headers in document)
|
||||
function freshen_offsets() {
|
||||
var top = yscroll();
|
||||
var top = window.pageYOffset || document.documentElement.scrollTop;
|
||||
for (var a = anchors.length - 1; a >= 0; a--) {
|
||||
var y = top + anchors[a].elm.getBoundingClientRect().top;
|
||||
y = Math.round(y * 10.0) / 10;
|
||||
@@ -411,7 +447,7 @@ function init_toc() {
|
||||
if (anchors.length == 0)
|
||||
return;
|
||||
|
||||
var ptop = yscroll();
|
||||
var ptop = window.pageYOffset || document.documentElement.scrollTop;
|
||||
var hit = anchors.length - 1;
|
||||
for (var a = 0; a < anchors.length; a++) {
|
||||
if (anchors[a].y >= ptop - 8) { //???
|
||||
@@ -425,11 +461,11 @@ function init_toc() {
|
||||
for (var a = 0; a < anchors.length; a++) {
|
||||
if (anchors[a].active) {
|
||||
anchors[a].active = false;
|
||||
links[a].className = '';
|
||||
links[a].setAttribute('class', '');
|
||||
}
|
||||
}
|
||||
anchors[hit].active = true;
|
||||
links[hit].className = 'act';
|
||||
links[hit].setAttribute('class', 'act');
|
||||
}
|
||||
|
||||
var pane_height = parseInt(getComputedStyle(dom_toc).height);
|
||||
@@ -454,16 +490,13 @@ function init_toc() {
|
||||
// "main" :p
|
||||
convert_markdown(dom_src.value, dom_pre);
|
||||
var toc = init_toc();
|
||||
img_load.callbacks = [toc.refresh];
|
||||
|
||||
|
||||
// scroll handler
|
||||
var redraw = (function () {
|
||||
var sbs = true;
|
||||
var sbs = false;
|
||||
function onresize() {
|
||||
if (window.matchMedia)
|
||||
sbs = window.matchMedia('(min-width: 64em)').matches;
|
||||
|
||||
var y = (dom_hbar.offsetTop + dom_hbar.offsetHeight) + 'px';
|
||||
if (sbs) {
|
||||
dom_toc.style.top = y;
|
||||
@@ -497,6 +530,3 @@ dom_navtgl.onclick = function () {
|
||||
|
||||
if (sread('hidenav') == 1)
|
||||
dom_navtgl.onclick();
|
||||
|
||||
if (window['tt'])
|
||||
tt.init();
|
||||
|
||||
@@ -36,11 +36,6 @@
|
||||
width: 55em;
|
||||
width: min(55em, calc(100% - 2em));
|
||||
}
|
||||
#mtw.single.editor,
|
||||
#mw.single.editor {
|
||||
width: calc(100% - 1em);
|
||||
left: .5em;
|
||||
}
|
||||
|
||||
|
||||
#mp {
|
||||
@@ -55,7 +50,7 @@
|
||||
outline: none;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
font-family: 'scp', monospace, monospace;
|
||||
font-family: 'consolas', monospace, monospace;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
overflow-wrap: break-word;
|
||||
@@ -66,7 +61,7 @@
|
||||
position: relative;
|
||||
scrollbar-color: #eb0 #f7f7f7;
|
||||
}
|
||||
html.z #mt {
|
||||
html.dark #mt {
|
||||
color: #eee;
|
||||
background: #222;
|
||||
border: 1px solid #777;
|
||||
@@ -82,17 +77,20 @@ html.z #mt {
|
||||
background: #f97;
|
||||
border-radius: .15em;
|
||||
}
|
||||
html.z #save.force-save {
|
||||
html.dark #save.force-save {
|
||||
color: #fca;
|
||||
background: #720;
|
||||
}
|
||||
#save.disabled {
|
||||
opacity: .4;
|
||||
}
|
||||
#helpbox {
|
||||
#helpbox,
|
||||
#toast {
|
||||
background: #f7f7f7;
|
||||
border-radius: .4em;
|
||||
z-index: 9001;
|
||||
}
|
||||
#helpbox {
|
||||
display: none;
|
||||
position: fixed;
|
||||
padding: 2em;
|
||||
@@ -107,9 +105,21 @@ html.z #save.force-save {
|
||||
#helpclose {
|
||||
display: block;
|
||||
}
|
||||
html.z #helpbox {
|
||||
html.dark #helpbox {
|
||||
box-shadow: 0 .5em 2em #444;
|
||||
}
|
||||
html.dark #helpbox,
|
||||
html.dark #toast {
|
||||
background: #222;
|
||||
border: 1px solid #079;
|
||||
border-width: 1px 0;
|
||||
}
|
||||
#toast {
|
||||
font-weight: bold;
|
||||
text-align: center;
|
||||
padding: .6em 0;
|
||||
position: fixed;
|
||||
top: 30%;
|
||||
transition: opacity 0.2s ease-in-out;
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
@@ -16,7 +16,8 @@ var dom_sbs = ebi('sbs');
|
||||
var dom_nsbs = ebi('nsbs');
|
||||
var dom_tbox = ebi('toolsbox');
|
||||
var dom_ref = (function () {
|
||||
var d = mknod('div', 'mtr');
|
||||
var d = mknod('div');
|
||||
d.setAttribute('id', 'mtr');
|
||||
dom_swrap.appendChild(d);
|
||||
d = ebi('mtr');
|
||||
// hide behind the textarea (offsetTop is not computed if display:none)
|
||||
@@ -97,7 +98,7 @@ var draw_md = (function () {
|
||||
var src = dom_src.value;
|
||||
convert_markdown(src, dom_pre);
|
||||
|
||||
var lines = esc(src).replace(/\r/g, "").split('\n');
|
||||
var lines = hesc(src).replace(/\r/g, "").split('\n');
|
||||
nlines = lines.length;
|
||||
var html = [];
|
||||
for (var a = 0; a < lines.length; a++)
|
||||
@@ -107,7 +108,7 @@ var draw_md = (function () {
|
||||
map_src = genmap(dom_ref, map_src);
|
||||
map_pre = genmap(dom_pre, map_pre);
|
||||
|
||||
clmod(ebi('save'), 'disabled', src == server_md);
|
||||
cls(ebi('save'), 'disabled', src == server_md);
|
||||
|
||||
var t1 = Date.now();
|
||||
delay = t1 - t0 > 100 ? 25 : 1;
|
||||
@@ -126,12 +127,6 @@ var draw_md = (function () {
|
||||
})();
|
||||
|
||||
|
||||
// discard TOC callback, just regen editor scroll map
|
||||
img_load.callbacks = [function () {
|
||||
map_pre = genmap(dom_pre, map_pre);
|
||||
}];
|
||||
|
||||
|
||||
// resize handler
|
||||
redraw = (function () {
|
||||
function onresize() {
|
||||
@@ -141,18 +136,19 @@ redraw = (function () {
|
||||
dom_ref.style.width = getComputedStyle(dom_src).offsetWidth + 'px';
|
||||
map_src = genmap(dom_ref, map_src);
|
||||
map_pre = genmap(dom_pre, map_pre);
|
||||
dbg(document.body.clientWidth + 'x' + document.body.clientHeight);
|
||||
}
|
||||
function setsbs() {
|
||||
dom_wrap.className = '';
|
||||
dom_swrap.className = '';
|
||||
dom_wrap.setAttribute('class', '');
|
||||
dom_swrap.setAttribute('class', '');
|
||||
onresize();
|
||||
}
|
||||
function modetoggle() {
|
||||
var mode = dom_nsbs.innerHTML;
|
||||
dom_nsbs.innerHTML = mode == 'editor' ? 'preview' : 'editor';
|
||||
mode += ' single';
|
||||
dom_wrap.className = mode;
|
||||
dom_swrap.className = mode;
|
||||
dom_wrap.setAttribute('class', mode);
|
||||
dom_swrap.setAttribute('class', mode);
|
||||
onresize();
|
||||
}
|
||||
|
||||
@@ -229,44 +225,51 @@ redraw = (function () {
|
||||
|
||||
// modification checker
|
||||
function Modpoll() {
|
||||
var r = {
|
||||
skip_one: true,
|
||||
disabled: false
|
||||
};
|
||||
this.skip_one = true;
|
||||
this.disabled = false;
|
||||
|
||||
this.periodic = function () {
|
||||
var that = this;
|
||||
setTimeout(function () {
|
||||
that.periodic();
|
||||
}, 1000 * md_opt.modpoll_freq);
|
||||
|
||||
r.periodic = function () {
|
||||
var skip = null;
|
||||
|
||||
if (toast.visible)
|
||||
if (ebi('toast'))
|
||||
skip = 'toast';
|
||||
|
||||
else if (r.skip_one)
|
||||
else if (this.skip_one)
|
||||
skip = 'saved';
|
||||
|
||||
else if (r.disabled)
|
||||
else if (this.disabled)
|
||||
skip = 'disabled';
|
||||
|
||||
if (skip) {
|
||||
console.log('modpoll skip, ' + skip);
|
||||
r.skip_one = false;
|
||||
this.skip_one = false;
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('modpoll...');
|
||||
var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
|
||||
var xhr = new XHR();
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.modpoll = this;
|
||||
xhr.open('GET', url, true);
|
||||
xhr.responseType = 'text';
|
||||
xhr.onload = xhr.onerror = r.cb;
|
||||
xhr.onreadystatechange = this.cb;
|
||||
xhr.send();
|
||||
};
|
||||
}
|
||||
|
||||
r.cb = function () {
|
||||
if (r.disabled || r.skip_one) {
|
||||
this.cb = function () {
|
||||
if (this.modpoll.disabled || this.modpoll.skip_one) {
|
||||
console.log('modpoll abort');
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.readyState != XMLHttpRequest.DONE)
|
||||
return;
|
||||
|
||||
if (this.status !== 200) {
|
||||
console.log('modpoll err ' + this.status + ": " + this.responseText);
|
||||
return;
|
||||
@@ -280,32 +283,33 @@ function Modpoll() {
|
||||
|
||||
if (server_ref != server_now) {
|
||||
console.log("modpoll diff |" + server_ref.length + "|, |" + server_now.length + "|");
|
||||
r.disabled = true;
|
||||
this.modpoll.disabled = true;
|
||||
var msg = [
|
||||
"The document has changed on the server.",
|
||||
"The document has changed on the server.<br />" +
|
||||
"The changes will NOT be loaded into your editor automatically.",
|
||||
"",
|
||||
"Press F5 or CTRL-R to refresh the page,",
|
||||
|
||||
"Press F5 or CTRL-R to refresh the page,<br />" +
|
||||
"replacing your document with the server copy.",
|
||||
"",
|
||||
"You can close this message to ignore and contnue."
|
||||
|
||||
"You can click this message to ignore and contnue."
|
||||
];
|
||||
return toast.warn(0, msg.join('\n'));
|
||||
return toast(false, "box-shadow:0 1em 2em rgba(64,64,64,0.8);font-weight:normal",
|
||||
36, "<p>" + msg.join('</p>\n<p>') + '</p>');
|
||||
}
|
||||
|
||||
console.log('modpoll eq');
|
||||
};
|
||||
}
|
||||
|
||||
if (md_opt.modpoll_freq > 0)
|
||||
setInterval(r.periodic, 1000 * md_opt.modpoll_freq);
|
||||
this.periodic();
|
||||
|
||||
return r;
|
||||
return this;
|
||||
}
|
||||
var modpoll = new Modpoll();
|
||||
|
||||
|
||||
window.onbeforeunload = function (e) {
|
||||
if ((ebi("save").className + '').indexOf('disabled') >= 0)
|
||||
if ((ebi("save").getAttribute('class') + '').indexOf('disabled') >= 0)
|
||||
return; //nice (todo)
|
||||
|
||||
e.preventDefault(); //ff
|
||||
@@ -317,25 +321,31 @@ window.onbeforeunload = function (e) {
|
||||
function save(e) {
|
||||
if (e) e.preventDefault();
|
||||
var save_btn = ebi("save"),
|
||||
save_cls = save_btn.className + '';
|
||||
save_cls = save_btn.getAttribute('class') + '';
|
||||
|
||||
if (save_cls.indexOf('disabled') >= 0)
|
||||
return toast.inf(2, "no changes");
|
||||
if (save_cls.indexOf('disabled') >= 0) {
|
||||
toast(true, ";font-size:2em;color:#c90", 9, "no changes");
|
||||
return;
|
||||
}
|
||||
|
||||
var force = (save_cls.indexOf('force-save') >= 0);
|
||||
function save2() {
|
||||
var txt = dom_src.value,
|
||||
fd = new FormData();
|
||||
if (force && !confirm('confirm that you wish to lose the changes made on the server since you opened this document')) {
|
||||
alert('ok, aborted');
|
||||
return;
|
||||
}
|
||||
|
||||
var txt = dom_src.value;
|
||||
|
||||
var fd = new FormData();
|
||||
fd.append("act", "tput");
|
||||
fd.append("lastmod", (force ? -1 : last_modified));
|
||||
fd.append("body", txt);
|
||||
|
||||
var url = (document.location + '').split('?')[0];
|
||||
var xhr = new XHR();
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', url, true);
|
||||
xhr.responseType = 'text';
|
||||
xhr.onload = xhr.onerror = save_cb;
|
||||
xhr.onreadystatechange = save_cb;
|
||||
xhr.btn = save_btn;
|
||||
xhr.txt = txt;
|
||||
|
||||
@@ -343,29 +353,27 @@ function save(e) {
|
||||
xhr.send(fd);
|
||||
}
|
||||
|
||||
if (!force)
|
||||
save2();
|
||||
else
|
||||
modal.confirm('confirm that you wish to lose the changes made on the server since you opened this document', save2, function () {
|
||||
toast.inf(3, 'aborted');
|
||||
});
|
||||
}
|
||||
|
||||
function save_cb() {
|
||||
if (this.status !== 200)
|
||||
return toast.err(0, 'Error! The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
|
||||
if (this.readyState != XMLHttpRequest.DONE)
|
||||
return;
|
||||
|
||||
if (this.status !== 200) {
|
||||
alert('Error! The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
|
||||
return;
|
||||
}
|
||||
|
||||
var r;
|
||||
try {
|
||||
r = JSON.parse(this.responseText);
|
||||
}
|
||||
catch (ex) {
|
||||
return toast.err(0, 'Failed to parse reply from server:\n\n' + this.responseText);
|
||||
alert('Failed to parse reply from server:\n\n' + this.responseText);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!r.ok) {
|
||||
if (!clgot(this.btn, 'force-save')) {
|
||||
clmod(this.btn, 'force-save', 1);
|
||||
if (!this.btn.classList.contains('force-save')) {
|
||||
this.btn.classList.add('force-save');
|
||||
var msg = [
|
||||
'This file has been modified since you started editing it!\n',
|
||||
'if you really want to overwrite, press save again.\n',
|
||||
@@ -375,13 +383,15 @@ function save_cb() {
|
||||
r.lastmod + ' lastmod on the server now,',
|
||||
r.now + ' server time now,\n',
|
||||
];
|
||||
return toast.err(0, msg.join('\n'));
|
||||
alert(msg.join('\n'));
|
||||
}
|
||||
else
|
||||
return toast.err(0, 'Error! Save failed. Maybe this JSON explains why:\n\n' + this.responseText);
|
||||
else {
|
||||
alert('Error! Save failed. Maybe this JSON explains why:\n\n' + this.responseText);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
clmod(this.btn, 'force-save');
|
||||
this.btn.classList.remove('force-save');
|
||||
//alert('save OK -- wrote ' + r.size + ' bytes.\n\nsha512: ' + r.sha512);
|
||||
|
||||
run_savechk(r.lastmod, this.txt, this.btn, 0);
|
||||
@@ -390,10 +400,10 @@ function save_cb() {
|
||||
function run_savechk(lastmod, txt, btn, ntry) {
|
||||
// download the saved doc from the server and compare
|
||||
var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
|
||||
var xhr = new XHR();
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open('GET', url, true);
|
||||
xhr.responseType = 'text';
|
||||
xhr.onload = xhr.onerror = savechk_cb;
|
||||
xhr.onreadystatechange = savechk_cb;
|
||||
xhr.lastmod = lastmod;
|
||||
xhr.txt = txt;
|
||||
xhr.btn = btn;
|
||||
@@ -402,8 +412,13 @@ function run_savechk(lastmod, txt, btn, ntry) {
|
||||
}
|
||||
|
||||
function savechk_cb() {
|
||||
if (this.status !== 200)
|
||||
return toast.err(0, 'Error! The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
|
||||
if (this.readyState != XMLHttpRequest.DONE)
|
||||
return;
|
||||
|
||||
if (this.status !== 200) {
|
||||
alert('Error! The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
|
||||
return;
|
||||
}
|
||||
|
||||
var doc1 = this.txt.replace(/\r\n/g, "\n");
|
||||
var doc2 = this.responseText.replace(/\r\n/g, "\n");
|
||||
@@ -416,22 +431,58 @@ function savechk_cb() {
|
||||
}, 100);
|
||||
return;
|
||||
}
|
||||
modal.alert(
|
||||
alert(
|
||||
'Error! The document on the server does not appear to have saved correctly (your editor contents and the server copy is not identical). Place the document on your clipboard for now and check the server logs for hints\n\n' +
|
||||
'Length: yours=' + doc1.length + ', server=' + doc2.length
|
||||
);
|
||||
modal.alert('yours, ' + doc1.length + ' byte:\n[' + doc1 + ']');
|
||||
modal.alert('server, ' + doc2.length + ' byte:\n[' + doc2 + ']');
|
||||
alert('yours, ' + doc1.length + ' byte:\n[' + doc1 + ']');
|
||||
alert('server, ' + doc2.length + ' byte:\n[' + doc2 + ']');
|
||||
return;
|
||||
}
|
||||
|
||||
last_modified = this.lastmod;
|
||||
server_md = this.txt;
|
||||
draw_md();
|
||||
toast.ok(2, 'save OK' + (this.ntry ? '\nattempt ' + this.ntry : ''));
|
||||
toast(true, ";font-size:6em;font-family:serif;color:#9b4", 4,
|
||||
'OK✔️<span style="font-size:.2em;color:#999;position:absolute">' + this.ntry + '</span>');
|
||||
|
||||
modpoll.disabled = false;
|
||||
}
|
||||
|
||||
function toast(autoclose, style, width, msg) {
|
||||
var ok = ebi("toast");
|
||||
if (ok)
|
||||
ok.parentNode.removeChild(ok);
|
||||
|
||||
style = "width:" + width + "em;left:calc(50% - " + (width / 2) + "em);" + style;
|
||||
ok = mknod('div');
|
||||
ok.setAttribute('id', 'toast');
|
||||
ok.setAttribute('style', style);
|
||||
ok.innerHTML = msg;
|
||||
var parent = ebi('m');
|
||||
document.documentElement.appendChild(ok);
|
||||
|
||||
var hide = function (delay) {
|
||||
delay = delay || 0;
|
||||
|
||||
setTimeout(function () {
|
||||
ok.style.opacity = 0;
|
||||
}, delay);
|
||||
|
||||
setTimeout(function () {
|
||||
if (ok.parentNode)
|
||||
ok.parentNode.removeChild(ok);
|
||||
}, delay + 250);
|
||||
}
|
||||
|
||||
ok.onclick = function () {
|
||||
hide(0);
|
||||
};
|
||||
|
||||
if (autoclose)
|
||||
hide(500);
|
||||
}
|
||||
|
||||
|
||||
// firefox bug: initial selection offset isn't cleared properly through js
|
||||
var ff_clearsel = (function () {
|
||||
@@ -508,20 +559,6 @@ function setsel(s) {
|
||||
}
|
||||
|
||||
|
||||
// cut/copy current line
|
||||
function md_cut(cut) {
|
||||
var s = linebounds();
|
||||
if (s.car != s.cdr)
|
||||
return;
|
||||
|
||||
dom_src.setSelectionRange(s.n1, s.n2 + 1, 'forward');
|
||||
setTimeout(function () {
|
||||
var i = cut ? s.n1 : s.car;
|
||||
dom_src.setSelectionRange(i, i, 'forward');
|
||||
}, 1);
|
||||
}
|
||||
|
||||
|
||||
// indent/dedent
|
||||
function md_indent(dedent) {
|
||||
var s = getsel(),
|
||||
@@ -682,7 +719,7 @@ function reLastIndexOf(txt, ptn, end) {
|
||||
// table formatter
|
||||
function fmt_table(e) {
|
||||
if (e) e.preventDefault();
|
||||
//dom_tbox.className = '';
|
||||
//dom_tbox.setAttribute('class', '');
|
||||
|
||||
var txt = dom_src.value,
|
||||
ofs = dom_src.selectionStart,
|
||||
@@ -724,7 +761,7 @@ function fmt_table(e) {
|
||||
|
||||
var ind2 = tab[a].match(re_ind)[0];
|
||||
if (ind != ind2 && a != 1) // the table can be a list entry or something, ignore [0]
|
||||
return toast.err(7, err + 'indentation mismatch on row#2 and ' + row_name + ',\n' + tab[a]);
|
||||
return alert(err + 'indentation mismatch on row#2 and ' + row_name + ',\n' + tab[a]);
|
||||
|
||||
var t = tab[a].slice(ind.length);
|
||||
t = t.replace(re_lpipe, "");
|
||||
@@ -734,7 +771,7 @@ function fmt_table(e) {
|
||||
if (a == 0)
|
||||
ncols = tab[a].length;
|
||||
else if (ncols < tab[a].length)
|
||||
return toast.err(7, err + 'num.columns(' + row_name + ') exceeding row#2; ' + ncols + ' < ' + tab[a].length);
|
||||
return alert(err + 'num.columns(' + row_name + ') exceeding row#2; ' + ncols + ' < ' + tab[a].length);
|
||||
|
||||
// if row has less columns than row2, fill them in
|
||||
while (tab[a].length < ncols)
|
||||
@@ -751,7 +788,7 @@ function fmt_table(e) {
|
||||
for (var col = 0; col < tab[1].length; col++) {
|
||||
var m = tab[1][col].match(re_align);
|
||||
if (!m)
|
||||
return toast.err(7, err + 'invalid column specification, row#2, col ' + (col + 1) + ', [' + tab[1][col] + ']');
|
||||
return alert(err + 'invalid column specification, row#2, col ' + (col + 1) + ', [' + tab[1][col] + ']');
|
||||
|
||||
if (m[2]) {
|
||||
if (m[1])
|
||||
@@ -833,15 +870,16 @@ function fmt_table(e) {
|
||||
// show unicode
|
||||
function mark_uni(e) {
|
||||
if (e) e.preventDefault();
|
||||
dom_tbox.className = '';
|
||||
dom_tbox.setAttribute('class', '');
|
||||
|
||||
var txt = dom_src.value,
|
||||
ptn = new RegExp('([^' + js_uni_whitelist + ']+)', 'g'),
|
||||
mod = txt.replace(/\r/g, "").replace(ptn, "\u2588\u2770$1\u2771");
|
||||
|
||||
if (txt == mod)
|
||||
return toast.inf(5, 'no results; no modifications were made');
|
||||
|
||||
if (txt == mod) {
|
||||
alert('no results; no modifications were made');
|
||||
return;
|
||||
}
|
||||
dom_src.value = mod;
|
||||
}
|
||||
|
||||
@@ -855,9 +893,10 @@ function iter_uni(e) {
|
||||
re = new RegExp('([^' + js_uni_whitelist + ']+)'),
|
||||
m = re.exec(txt.slice(ofs));
|
||||
|
||||
if (!m)
|
||||
return toast.inf(5, 'no more hits from cursor onwards');
|
||||
|
||||
if (!m) {
|
||||
alert('no more hits from cursor onwards');
|
||||
return;
|
||||
}
|
||||
ofs += m.index;
|
||||
|
||||
dom_src.setSelectionRange(ofs, ofs + m[0].length, "forward");
|
||||
@@ -872,54 +911,23 @@ function iter_uni(e) {
|
||||
function cfg_uni(e) {
|
||||
if (e) e.preventDefault();
|
||||
|
||||
modal.prompt("unicode whitelist", esc_uni_whitelist, function (reply) {
|
||||
esc_uni_whitelist = reply;
|
||||
js_uni_whitelist = eval('\'' + esc_uni_whitelist + '\'');
|
||||
}, null);
|
||||
}
|
||||
|
||||
|
||||
var set_lno = (function () {
|
||||
var t = null,
|
||||
pi = null,
|
||||
pv = null,
|
||||
lno = ebi('lno');
|
||||
|
||||
function poke() {
|
||||
clearTimeout(t);
|
||||
t = setTimeout(fire, 20);
|
||||
}
|
||||
|
||||
function fire() {
|
||||
try {
|
||||
clearTimeout(t);
|
||||
|
||||
var i = dom_src.selectionStart;
|
||||
if (i === pi)
|
||||
var reply = prompt("unicode whitelist", esc_uni_whitelist);
|
||||
if (reply === null)
|
||||
return;
|
||||
|
||||
var v = 'L' + dom_src.value.slice(0, i).split('\n').length;
|
||||
if (v != pv)
|
||||
lno.innerHTML = v;
|
||||
|
||||
pi = i;
|
||||
pv = v;
|
||||
esc_uni_whitelist = reply;
|
||||
js_uni_whitelist = eval('\'' + esc_uni_whitelist + '\'');
|
||||
}
|
||||
catch (e) { }
|
||||
}
|
||||
|
||||
timer.add(fire);
|
||||
return poke;
|
||||
})();
|
||||
|
||||
|
||||
// hotkeys / toolbar
|
||||
(function () {
|
||||
function keydown(ev) {
|
||||
ev = ev || window.event;
|
||||
var kc = ev.code || ev.keyCode || ev.which;
|
||||
//console.log(ev.key, ev.code, ev.keyCode, ev.which);
|
||||
if (ctrl(ev) && (ev.code == "KeyS" || kc == 83)) {
|
||||
var kc = ev.keyCode || ev.which;
|
||||
var ctrl = ev.ctrlKey || ev.metaKey;
|
||||
//console.log(ev.code, kc);
|
||||
if (ctrl && (ev.code == "KeyS" || kc == 83)) {
|
||||
save();
|
||||
return false;
|
||||
}
|
||||
@@ -928,17 +936,23 @@ var set_lno = (function () {
|
||||
if (d)
|
||||
d.click();
|
||||
}
|
||||
if (document.activeElement != dom_src)
|
||||
return true;
|
||||
|
||||
set_lno();
|
||||
|
||||
if (ctrl(ev)) {
|
||||
if (ev.code == "KeyH" || kc == 72) {
|
||||
if (document.activeElement == dom_src) {
|
||||
if (ev.code == "Tab" || kc == 9) {
|
||||
md_indent(ev.shiftKey);
|
||||
return false;
|
||||
}
|
||||
if (ctrl && (ev.code == "KeyH" || kc == 72)) {
|
||||
md_header(ev.shiftKey);
|
||||
return false;
|
||||
}
|
||||
if (ev.code == "KeyZ" || kc == 90) {
|
||||
if (!ctrl && (ev.code == "Home" || kc == 36)) {
|
||||
md_home(ev.shiftKey);
|
||||
return false;
|
||||
}
|
||||
if (!ctrl && !ev.shiftKey && (ev.code == "Enter" || kc == 13)) {
|
||||
return md_newline();
|
||||
}
|
||||
if (ctrl && (ev.code == "KeyZ" || kc == 90)) {
|
||||
if (ev.shiftKey)
|
||||
action_stack.redo();
|
||||
else
|
||||
@@ -946,48 +960,32 @@ var set_lno = (function () {
|
||||
|
||||
return false;
|
||||
}
|
||||
if (ev.code == "KeyY" || kc == 89) {
|
||||
if (ctrl && (ev.code == "KeyY" || kc == 89)) {
|
||||
action_stack.redo();
|
||||
return false;
|
||||
}
|
||||
if (ev.code == "KeyK") {
|
||||
if (!ctrl && !ev.shiftKey && kc == 8) {
|
||||
return md_backspace();
|
||||
}
|
||||
if (ctrl && (ev.code == "KeyK")) {
|
||||
fmt_table();
|
||||
return false;
|
||||
}
|
||||
if (ev.code == "KeyU") {
|
||||
if (ctrl && (ev.code == "KeyU")) {
|
||||
iter_uni();
|
||||
return false;
|
||||
}
|
||||
if (ev.code == "KeyE") {
|
||||
if (ctrl && (ev.code == "KeyE")) {
|
||||
dom_nsbs.click();
|
||||
//fmt_table();
|
||||
return false;
|
||||
}
|
||||
var up = ev.code == "ArrowUp" || kc == 38;
|
||||
var dn = ev.code == "ArrowDown" || kc == 40;
|
||||
if (up || dn) {
|
||||
if (ctrl && (up || dn)) {
|
||||
md_p_jump(dn);
|
||||
return false;
|
||||
}
|
||||
if (ev.code == "KeyX" || ev.code == "KeyC") {
|
||||
md_cut(ev.code == "KeyX");
|
||||
return true; //sic
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (ev.code == "Tab" || kc == 9) {
|
||||
md_indent(ev.shiftKey);
|
||||
return false;
|
||||
}
|
||||
if (ev.code == "Home" || kc == 36) {
|
||||
md_home(ev.shiftKey);
|
||||
return false;
|
||||
}
|
||||
if (!ev.shiftKey && (ev.code == "Enter" || kc == 13)) {
|
||||
return md_newline();
|
||||
}
|
||||
if (!ev.shiftKey && kc == 8) {
|
||||
return md_backspace();
|
||||
}
|
||||
}
|
||||
}
|
||||
document.onkeydown = keydown;
|
||||
@@ -997,14 +995,14 @@ var set_lno = (function () {
|
||||
|
||||
ebi('tools').onclick = function (e) {
|
||||
if (e) e.preventDefault();
|
||||
var is_open = dom_tbox.className != 'open';
|
||||
dom_tbox.className = is_open ? 'open' : '';
|
||||
var is_open = dom_tbox.getAttribute('class') != 'open';
|
||||
dom_tbox.setAttribute('class', is_open ? 'open' : '');
|
||||
};
|
||||
|
||||
|
||||
ebi('help').onclick = function (e) {
|
||||
if (e) e.preventDefault();
|
||||
dom_tbox.className = '';
|
||||
dom_tbox.setAttribute('class', '');
|
||||
|
||||
var dom = ebi('helpbox');
|
||||
var dtxt = dom.getElementsByTagName('textarea');
|
||||
@@ -1131,9 +1129,9 @@ action_stack = (function () {
|
||||
ref = newtxt;
|
||||
dbg('undos(%d) redos(%d)', hist.un.length, hist.re.length);
|
||||
if (hist.un.length > 0)
|
||||
dbg(jcp(hist.un.slice(-1)[0]));
|
||||
dbg(statify(hist.un.slice(-1)[0]));
|
||||
if (hist.re.length > 0)
|
||||
dbg(jcp(hist.re.slice(-1)[0]));
|
||||
dbg(statify(hist.re.slice(-1)[0]));
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -7,8 +7,6 @@ html .editor-toolbar>button.active { border-color: rgba(0,0,0,0.4); background:
|
||||
html .editor-toolbar>i.separator { border-left: 1px solid #ccc; }
|
||||
html .editor-toolbar.disabled-for-preview>button:not(.no-disable) { opacity: .35 }
|
||||
|
||||
|
||||
|
||||
html {
|
||||
line-height: 1.5em;
|
||||
}
|
||||
@@ -20,22 +18,6 @@ html, body {
|
||||
background: #f7f7f7;
|
||||
color: #333;
|
||||
}
|
||||
#toast {
|
||||
bottom: auto;
|
||||
top: 1.4em;
|
||||
}
|
||||
#repl {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
right: .5em;
|
||||
border: none;
|
||||
color: inherit;
|
||||
background: none;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#mn {
|
||||
font-weight: normal;
|
||||
margin: 1.3em 0 .7em 1em;
|
||||
@@ -77,79 +59,253 @@ html .editor-toolbar>button.disabled {
|
||||
html .editor-toolbar>button.save.force-save {
|
||||
background: #f97;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/* copied from md.css for now */
|
||||
.mdo pre,
|
||||
.mdo code,
|
||||
.mdo a {
|
||||
color: #480;
|
||||
background: #f7f7f7;
|
||||
border: .07em solid #ddd;
|
||||
border-radius: .2em;
|
||||
padding: .1em .3em;
|
||||
margin: 0 .1em;
|
||||
}
|
||||
.mdo code {
|
||||
font-size: .96em;
|
||||
}
|
||||
.mdo pre,
|
||||
.mdo code {
|
||||
font-family: monospace, monospace;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-all;
|
||||
}
|
||||
.mdo pre code {
|
||||
display: block;
|
||||
margin: 0 -.3em;
|
||||
padding: .4em .5em;
|
||||
line-height: 1.1em;
|
||||
}
|
||||
.mdo a {
|
||||
color: #fff;
|
||||
background: #39b;
|
||||
text-decoration: none;
|
||||
padding: 0 .3em;
|
||||
border: none;
|
||||
border-bottom: .07em solid #079;
|
||||
}
|
||||
.mdo h2 {
|
||||
color: #fff;
|
||||
background: #555;
|
||||
margin-top: 2em;
|
||||
border-bottom: .22em solid #999;
|
||||
border-top: none;
|
||||
}
|
||||
.mdo h1 {
|
||||
color: #fff;
|
||||
background: #444;
|
||||
font-weight: normal;
|
||||
border-top: .4em solid #fb0;
|
||||
border-bottom: .4em solid #777;
|
||||
border-radius: 0 1em 0 1em;
|
||||
margin: 3em 0 1em 0;
|
||||
padding: .5em 0;
|
||||
}
|
||||
h1, h2 {
|
||||
line-height: 1.5em;
|
||||
}
|
||||
h1 {
|
||||
font-size: 1.7em;
|
||||
text-align: center;
|
||||
border: 1em solid #777;
|
||||
border-width: .05em 0;
|
||||
margin: 3em 0;
|
||||
}
|
||||
h2 {
|
||||
font-size: 1.5em;
|
||||
font-weight: normal;
|
||||
background: #f7f7f7;
|
||||
border-top: .07em solid #fff;
|
||||
border-bottom: .07em solid #bbb;
|
||||
border-radius: .5em .5em 0 0;
|
||||
padding-left: .4em;
|
||||
margin-top: 3em;
|
||||
}
|
||||
.mdo ul,
|
||||
.mdo ol {
|
||||
border-left: .3em solid #ddd;
|
||||
}
|
||||
.mdo>ul,
|
||||
.mdo>ol {
|
||||
border-color: #bbb;
|
||||
}
|
||||
.mdo ul>li {
|
||||
list-style-type: disc;
|
||||
}
|
||||
.mdo ul>li,
|
||||
.mdo ol>li {
|
||||
margin: .7em 0;
|
||||
}
|
||||
strong {
|
||||
color: #000;
|
||||
}
|
||||
p>em,
|
||||
li>em,
|
||||
td>em {
|
||||
color: #c50;
|
||||
padding: .1em;
|
||||
border-bottom: .1em solid #bbb;
|
||||
}
|
||||
blockquote {
|
||||
font-family: serif;
|
||||
background: #f7f7f7;
|
||||
border: .07em dashed #ccc;
|
||||
padding: 0 2em;
|
||||
margin: 1em 0;
|
||||
}
|
||||
small {
|
||||
opacity: .8;
|
||||
}
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
}
|
||||
td {
|
||||
padding: .2em .5em;
|
||||
border: .12em solid #aaa;
|
||||
}
|
||||
th {
|
||||
border: .12em solid #aaa;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/* mde support */
|
||||
.mdo {
|
||||
padding: 1em;
|
||||
background: #f7f7f7;
|
||||
}
|
||||
html.dark .mdo {
|
||||
background: #1c1c1c;
|
||||
}
|
||||
.CodeMirror {
|
||||
background: #f7f7f7;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/* darkmode */
|
||||
html.z .mdo,
|
||||
html.z .CodeMirror {
|
||||
html.dark .mdo,
|
||||
html.dark .CodeMirror {
|
||||
border-color: #222;
|
||||
}
|
||||
html.z,
|
||||
html.z body,
|
||||
html.z .CodeMirror {
|
||||
html.dark,
|
||||
html.dark body,
|
||||
html.dark .CodeMirror {
|
||||
background: #222;
|
||||
color: #ccc;
|
||||
}
|
||||
html.z .CodeMirror-cursor {
|
||||
html.dark .CodeMirror-cursor {
|
||||
border-color: #fff;
|
||||
}
|
||||
html.z .CodeMirror-selected {
|
||||
html.dark .CodeMirror-selected {
|
||||
box-shadow: 0 0 1px #0cf inset;
|
||||
}
|
||||
html.z .CodeMirror-selected,
|
||||
html.z .CodeMirror-selectedtext {
|
||||
html.dark .CodeMirror-selected,
|
||||
html.dark .CodeMirror-selectedtext {
|
||||
border-radius: .1em;
|
||||
background: #246;
|
||||
color: #fff;
|
||||
}
|
||||
html.dark .mdo a {
|
||||
background: #057;
|
||||
}
|
||||
html.dark .mdo h1 a, html.dark .mdo h4 a,
|
||||
html.dark .mdo h2 a, html.dark .mdo h5 a,
|
||||
html.dark .mdo h3 a, html.dark .mdo h6 a {
|
||||
color: inherit;
|
||||
background: none;
|
||||
}
|
||||
html.dark pre,
|
||||
html.dark code {
|
||||
color: #8c0;
|
||||
background: #1a1a1a;
|
||||
border: .07em solid #333;
|
||||
}
|
||||
html.dark .mdo ul,
|
||||
html.dark .mdo ol {
|
||||
border-color: #444;
|
||||
}
|
||||
html.dark .mdo>ul,
|
||||
html.dark .mdo>ol {
|
||||
border-color: #555;
|
||||
}
|
||||
html.dark strong {
|
||||
color: #fff;
|
||||
}
|
||||
html.dark p>em,
|
||||
html.dark li>em,
|
||||
html.dark td>em {
|
||||
color: #f94;
|
||||
border-color: #666;
|
||||
}
|
||||
html.dark h1 {
|
||||
background: #383838;
|
||||
border-top: .4em solid #b80;
|
||||
border-bottom: .4em solid #4c4c4c;
|
||||
}
|
||||
html.dark h2 {
|
||||
background: #444;
|
||||
border-bottom: .22em solid #555;
|
||||
}
|
||||
html.dark td,
|
||||
html.dark th {
|
||||
border-color: #444;
|
||||
}
|
||||
html.dark blockquote {
|
||||
background: #282828;
|
||||
border: .07em dashed #444;
|
||||
}
|
||||
|
||||
|
||||
|
||||
html.z #mn a {
|
||||
html.dark #mn a {
|
||||
color: #ccc;
|
||||
}
|
||||
html.z #mn a:not(:last-child):after {
|
||||
html.dark #mn a:not(:last-child):after {
|
||||
border-color: rgba(255,255,255,0.3);
|
||||
}
|
||||
html.z .editor-toolbar {
|
||||
html.dark .editor-toolbar {
|
||||
border-color: #2c2c2c;
|
||||
background: #1c1c1c;
|
||||
}
|
||||
html.z .editor-toolbar>i.separator {
|
||||
html.dark .editor-toolbar>i.separator {
|
||||
border-left: 1px solid #444;
|
||||
border-right: 1px solid #111;
|
||||
}
|
||||
html.z .editor-toolbar>button {
|
||||
html.dark .editor-toolbar>button {
|
||||
margin-left: -1px; border: 1px solid rgba(255,255,255,0.1);
|
||||
color: #aaa;
|
||||
}
|
||||
|
||||
|
||||
|
||||
html.z .editor-toolbar>button:hover {
|
||||
html.dark .editor-toolbar>button:hover {
|
||||
color: #333;
|
||||
}
|
||||
html.z .editor-toolbar>button.active {
|
||||
html.dark .editor-toolbar>button.active {
|
||||
color: #333;
|
||||
border-color: #ec1;
|
||||
background: #c90;
|
||||
}
|
||||
html.z .editor-toolbar::after,
|
||||
html.z .editor-toolbar::before {
|
||||
html.dark .editor-toolbar::after,
|
||||
html.dark .editor-toolbar::before {
|
||||
background: none;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* ui.css overrides */
|
||||
.mdo {
|
||||
padding: 1em;
|
||||
background: #f7f7f7;
|
||||
}
|
||||
html.z .mdo {
|
||||
background: #1c1c1c;
|
||||
}
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
<!DOCTYPE html><html><head>
|
||||
<meta charset="utf-8">
|
||||
<title>📝 {{ title }}</title>
|
||||
<title>📝🎉 {{ title }}</title>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=0.7">
|
||||
{{ html_head }}
|
||||
<link rel="stylesheet" href="/.cpr/ui.css?_={{ ts }}">
|
||||
<link rel="stylesheet" href="/.cpr/mde.css?_={{ ts }}">
|
||||
<link rel="stylesheet" href="/.cpr/deps/mini-fa.css?_={{ ts }}">
|
||||
<link rel="stylesheet" href="/.cpr/deps/easymde.css?_={{ ts }}">
|
||||
<link href="/.cpr/mde.css?_={{ ts }}" rel="stylesheet">
|
||||
<link href="/.cpr/deps/mini-fa.css?_={{ ts }}" rel="stylesheet">
|
||||
<link href="/.cpr/deps/easymde.css?_={{ ts }}" rel="stylesheet">
|
||||
</head>
|
||||
<body>
|
||||
<div id="mw">
|
||||
@@ -22,33 +20,30 @@
|
||||
<textarea id="mt" style="display:none" autocomplete="off">{{ md }}</textarea>
|
||||
</div>
|
||||
</div>
|
||||
<a href="#" id="repl">π</a>
|
||||
<script>
|
||||
|
||||
var last_modified = {{ lastmod }},
|
||||
have_emp = {{ have_emp|tojson }},
|
||||
dfavico = "{{ favico }}";
|
||||
|
||||
var last_modified = {{ lastmod }};
|
||||
var md_opt = {
|
||||
link_md_as_html: false,
|
||||
allow_plugins: {{ md_plug }},
|
||||
modpoll_freq: {{ md_chk_rate }}
|
||||
};
|
||||
|
||||
var lightswitch = (function () {
|
||||
var l = localStorage,
|
||||
drk = l.light != 1,
|
||||
f = function (e) {
|
||||
if (e) drk = !drk;
|
||||
document.documentElement.className = drk? "z":"y";
|
||||
l.light = drk? 0:1;
|
||||
var fun = function () {
|
||||
var dark = !document.documentElement.getAttribute("class");
|
||||
document.documentElement.setAttribute("class", dark ? "dark" : "");
|
||||
if (window.localStorage)
|
||||
localStorage.setItem('lightmode', dark ? 0 : 1);
|
||||
};
|
||||
f();
|
||||
return f;
|
||||
if (window.localStorage && localStorage.getItem('lightmode') != 1)
|
||||
fun();
|
||||
|
||||
return fun;
|
||||
})();
|
||||
|
||||
</script>
|
||||
<script src="/.cpr/util.js?_={{ ts }}"></script>
|
||||
<script src="/.cpr/deps/marked.js?_={{ ts }}"></script>
|
||||
<script src="/.cpr/deps/easymde.js?_={{ ts }}"></script>
|
||||
<script src="/.cpr/mde.js?_={{ ts }}"></script>
|
||||
</body></html>
|
||||
|
||||
@@ -15,7 +15,7 @@ var dom_md = ebi('mt');
|
||||
if (a > 0)
|
||||
loc.push(n[a]);
|
||||
|
||||
var dec = uricom_dec(n[a]).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
||||
var dec = uricom_dec(n[a])[0].replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
||||
|
||||
nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
|
||||
}
|
||||
@@ -65,7 +65,8 @@ var mde = (function () {
|
||||
mde.codemirror.on("change", function () {
|
||||
md_changed(mde);
|
||||
});
|
||||
qsr('#ml');
|
||||
var loader = ebi('ml');
|
||||
loader.parentNode.removeChild(loader);
|
||||
return mde;
|
||||
})();
|
||||
|
||||
@@ -74,7 +75,7 @@ function set_jumpto() {
|
||||
}
|
||||
|
||||
function jumpto(ev) {
|
||||
var tgt = ev.target;
|
||||
var tgt = ev.target || ev.srcElement;
|
||||
var ln = null;
|
||||
while (tgt && !ln) {
|
||||
ln = tgt.getAttribute('data-ln');
|
||||
@@ -95,17 +96,26 @@ function md_changed(mde, on_srv) {
|
||||
var md_now = mde.value();
|
||||
var save_btn = QS('.editor-toolbar button.save');
|
||||
|
||||
clmod(save_btn, 'disabled', md_now == window.md_saved);
|
||||
if (md_now == window.md_saved)
|
||||
save_btn.classList.add('disabled');
|
||||
else
|
||||
save_btn.classList.remove('disabled');
|
||||
|
||||
set_jumpto();
|
||||
}
|
||||
|
||||
function save(mde) {
|
||||
var save_btn = QS('.editor-toolbar button.save');
|
||||
if (clgot(save_btn, 'disabled'))
|
||||
return toast.inf(2, 'no changes');
|
||||
if (save_btn.classList.contains('disabled')) {
|
||||
alert('there is nothing to save');
|
||||
return;
|
||||
}
|
||||
var force = save_btn.classList.contains('force-save');
|
||||
if (force && !confirm('confirm that you wish to lose the changes made on the server since you opened this document')) {
|
||||
alert('ok, aborted');
|
||||
return;
|
||||
}
|
||||
|
||||
var force = clgot(save_btn, 'force-save');
|
||||
function save2() {
|
||||
var txt = mde.value();
|
||||
|
||||
var fd = new FormData();
|
||||
@@ -114,39 +124,37 @@ function save(mde) {
|
||||
fd.append("body", txt);
|
||||
|
||||
var url = (document.location + '').split('?')[0];
|
||||
var xhr = new XHR();
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', url, true);
|
||||
xhr.responseType = 'text';
|
||||
xhr.onload = xhr.onerror = save_cb;
|
||||
xhr.onreadystatechange = save_cb;
|
||||
xhr.btn = save_btn;
|
||||
xhr.mde = mde;
|
||||
xhr.txt = txt;
|
||||
xhr.send(fd);
|
||||
}
|
||||
|
||||
if (!force)
|
||||
save2();
|
||||
else
|
||||
modal.confirm('confirm that you wish to lose the changes made on the server since you opened this document', save2, function () {
|
||||
toast.inf(3, 'aborted');
|
||||
});
|
||||
}
|
||||
|
||||
function save_cb() {
|
||||
if (this.status !== 200)
|
||||
return toast.err(0, 'Error! The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
|
||||
if (this.readyState != XMLHttpRequest.DONE)
|
||||
return;
|
||||
|
||||
if (this.status !== 200) {
|
||||
alert('Error! The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
|
||||
return;
|
||||
}
|
||||
|
||||
var r;
|
||||
try {
|
||||
r = JSON.parse(this.responseText);
|
||||
}
|
||||
catch (ex) {
|
||||
return toast.err(0, 'Failed to parse reply from server:\n\n' + this.responseText);
|
||||
alert('Failed to parse reply from server:\n\n' + this.responseText);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!r.ok) {
|
||||
if (!clgot(this.btn, 'force-save')) {
|
||||
clmod(this.btn, 'force-save', 1);
|
||||
if (!this.btn.classList.contains('force-save')) {
|
||||
this.btn.classList.add('force-save');
|
||||
var msg = [
|
||||
'This file has been modified since you started editing it!\n',
|
||||
'if you really want to overwrite, press save again.\n',
|
||||
@@ -156,21 +164,23 @@ function save_cb() {
|
||||
r.lastmod + ' lastmod on the server now,',
|
||||
r.now + ' server time now,\n',
|
||||
];
|
||||
return toast.err(0, msg.join('\n'));
|
||||
alert(msg.join('\n'));
|
||||
}
|
||||
else
|
||||
return toast.err(0, 'Error! Save failed. Maybe this JSON explains why:\n\n' + this.responseText);
|
||||
else {
|
||||
alert('Error! Save failed. Maybe this JSON explains why:\n\n' + this.responseText);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
clmod(this.btn, 'force-save');
|
||||
this.btn.classList.remove('force-save');
|
||||
//alert('save OK -- wrote ' + r.size + ' bytes.\n\nsha512: ' + r.sha512);
|
||||
|
||||
// download the saved doc from the server and compare
|
||||
var url = (document.location + '').split('?')[0] + '?raw';
|
||||
var xhr = new XHR();
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open('GET', url, true);
|
||||
xhr.responseType = 'text';
|
||||
xhr.onload = xhr.onerror = save_chk;
|
||||
xhr.onreadystatechange = save_chk;
|
||||
xhr.btn = this.save_btn;
|
||||
xhr.mde = this.mde;
|
||||
xhr.txt = this.txt;
|
||||
@@ -179,23 +189,38 @@ function save_cb() {
|
||||
}
|
||||
|
||||
function save_chk() {
|
||||
if (this.status !== 200)
|
||||
return toast.err(0, 'Error! The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
|
||||
if (this.readyState != XMLHttpRequest.DONE)
|
||||
return;
|
||||
|
||||
if (this.status !== 200) {
|
||||
alert('Error! The file was NOT saved.\n\n' + this.status + ": " + (this.responseText + '').replace(/^<pre>/, ""));
|
||||
return;
|
||||
}
|
||||
|
||||
var doc1 = this.txt.replace(/\r\n/g, "\n");
|
||||
var doc2 = this.responseText.replace(/\r\n/g, "\n");
|
||||
if (doc1 != doc2) {
|
||||
modal.alert(
|
||||
alert(
|
||||
'Error! The document on the server does not appear to have saved correctly (your editor contents and the server copy is not identical). Place the document on your clipboard for now and check the server logs for hints\n\n' +
|
||||
'Length: yours=' + doc1.length + ', server=' + doc2.length
|
||||
);
|
||||
modal.alert('yours, ' + doc1.length + ' byte:\n[' + doc1 + ']');
|
||||
modal.alert('server, ' + doc2.length + ' byte:\n[' + doc2 + ']');
|
||||
alert('yours, ' + doc1.length + ' byte:\n[' + doc1 + ']');
|
||||
alert('server, ' + doc2.length + ' byte:\n[' + doc2 + ']');
|
||||
return;
|
||||
}
|
||||
|
||||
last_modified = this.lastmod;
|
||||
md_changed(this.mde, true);
|
||||
|
||||
toast.ok(2, 'save OK' + (this.ntry ? '\nattempt ' + this.ntry : ''));
|
||||
var ok = mknod('div');
|
||||
ok.setAttribute('style', 'font-size:6em;font-family:serif;font-weight:bold;color:#cf6;background:#444;border-radius:.3em;padding:.6em 0;position:fixed;top:30%;left:calc(50% - 2em);width:4em;text-align:center;z-index:9001;transition:opacity 0.2s ease-in-out;opacity:1');
|
||||
ok.innerHTML = 'OK✔️';
|
||||
var parent = ebi('m');
|
||||
document.documentElement.appendChild(ok);
|
||||
setTimeout(function () {
|
||||
ok.style.opacity = 0;
|
||||
}, 500);
|
||||
setTimeout(function () {
|
||||
ok.parentNode.removeChild(ok);
|
||||
}, 750);
|
||||
}
|
||||
|
||||
@@ -11,12 +11,14 @@ html {
|
||||
background: #333;
|
||||
font-family: sans-serif;
|
||||
text-shadow: 1px 1px 0px #000;
|
||||
touch-action: manipulation;
|
||||
}
|
||||
html, body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
body {
|
||||
padding-bottom: 5em;
|
||||
}
|
||||
#box {
|
||||
padding: .5em 1em;
|
||||
background: #2c2c2c;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user