Compare commits

..

1 Commits
v1.6.3 ... vcr

Author SHA1 Message Date
ed
ff8313d0fb add mistake 2021-07-01 21:49:44 +02:00
220 changed files with 8196 additions and 42315 deletions

View File

@@ -1,40 +0,0 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: bug
assignees: '9001'
---
NOTE:
all of the below are optional, consider them as inspiration, delete and rewrite at will, thx md
**Describe the bug**
a description of what the bug is
**To Reproduce**
List of steps to reproduce the issue, or, if it's hard to reproduce, then at least a detailed explanation of what you did to run into it
**Expected behavior**
a description of what you expected to happen
**Screenshots**
if applicable, add screenshots to help explain your problem, such as the kickass crashpage :^)
**Server details**
if the issue is possibly on the server-side, then mention some of the following:
* server OS / version:
* python version:
* copyparty arguments:
* filesystem (`lsblk -f` on linux):
**Client details**
if the issue is possibly on the client-side, then mention some of the following:
* the device type and model:
* OS version:
* browser version:
**Additional context**
any other context about the problem here

View File

@@ -1,22 +0,0 @@
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: enhancement
assignees: '9001'
---
all of the below are optional, consider them as inspiration, delete and rewrite at will
**is your feature request related to a problem? Please describe.**
a description of what the problem is, for example, `I'm always frustrated when [...]` or `Why is it not possible to [...]`
**Describe the idea / solution you'd like**
a description of what you want to happen
**Describe any alternatives you've considered**
a description of any alternative solutions or features you've considered
**Additional context**
add any other context or screenshots about the feature request here

View File

@@ -1,10 +0,0 @@
---
name: Something else
about: "┐(゚∀゚)┌"
title: ''
labels: ''
assignees: ''
---

View File

@@ -1,7 +0,0 @@
modernize your local checkout of the repo like so,
```sh
git branch -m master hovudstraum
git fetch origin
git branch -u origin/hovudstraum hovudstraum
git remote set-head origin -a
```

17
.gitignore vendored
View File

@@ -5,16 +5,12 @@ __pycache__/
MANIFEST.in
MANIFEST
copyparty.egg-info/
buildenv/
build/
dist/
sfx/
.venv/
/buildenv/
/build/
/dist/
/py2/
/sfx*
/unt/
/log/
# ide
*.sublime-workspace
@@ -22,10 +18,5 @@ copyparty.egg-info/
*.bak
# derived
copyparty/res/COPYING.txt
copyparty/web/deps/
srv/
# state/logs
up.*.txt
.hist/

3
.vscode/launch.json vendored
View File

@@ -8,7 +8,6 @@
"module": "copyparty",
"console": "integratedTerminal",
"cwd": "${workspaceFolder}",
"justMyCode": false,
"args": [
//"-nw",
"-ed",
@@ -18,7 +17,7 @@
"-mtp",
".bpm=f,bin/mtag/audio-bpm.py",
"-aed:wark",
"-vsrv::r:rw,ed:c,dupe",
"-vsrv::r:aed:cnodupe",
"-vdist:dist:r"
]
},

8
.vscode/launch.py vendored Executable file → Normal file
View File

@@ -1,5 +1,3 @@
#!/usr/bin/env python3
# takes arguments from launch.json
# is used by no_dbg in tasks.json
# launches 10x faster than mspython debugpy
@@ -11,15 +9,15 @@ import sys
print(sys.executable)
import json5
import shlex
import jstyleson
import subprocess as sp
with open(".vscode/launch.json", "r", encoding="utf-8") as f:
tj = f.read()
oj = json5.loads(tj)
oj = jstyleson.loads(tj)
argv = oj["configurations"][0]["args"]
try:
@@ -30,8 +28,6 @@ except:
argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
argv += sys.argv[1:]
if re.search(" -j ?[0-9]", " ".join(argv)):
argv = [sys.executable, "-m", "copyparty"] + argv
sp.check_call(argv)

29
.vscode/settings.json vendored
View File

@@ -23,6 +23,7 @@
"terminal.ansiBrightWhite": "#ffffff",
},
"python.testing.pytestEnabled": false,
"python.testing.nosetestsEnabled": false,
"python.testing.unittestEnabled": true,
"python.testing.unittestArgs": [
"-v",
@@ -34,42 +35,17 @@
"python.linting.pylintEnabled": true,
"python.linting.flake8Enabled": true,
"python.linting.banditEnabled": true,
"python.linting.mypyEnabled": true,
"python.linting.mypyArgs": [
"--ignore-missing-imports",
"--follow-imports=silent",
"--show-column-numbers",
"--strict"
],
"python.linting.flake8Args": [
"--max-line-length=120",
"--ignore=E722,F405,E203,W503,W293,E402,E501,E128",
"--ignore=E722,F405,E203,W503,W293,E402",
],
"python.linting.banditArgs": [
"--ignore=B104"
],
"python.linting.pylintArgs": [
"--disable=missing-module-docstring",
"--disable=missing-class-docstring",
"--disable=missing-function-docstring",
"--disable=import-outside-toplevel",
"--disable=wrong-import-position",
"--disable=raise-missing-from",
"--disable=bare-except",
"--disable=broad-except",
"--disable=invalid-name",
"--disable=line-too-long",
"--disable=consider-using-f-string"
],
// python3 -m isort --py=27 --profile=black copyparty/
"python.formatting.provider": "black",
"editor.formatOnSave": true,
"[html]": {
"editor.formatOnSave": false,
"editor.autoIndent": "keep",
},
"[css]": {
"editor.formatOnSave": false,
},
"files.associations": {
"*.makefile": "makefile"
@@ -79,5 +55,4 @@
"py27"
],
"python.linting.enabled": true,
"python.pythonPath": "/usr/bin/python3"
}

5
.vscode/tasks.json vendored
View File

@@ -9,10 +9,7 @@
{
"label": "no_dbg",
"type": "shell",
"command": "${config:python.pythonPath}",
"args": [
".vscode/launch.py"
]
"command": "${config:python.pythonPath} .vscode/launch.py"
}
]
}

View File

@@ -1,24 +0,0 @@
in the words of Abraham Lincoln:
> Be excellent to each other... and... PARTY ON, DUDES!
more specifically I'll paraphrase some examples from a german automotive corporation as they cover all the bases without being too wordy
## Examples of unacceptable behavior
* intimidation, harassment, trolling
* insulting, derogatory, harmful or prejudicial comments
* posting private information without permission
* political or personal attacks
## Examples of expected behavior
* being nice, friendly, welcoming, inclusive, mindful and empathetic
* acting considerate, modest, respectful
* using polite and inclusive language
* criticize constructively and accept constructive criticism
* respect different points of view
## finally and even more specifically,
* parse opinions and feedback objectively without prejudice
* it's the message that matters, not who said it
aaand that's how you say `be nice` in a way that fills half a floppy w

View File

@@ -1,3 +0,0 @@
* do something cool
really tho, send a PR or an issue or whatever, all appreciated, anything goes, just behave aight

1391
README.md

File diff suppressed because it is too large Load Diff

View File

@@ -1,9 +0,0 @@
# Security Policy
if you hit something extra juicy pls let me know on either of the following
* email -- `copyparty@ocv.ze` except `ze` should be `me`
* [mastodon dm](https://layer8.space/@tripflag) -- `@tripflag@layer8.space`
* [github private vulnerability report](https://github.com/9001/copyparty/security/advisories/new), wow that form is complicated
* [twitter dm](https://twitter.com/tripflag) (if im somehow not banned yet)
no bug bounties sorry! all i can offer is greetz in the release notes

View File

@@ -1,18 +1,4 @@
# [`up2k.py`](up2k.py)
* command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
* file uploads, file-search, autoresume of aborted/broken uploads
* sync local folder to server
* generally faster than browsers
* if something breaks just restart it
# [`partyjournal.py`](partyjournal.py)
produces a chronological list of all uploads by collecting info from up2k databases and the filesystem
* outputs a standalone html file
* optional mapping from IP-addresses to nicknames
# [`partyfuse.py`](partyfuse.py)
# [`copyparty-fuse.py`](copyparty-fuse.py)
* mount a copyparty server as a local filesystem (read-only)
* **supports Windows!** -- expect `194 MiB/s` sequential read
* **supports Linux** -- expect `117 MiB/s` sequential read
@@ -31,19 +17,19 @@ also consider using [../docs/rclone.md](../docs/rclone.md) instead for 5x perfor
* install [winfsp](https://github.com/billziss-gh/winfsp/releases/latest) and [python 3](https://www.python.org/downloads/)
* [x] add python 3.x to PATH (it asks during install)
* `python -m pip install --user fusepy`
* `python ./partyfuse.py n: http://192.168.1.69:3923/`
* `python ./copyparty-fuse.py n: http://192.168.1.69:3923/`
10% faster in [msys2](https://www.msys2.org/), 700% faster if debug prints are enabled:
* `pacman -S mingw64/mingw-w64-x86_64-python{,-pip}`
* `/mingw64/bin/python3 -m pip install --user fusepy`
* `/mingw64/bin/python3 ./partyfuse.py [...]`
* `/mingw64/bin/python3 ./copyparty-fuse.py [...]`
you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releases/latest), let me know if you [figure out how](https://github.com/dokan-dev/dokany/wiki/FUSE)
(winfsp's sshfs leaks, doesn't look like winfsp itself does, should be fine)
# [`partyfuse2.py`](partyfuse2.py)
# [`copyparty-fuse🅱️.py`](copyparty-fuseb.py)
* mount a copyparty server as a local filesystem (read-only)
* does the same thing except more correct, `samba` approves
* **supports Linux** -- expect `18 MiB/s` (wait what)
@@ -51,7 +37,7 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
# [`partyfuse-streaming.py`](partyfuse-streaming.py)
# [`copyparty-fuse-streaming.py`](copyparty-fuse-streaming.py)
* pretend this doesn't exist
@@ -61,7 +47,6 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
* copyparty can Popen programs like these during file indexing to collect additional metadata
# [`dbtool.py`](dbtool.py)
upgrade utility which can show db info and help transfer data between databases, for example when a new version of copyparty is incompatible with the old DB and automatically rebuilds the DB from scratch, but you have some really expensive `-mtp` parsers and want to copy over the tags from the old db
@@ -76,9 +61,3 @@ cd /mnt/nas/music/.hist
~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -rm-mtp-flag -copy key
~/src/copyparty/bin/dbtool.py -src up2k.*.v3 up2k.db -rm-mtp-flag -copy .bpm -vac
```
# [`prisonparty.sh`](prisonparty.sh)
* run copyparty in a chroot, preventing any accidental file access
* creates bindmounts for /bin, /lib, and so on, see `sysdirs=`

View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python3
from __future__ import print_function, unicode_literals
"""partyfuse-streaming: remote copyparty as a local filesystem"""
"""copyparty-fuse-streaming: remote copyparty as a local filesystem"""
__author__ = "ed <copyparty@ocv.me>"
__copyright__ = 2020
__license__ = "MIT"
@@ -12,7 +12,7 @@ __url__ = "https://github.com/9001/copyparty/"
mount a copyparty server (local or remote) as a filesystem
usage:
python partyfuse-streaming.py http://192.168.1.69:3923/ ./music
python copyparty-fuse-streaming.py http://192.168.1.69:3923/ ./music
dependencies:
python3 -m pip install --user fusepy
@@ -21,7 +21,7 @@ dependencies:
+ on Windows: https://github.com/billziss-gh/winfsp/releases/latest
this was a mistake:
fork of partyfuse.py with a streaming cache rather than readahead,
fork of copyparty-fuse.py with a streaming cache rather than readahead,
thought this was gonna be way faster (and it kind of is)
except the overhead of reopening connections on trunc totally kills it
"""
@@ -42,7 +42,6 @@ import threading
import traceback
import http.client # py2: httplib
import urllib.parse
import calendar
from datetime import datetime
from urllib.parse import quote_from_bytes as quote
from urllib.parse import unquote_to_bytes as unquote
@@ -62,12 +61,12 @@ except:
else:
libfuse = "apt install libfuse\n modprobe fuse"
m = """\033[33m
could not import fuse; these may help:
{} -m pip install --user fusepy
{}
\033[0m"""
print(m.format(sys.executable, libfuse))
print(
"\n could not import fuse; these may help:"
+ "\n python3 -m pip install --user fusepy\n "
+ libfuse
+ "\n"
)
raise
@@ -154,7 +153,7 @@ def dewin(txt):
class RecentLog(object):
def __init__(self):
self.mtx = threading.Lock()
self.f = None # open("partyfuse.log", "wb")
self.f = None # open("copyparty-fuse.log", "wb")
self.q = []
thr = threading.Thread(target=self.printer)
@@ -185,9 +184,9 @@ class RecentLog(object):
print("".join(q), end="")
# [windows/cmd/cpy3] python dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/partyfuse.py q: http://192.168.1.159:1234/
# [windows/cmd/cpy3] python dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/copyparty-fuse.py q: http://192.168.1.159:1234/
#
# [windows] find /q/music/albums/Phant*24bit -printf '%s %p\n' | sort -n | tail -n 8 | sed -r 's/^[0-9]+ //' | while IFS= read -r x; do dd if="$x" of=/dev/null bs=4k count=8192 & done
# [alpine] ll t; for x in t/2020_0724_16{2,3}*; do dd if="$x" of=/dev/null bs=4k count=10240 & done
@@ -346,7 +345,7 @@ class Gateway(object):
except:
pass
def sendreq(self, meth, path, headers, **kwargs):
def sendreq(self, *args, headers={}, **kwargs):
if self.password:
headers["Cookie"] = "=".join(["cppwd", self.password])
@@ -355,21 +354,21 @@ class Gateway(object):
if c.rx_path:
raise Exception()
c.request(meth, path, headers=headers, **kwargs)
c.request(*list(args), headers=headers, **kwargs)
c.rx = c.getresponse()
return c
except:
tid = threading.current_thread().ident
dbg(
"\033[1;37;44mbad conn {:x}\n {} {}\n {}\033[0m".format(
tid, meth, path, c.rx_path if c else "(null)"
"\033[1;37;44mbad conn {:x}\n {}\n {}\033[0m".format(
tid, " ".join(str(x) for x in args), c.rx_path if c else "(null)"
)
)
self.closeconn(c)
c = self.getconn()
try:
c.request(meth, path, headers=headers, **kwargs)
c.request(*list(args), headers=headers, **kwargs)
c.rx = c.getresponse()
return c
except:
@@ -387,7 +386,7 @@ class Gateway(object):
path = dewin(path)
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
c = self.sendreq("GET", web_path, {})
c = self.sendreq("GET", web_path)
if c.rx.status != 200:
self.closeconn(c)
log(
@@ -441,7 +440,7 @@ class Gateway(object):
)
)
c = self.sendreq("GET", web_path, {"Range": hdr_range})
c = self.sendreq("GET", web_path, headers={"Range": hdr_range})
if c.rx.status != http.client.PARTIAL_CONTENT:
self.closeconn(c)
raise Exception(
@@ -496,7 +495,7 @@ class Gateway(object):
ts = 60 * 60 * 24 * 2
try:
sz = int(fsize)
ts = calendar.timegm(time.strptime(fdate, "%Y-%m-%d %H:%M:%S"))
ts = datetime.strptime(fdate, "%Y-%m-%d %H:%M:%S").timestamp()
except:
info("bad HTML or OS [{}] [{}]".format(fdate, fsize))
# python cannot strptime(1959-01-01) on windows

View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python3
from __future__ import print_function, unicode_literals
"""partyfuse: remote copyparty as a local filesystem"""
"""copyparty-fuse: remote copyparty as a local filesystem"""
__author__ = "ed <copyparty@ocv.me>"
__copyright__ = 2019
__license__ = "MIT"
@@ -12,7 +12,7 @@ __url__ = "https://github.com/9001/copyparty/"
mount a copyparty server (local or remote) as a filesystem
usage:
python partyfuse.py http://192.168.1.69:3923/ ./music
python copyparty-fuse.py http://192.168.1.69:3923/ ./music
dependencies:
python3 -m pip install --user fusepy
@@ -22,7 +22,7 @@ dependencies:
note:
you probably want to run this on windows clients:
https://github.com/9001/copyparty/blob/hovudstraum/contrib/explorer-nothumbs-nofoldertypes.reg
https://github.com/9001/copyparty/blob/master/contrib/explorer-nothumbs-nofoldertypes.reg
get server cert:
awk '/-BEGIN CERTIFICATE-/ {a=1} a; /-END CERTIFICATE-/{exit}' <(openssl s_client -connect 127.0.0.1:3923 </dev/null 2>/dev/null) >cert.pem
@@ -45,7 +45,6 @@ import threading
import traceback
import http.client # py2: httplib
import urllib.parse
import calendar
from datetime import datetime
from urllib.parse import quote_from_bytes as quote
from urllib.parse import unquote_to_bytes as unquote
@@ -55,13 +54,10 @@ MACOS = platform.system() == "Darwin"
info = log = dbg = None
print(
"{} v{} @ {}".format(
platform.python_implementation(),
".".join([str(x) for x in sys.version_info]),
sys.executable,
)
)
print("{} v{} @ {}".format(
platform.python_implementation(),
".".join([str(x) for x in sys.version_info]),
sys.executable))
try:
@@ -72,14 +68,14 @@ except:
elif MACOS:
libfuse = "install https://osxfuse.github.io/"
else:
libfuse = "apt install libfuse3-3\n modprobe fuse"
libfuse = "apt install libfuse\n modprobe fuse"
m = """\033[33m
could not import fuse; these may help:
{} -m pip install --user fusepy
{}
\033[0m"""
print(m.format(sys.executable, libfuse))
print(
"\n could not import fuse; these may help:"
+ "\n python3 -m pip install --user fusepy\n "
+ libfuse
+ "\n"
)
raise
@@ -166,7 +162,7 @@ def dewin(txt):
class RecentLog(object):
def __init__(self):
self.mtx = threading.Lock()
self.f = None # open("partyfuse.log", "wb")
self.f = None # open("copyparty-fuse.log", "wb")
self.q = []
thr = threading.Thread(target=self.printer)
@@ -197,9 +193,9 @@ class RecentLog(object):
print("".join(q), end="")
# [windows/cmd/cpy3] python dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/partyfuse.py q: http://192.168.1.159:1234/
# [windows/cmd/cpy3] python dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/copyparty-fuse.py q: http://192.168.1.159:1234/
#
# [windows] find /q/music/albums/Phant*24bit -printf '%s %p\n' | sort -n | tail -n 8 | sed -r 's/^[0-9]+ //' | while IFS= read -r x; do dd if="$x" of=/dev/null bs=4k count=8192 & done
# [alpine] ll t; for x in t/2020_0724_16{2,3}*; do dd if="$x" of=/dev/null bs=4k count=10240 & done
@@ -303,14 +299,14 @@ class Gateway(object):
except:
pass
def sendreq(self, meth, path, headers, **kwargs):
def sendreq(self, *args, headers={}, **kwargs):
tid = get_tid()
if self.password:
headers["Cookie"] = "=".join(["cppwd", self.password])
try:
c = self.getconn(tid)
c.request(meth, path, headers=headers, **kwargs)
c.request(*list(args), headers=headers, **kwargs)
return c.getresponse()
except:
dbg("bad conn")
@@ -318,7 +314,7 @@ class Gateway(object):
self.closeconn(tid)
try:
c = self.getconn(tid)
c.request(meth, path, headers=headers, **kwargs)
c.request(*list(args), headers=headers, **kwargs)
return c.getresponse()
except:
info("http connection failed:\n" + traceback.format_exc())
@@ -335,7 +331,7 @@ class Gateway(object):
path = dewin(path)
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
r = self.sendreq("GET", web_path, {})
r = self.sendreq("GET", web_path)
if r.status != 200:
self.closeconn()
log(
@@ -372,7 +368,7 @@ class Gateway(object):
)
)
r = self.sendreq("GET", web_path, {"Range": hdr_range})
r = self.sendreq("GET", web_path, headers={"Range": hdr_range})
if r.status != http.client.PARTIAL_CONTENT:
self.closeconn()
raise Exception(
@@ -394,16 +390,15 @@ class Gateway(object):
rsp = json.loads(rsp.decode("utf-8"))
ret = []
for statfun, nodes in [
[self.stat_dir, rsp["dirs"]],
[self.stat_file, rsp["files"]],
]:
for is_dir, nodes in [[True, rsp["dirs"]], [False, rsp["files"]]]:
for n in nodes:
fname = unquote(n["href"].split("?")[0]).rstrip(b"/").decode("wtf-8")
fname = unquote(n["href"]).rstrip(b"/")
fname = fname.decode("wtf-8")
if bad_good:
fname = enwin(fname)
ret.append([fname, statfun(n["ts"], n["sz"]), 0])
fun = self.stat_dir if is_dir else self.stat_file
ret.append([fname, fun(n["ts"], n["sz"]), 0])
return ret
@@ -444,7 +439,7 @@ class Gateway(object):
ts = 60 * 60 * 24 * 2
try:
sz = int(fsize)
ts = calendar.timegm(time.strptime(fdate, "%Y-%m-%d %H:%M:%S"))
ts = datetime.strptime(fdate, "%Y-%m-%d %H:%M:%S").timestamp()
except:
info("bad HTML or OS [{}] [{}]".format(fdate, fsize))
# python cannot strptime(1959-01-01) on windows
@@ -997,7 +992,7 @@ def main():
ap.add_argument(
"-cf", metavar="NUM_BLOCKS", type=int, default=nf, help="file cache"
)
ap.add_argument("-a", metavar="PASSWORD", help="password or $filepath")
ap.add_argument("-a", metavar="PASSWORD", help="password")
ap.add_argument("-d", action="store_true", help="enable debug")
ap.add_argument("-te", metavar="PEM_FILE", help="certificate to expect/verify")
ap.add_argument("-td", action="store_true", help="disable certificate check")

View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python3
from __future__ import print_function, unicode_literals
"""partyfuse2: remote copyparty as a local filesystem"""
"""copyparty-fuseb: remote copyparty as a local filesystem"""
__author__ = "ed <copyparty@ocv.me>"
__copyright__ = 2020
__license__ = "MIT"
@@ -11,18 +11,14 @@ import re
import os
import sys
import time
import json
import stat
import errno
import struct
import codecs
import platform
import threading
import http.client # py2: httplib
import urllib.parse
from datetime import datetime
from urllib.parse import quote_from_bytes as quote
from urllib.parse import unquote_to_bytes as unquote
try:
import fuse
@@ -32,19 +28,9 @@ try:
if not hasattr(fuse, "__version__"):
raise Exception("your fuse-python is way old")
except:
if WINDOWS:
libfuse = "install https://github.com/billziss-gh/winfsp/releases/latest"
elif MACOS:
libfuse = "install https://osxfuse.github.io/"
else:
libfuse = "apt install libfuse\n modprobe fuse"
m = """\033[33m
could not import fuse; these may help:
{} -m pip install --user fuse-python
{}
\033[0m"""
print(m.format(sys.executable, libfuse))
print(
"\n could not import fuse; these may help:\n python3 -m pip install --user fuse-python\n apt install libfuse\n modprobe fuse\n"
)
raise
@@ -52,22 +38,18 @@ except:
mount a copyparty server (local or remote) as a filesystem
usage:
python ./partyfuse2.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas
python ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,url=http://192.168.1.69:3923 /mnt/nas
dependencies:
sudo apk add fuse-dev python3-dev
python3 -m pip install --user fuse-python
fork of partyfuse.py based on fuse-python which
fork of copyparty-fuse.py based on fuse-python which
appears to be more compliant than fusepy? since this works with samba
(probably just my garbage code tbh)
"""
WINDOWS = sys.platform == "win32"
MACOS = platform.system() == "Darwin"
def threadless_log(msg):
print(msg + "\n", end="")
@@ -111,41 +93,6 @@ def html_dec(txt):
)
def register_wtf8():
def wtf8_enc(text):
return str(text).encode("utf-8", "surrogateescape"), len(text)
def wtf8_dec(binary):
return bytes(binary).decode("utf-8", "surrogateescape"), len(binary)
def wtf8_search(encoding_name):
return codecs.CodecInfo(wtf8_enc, wtf8_dec, name="wtf-8")
codecs.register(wtf8_search)
bad_good = {}
good_bad = {}
def enwin(txt):
return "".join([bad_good.get(x, x) for x in txt])
for bad, good in bad_good.items():
txt = txt.replace(bad, good)
return txt
def dewin(txt):
return "".join([good_bad.get(x, x) for x in txt])
for bad, good in bad_good.items():
txt = txt.replace(good, bad)
return txt
class CacheNode(object):
def __init__(self, tag, data):
self.tag = tag
@@ -168,9 +115,8 @@ class Stat(fuse.Stat):
class Gateway(object):
def __init__(self, base_url, pw):
def __init__(self, base_url):
self.base_url = base_url
self.pw = pw
ui = urllib.parse.urlparse(base_url)
self.web_root = ui.path.strip("/")
@@ -189,7 +135,8 @@ class Gateway(object):
self.conns = {}
def quotep(self, path):
path = path.encode("wtf-8")
# TODO: mojibake support
path = path.encode("utf-8", "ignore")
return quote(path, safe="/")
def getconn(self, tid=None):
@@ -212,29 +159,20 @@ class Gateway(object):
except:
pass
def sendreq(self, *args, **ka):
def sendreq(self, *args, **kwargs):
tid = get_tid()
if self.pw:
ck = "cppwd=" + self.pw
try:
ka["headers"]["Cookie"] = ck
except:
ka["headers"] = {"Cookie": ck}
try:
c = self.getconn(tid)
c.request(*list(args), **ka)
c.request(*list(args), **kwargs)
return c.getresponse()
except:
self.closeconn(tid)
c = self.getconn(tid)
c.request(*list(args), **ka)
c.request(*list(args), **kwargs)
return c.getresponse()
def listdir(self, path):
if bad_good:
path = dewin(path)
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots&ls"
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?dots"
r = self.sendreq("GET", web_path)
if r.status != 200:
self.closeconn()
@@ -244,12 +182,9 @@ class Gateway(object):
)
)
return self.parse_jls(r)
return self.parse_html(r)
def download_file_range(self, path, ofs1, ofs2):
if bad_good:
path = dewin(path)
web_path = self.quotep("/" + "/".join([self.web_root, path])) + "?raw"
hdr_range = "bytes={}-{}".format(ofs1, ofs2 - 1)
log("downloading {}".format(hdr_range))
@@ -265,27 +200,40 @@ class Gateway(object):
return r.read()
def parse_jls(self, datasrc):
rsp = b""
def parse_html(self, datasrc):
ret = []
remainder = b""
ptn = re.compile(
r"^<tr><td>(-|DIR)</td><td><a [^>]+>([^<]+)</a></td><td>([^<]+)</td><td>([^<]+)</td></tr>$"
)
while True:
buf = datasrc.read(1024 * 32)
buf = remainder + datasrc.read(4096)
# print('[{}]'.format(buf.decode('utf-8')))
if not buf:
break
rsp += buf
remainder = b""
endpos = buf.rfind(b"\n")
if endpos >= 0:
remainder = buf[endpos + 1 :]
buf = buf[:endpos]
rsp = json.loads(rsp.decode("utf-8"))
ret = []
for statfun, nodes in [
[self.stat_dir, rsp["dirs"]],
[self.stat_file, rsp["files"]],
]:
for n in nodes:
fname = unquote(n["href"].split("?")[0]).rstrip(b"/").decode("wtf-8")
if bad_good:
fname = enwin(fname)
lines = buf.decode("utf-8").split("\n")
for line in lines:
m = ptn.match(line)
if not m:
# print(line)
continue
ret.append([fname, statfun(n["ts"], n["sz"]), 0])
ftype, fname, fsize, fdate = m.groups()
fname = html_dec(fname)
ts = datetime.strptime(fdate, "%Y-%m-%d %H:%M:%S").timestamp()
sz = int(fsize)
if ftype == "-":
ret.append([fname, self.stat_file(ts, sz), 0])
else:
ret.append([fname, self.stat_dir(ts, sz), 0])
return ret
@@ -314,7 +262,6 @@ class CPPF(Fuse):
Fuse.__init__(self, *args, **kwargs)
self.url = None
self.pw = None
self.dircache = []
self.dircache_mtx = threading.Lock()
@@ -324,7 +271,7 @@ class CPPF(Fuse):
def init2(self):
# TODO figure out how python-fuse wanted this to go
self.gw = Gateway(self.url, self.pw) # .decode('utf-8'))
self.gw = Gateway(self.url) # .decode('utf-8'))
info("up")
def clean_dircache(self):
@@ -589,8 +536,6 @@ class CPPF(Fuse):
def getattr(self, path):
log("getattr [{}]".format(path))
if WINDOWS:
path = enwin(path) # windows occasionally decodes f0xx to xx
path = path.strip("/")
try:
@@ -623,25 +568,9 @@ class CPPF(Fuse):
def main():
time.strptime("19970815", "%Y%m%d") # python#7980
register_wtf8()
if WINDOWS:
os.system("rem")
for ch in '<>:"\\|?*':
# microsoft maps illegal characters to f0xx
# (e000 to f8ff is basic-plane private-use)
bad_good[ch] = chr(ord(ch) + 0xF000)
for n in range(0, 0x100):
# map surrogateescape to another private-use area
bad_good[chr(n + 0xDC00)] = chr(n + 0xF100)
for k, v in bad_good.items():
good_bad[v] = k
server = CPPF()
server.parser.add_option(mountopt="url", metavar="BASE_URL", default=None)
server.parser.add_option(mountopt="pw", metavar="PASSWORD", default=None)
server.parse(values=server, errex=1)
if not server.url or not str(server.url).startswith("http"):
print("\nerror:")
@@ -649,7 +578,7 @@ def main():
print(" need argument: mount-path")
print("example:")
print(
" ./partyfuse2.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas"
" ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,url=http://192.168.1.69:3923 /mnt/nas"
)
sys.exit(1)

View File

@@ -8,10 +8,7 @@ import sqlite3
import argparse
DB_VER1 = 3
DB_VER2 = 5
BY_PATH = None
NC = None
DB_VER2 = 4
def die(msg):
@@ -60,13 +57,8 @@ def compare(n1, d1, n2, d2, verbose):
if rd.split("/", 1)[0] == ".hist":
continue
if BY_PATH:
q = "select w from up where rd = ? and fn = ?"
hit = d2.execute(q, (rd, fn)).fetchone()
else:
q = "select w from up where substr(w,1,16) = ? and +w = ?"
hit = d2.execute(q, (w1[:16], w1)).fetchone()
q = "select w from up where rd = ? and fn = ?"
hit = d2.execute(q, (rd, fn)).fetchone()
if not hit:
miss += 1
if verbose:
@@ -78,32 +70,27 @@ def compare(n1, d1, n2, d2, verbose):
n = 0
miss = {}
nmiss = 0
for w1s, k, v in d1.execute("select * from mt"):
for w1, k, v in d1.execute("select * from mt"):
n += 1
if n % 100_000 == 0:
m = f"\033[36mchecked {n:,} of {nt:,} tags in {n1} against {n2}, so far {nmiss} missing tags\033[0m"
print(m)
q = "select w, rd, fn from up where substr(w,1,16) = ?"
w1, rd, fn = d1.execute(q, (w1s,)).fetchone()
q = "select rd, fn from up where substr(w,1,16) = ?"
rd, fn = d1.execute(q, (w1,)).fetchone()
if rd.split("/", 1)[0] == ".hist":
continue
if BY_PATH:
q = "select w from up where rd = ? and fn = ?"
w2 = d2.execute(q, (rd, fn)).fetchone()
else:
q = "select w from up where substr(w,1,16) = ? and +w = ?"
w2 = d2.execute(q, (w1s, w1)).fetchone()
q = "select substr(w,1,16) from up where rd = ? and fn = ?"
w2 = d2.execute(q, (rd, fn)).fetchone()
if w2:
w2 = w2[0]
v2 = None
if w2:
v2 = d2.execute(
"select v from mt where w = ? and +k = ?", (w2[:16], k)
"select v from mt where w = ? and +k = ?", (w2, k)
).fetchone()
if v2:
v2 = v2[0]
@@ -137,7 +124,7 @@ def compare(n1, d1, n2, d2, verbose):
for k, v in sorted(miss.items()):
if v:
print(f"{n1} has {v:7} more {k:<7} tags than {n2}")
print(f"{n1} has {v:6} more {k:<6} tags than {n2}")
print(f"in total, {nmiss} missing tags in {n2}\n")
@@ -145,75 +132,47 @@ def compare(n1, d1, n2, d2, verbose):
def copy_mtp(d1, d2, tag, rm):
nt = next(d1.execute("select count(w) from mt where k = ?", (tag,)))[0]
n = 0
ncopy = 0
nskip = 0
for w1s, k, v in d1.execute("select * from mt where k = ?", (tag,)):
ndone = 0
for w1, k, v in d1.execute("select * from mt where k = ?", (tag,)):
n += 1
if n % 25_000 == 0:
m = f"\033[36m{n:,} of {nt:,} tags checked, so far {ncopy} copied, {nskip} skipped\033[0m"
m = f"\033[36m{n:,} of {nt:,} tags checked, so far {ndone} copied\033[0m"
print(m)
q = "select w, rd, fn from up where substr(w,1,16) = ?"
w1, rd, fn = d1.execute(q, (w1s,)).fetchone()
q = "select rd, fn from up where substr(w,1,16) = ?"
rd, fn = d1.execute(q, (w1,)).fetchone()
if rd.split("/", 1)[0] == ".hist":
continue
if BY_PATH:
q = "select w from up where rd = ? and fn = ?"
w2 = d2.execute(q, (rd, fn)).fetchone()
else:
q = "select w from up where substr(w,1,16) = ? and +w = ?"
w2 = d2.execute(q, (w1s, w1)).fetchone()
q = "select substr(w,1,16) from up where rd = ? and fn = ?"
w2 = d2.execute(q, (rd, fn)).fetchone()
if not w2:
continue
w2s = w2[0][:16]
hit = d2.execute("select v from mt where w = ? and +k = ?", (w2s, k)).fetchone()
w2 = w2[0]
hit = d2.execute("select v from mt where w = ? and +k = ?", (w2, k)).fetchone()
if hit:
hit = hit[0]
if hit != v:
if NC and hit is not None:
nskip += 1
continue
ncopy += 1
ndone += 1
if hit is not None:
d2.execute("delete from mt where w = ? and +k = ?", (w2s, k))
d2.execute("delete from mt where w = ? and +k = ?", (w2, k))
d2.execute("insert into mt values (?,?,?)", (w2s, k, v))
d2.execute("insert into mt values (?,?,?)", (w2, k, v))
if rm:
d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w2s,))
d2.execute("delete from mt where w = ? and +k = 't:mtp'", (w2,))
d2.commit()
print(f"copied {ncopy} {tag} tags over, skipped {nskip}")
def examples():
print(
"""
# clearing the journal
./dbtool.py up2k.db
# copy tags ".bpm" and "key" from old.db to up2k.db, and remove the mtp flag from matching files (so copyparty won't run any mtps on it)
./dbtool.py -ls up2k.db
./dbtool.py -src old.db up2k.db -cmp
./dbtool.py -src old.v3 up2k.db -rm-mtp-flag -copy key
./dbtool.py -src old.v3 up2k.db -rm-mtp-flag -copy .bpm -vac
"""
)
print(f"copied {ndone} {tag} tags over")
def main():
global NC, BY_PATH
os.system("")
print()
ap = argparse.ArgumentParser()
ap.add_argument("db", help="database to work on")
ap.add_argument("-h2", action="store_true", help="show examples")
ap.add_argument("-src", metavar="DB", type=str, help="database to copy from")
ap2 = ap.add_argument_group("informational / read-only stuff")
@@ -226,29 +185,11 @@ def main():
ap2.add_argument(
"-rm-mtp-flag",
action="store_true",
help="when an mtp tag is copied over, also mark that file as done, so copyparty won't run any mtps on those files",
help="when an mtp tag is copied over, also mark that as done, so copyparty won't run mtp on it",
)
ap2.add_argument("-vac", action="store_true", help="optimize DB")
ap2 = ap.add_argument_group("behavior modifiers")
ap2.add_argument(
"-nc",
action="store_true",
help="no-clobber; don't replace/overwrite existing tags",
)
ap2.add_argument(
"-by-path",
action="store_true",
help="match files based on location rather than warks (content-hash), use this if the databases have different wark salts",
)
ar = ap.parse_args()
if ar.h2:
examples()
return
NC = ar.nc
BY_PATH = ar.by_path
for v in [ar.db, ar.src]:
if v and not os.path.exists(v):

View File

@@ -1,19 +0,0 @@
standalone programs which are executed by copyparty when an event happens (upload, file rename, delete, ...)
these programs either take zero arguments, or a filepath (the affected file), or a json message with filepath + additional info
> **note:** in addition to event hooks (the stuff described here), copyparty has another api to run your programs/scripts while providing way more information such as audio tags / video codecs / etc and optionally daisychaining data between scripts in a processing pipeline; if that's what you want then see [mtp plugins](../mtag/) instead
# after upload
* [notify.py](notify.py) shows a desktop notification ([example](https://user-images.githubusercontent.com/241032/215335767-9c91ed24-d36e-4b6b-9766-fb95d12d163f.png))
* [discord-announce.py](discord-announce.py) announces new uploads on discord using webhooks ([example](https://user-images.githubusercontent.com/241032/215304439-1c1cb3c8-ec6f-4c17-9f27-81f969b1811a.png))
* [reject-mimetype.py](reject-mimetype.py) rejects uploads unless the mimetype is acceptable
# before upload
* [reject-extension.py](reject-extension.py) rejects uploads if they match a list of file extensions
# on message
* [wget.py](wget.py) lets you download files by POSTing URLs to copyparty

View File

@@ -1,61 +0,0 @@
#!/usr/bin/env python3
import sys
import json
import requests
from copyparty.util import humansize, quotep
_ = r"""
announces a new upload on discord
example usage as global config:
--xau f,t5,j,bin/hooks/discord-announce.py
example usage as a volflag (per-volume config):
-v srv/inc:inc:c,xau=f,t5,j,bin/hooks/discord-announce.py
parameters explained,
f = fork; don't wait for it to finish
t5 = timeout if it's still running after 5 sec
j = provide upload information as json; not just the filename
replace "xau" with "xbu" to announce Before upload starts instead of After completion
# how to discord:
first create the webhook url; https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks
then use this to design your message: https://discohook.org/
"""
def main():
WEBHOOK = "https://discord.com/api/webhooks/1234/base64"
# read info from copyparty
inf = json.loads(sys.argv[1])
vpath = inf["vp"]
filename = vpath.split("/")[-1]
url = f"https://{inf['host']}/{quotep(vpath)}"
# compose the message to discord
j = {
"title": filename,
"url": url,
"description": url.rsplit("/", 1)[0],
"color": 0x449900,
"fields": [
{"name": "Size", "value": humansize(inf["sz"])},
{"name": "User", "value": inf["user"]},
{"name": "IP", "value": inf["ip"]},
],
}
for v in j["fields"]:
v["inline"] = True
r = requests.post(WEBHOOK, json={"embeds": [j]})
print(f"discord: {r}\n", end="")
if __name__ == "__main__":
main()

View File

@@ -1,62 +0,0 @@
#!/usr/bin/env python3
import os
import sys
import subprocess as sp
from plyer import notification
_ = r"""
show os notification on upload; works on windows, linux, macos, android
depdencies:
windows: python3 -m pip install --user -U plyer
linux: python3 -m pip install --user -U plyer
macos: python3 -m pip install --user -U plyer pyobjus
android: just termux and termux-api
example usages; either as global config (all volumes) or as volflag:
--xau f,bin/hooks/notify.py
-v srv/inc:inc:c,xau=f,bin/hooks/notify.py
^^^^^^^^^^^^^^^^^^^^^^^^^^^
parameters explained,
xau = execute after upload
f = fork so it doesn't block uploads
"""
try:
from copyparty.util import humansize
except:
def humansize(n):
return n
def main():
fp = sys.argv[1]
dp, fn = os.path.split(fp)
try:
sz = humansize(os.path.getsize(fp))
except:
sz = "?"
msg = "{} ({})\n📁 {}".format(fn, sz, dp)
title = "File received"
if "com.termux" in sys.executable:
sp.run(["termux-notification", "-t", title, "-c", msg])
return
icon = "emblem-documents-symbolic" if sys.platform == "linux" else ""
notification.notify(
title=title,
message=msg,
app_icon=icon,
timeout=10,
)
if __name__ == "__main__":
main()

View File

@@ -1,30 +0,0 @@
#!/usr/bin/env python3
import sys
_ = r"""
reject file uploads by file extension
example usage as global config:
--xbu c,bin/hooks/reject-extension.py
example usage as a volflag (per-volume config):
-v srv/inc:inc:c,xbu=c,bin/hooks/reject-extension.py
parameters explained,
xbu = execute before upload
c = check result, reject upload if error
"""
def main():
bad = "exe scr com pif bat ps1 jar msi"
ext = sys.argv[1].split(".")[-1]
sys.exit(1 if ext in bad.split() else 0)
if __name__ == "__main__":
main()

View File

@@ -1,39 +0,0 @@
#!/usr/bin/env python3
import sys
import magic
_ = r"""
reject file uploads by mimetype
dependencies (linux, macos):
python3 -m pip install --user -U python-magic
dependencies (windows):
python3 -m pip install --user -U python-magic-bin
example usage as global config:
--xau c,bin/hooks/reject-mimetype.py
example usage as a volflag (per-volume config):
-v srv/inc:inc:c,xau=c,bin/hooks/reject-mimetype.py
parameters explained,
xau = execute after upload
c = check result, reject upload if error
"""
def main():
ok = ["image/jpeg", "image/png"]
mt = magic.from_file(sys.argv[1], mime=True)
print(mt)
sys.exit(1 if mt not in ok else 0)
if __name__ == "__main__":
main()

View File

@@ -1,54 +0,0 @@
#!/usr/bin/env python3
import os
import sys
import json
import subprocess as sp
_ = r"""
use copyparty as a file downloader by POSTing URLs as
application/x-www-form-urlencoded (for example using the
message/pager function on the website)
example usage as global config:
--xm f,j,t3600,bin/hooks/wget.py
example usage as a volflag (per-volume config):
-v srv/inc:inc:c,xm=f,j,t3600,bin/hooks/wget.py
parameters explained,
f = fork so it doesn't block uploads
j = provide message information as json; not just the text
c3 = mute all output
t3600 = timeout and kill download after 1 hour
"""
def main():
inf = json.loads(sys.argv[1])
url = inf["txt"]
if "://" not in url:
url = "https://" + url
os.chdir(inf["ap"])
name = url.split("?")[0].split("/")[-1]
tfn = "-- DOWNLOADING " + name
print(f"{tfn}\n", end="")
open(tfn, "wb").close()
cmd = ["wget", "--trust-server-names", "-nv", "--", url]
try:
sp.check_call(cmd)
except:
t = "-- FAILED TO DONWLOAD " + name
print(f"{t}\n", end="")
open(t, "wb").close()
os.unlink(tfn)
if __name__ == "__main__":
main()

View File

@@ -1,28 +1,10 @@
standalone programs which take an audio file as argument
you may want to forget about all this fancy complicated stuff and just use [event hooks](../hooks/) instead (which doesn't need `-e2ts` or ffmpeg)
----
**NOTE:** these all require `-e2ts` to be functional, meaning you need to do at least one of these: `apt install ffmpeg` or `pip3 install mutagen`
some of these rely on libraries which are not MIT-compatible
* [audio-bpm.py](./audio-bpm.py) detects the BPM of music using the BeatRoot Vamp Plugin; imports GPL2
* [audio-key.py](./audio-key.py) detects the melodic key of music using the Mixxx fork of keyfinder; imports GPL3
these invoke standalone programs which are GPL or similar, so is legally fine for most purposes:
* [media-hash.py](./media-hash.py) generates checksums for audio and video streams; uses FFmpeg (LGPL or GPL)
* [image-noexif.py](./image-noexif.py) removes exif tags from images; uses exiftool (GPLv1 or artistic-license)
these do not have any problematic dependencies at all:
* [cksum.py](./cksum.py) computes various checksums
* [exe.py](./exe.py) grabs metadata from .exe and .dll files (example for retrieving multiple tags with one parser)
* [wget.py](./wget.py) lets you download files by POSTing URLs to copyparty
* also available as an [event hook](../hooks/wget.py)
# dependencies
@@ -36,10 +18,7 @@ run [`install-deps.sh`](install-deps.sh) to build/install most dependencies requ
# usage from copyparty
`copyparty -e2dsa -e2ts` followed by any combination of these:
* `-mtp key=f,audio-key.py`
* `-mtp .bpm=f,audio-bpm.py`
* `-mtp ahash,vhash=f,media-hash.py`
`copyparty -e2dsa -e2ts -mtp key=f,audio-key.py -mtp .bpm=f,audio-bpm.py`
* `f,` makes the detected value replace any existing values
* the `.` in `.bpm` indicates numeric value
@@ -47,12 +26,9 @@ run [`install-deps.sh`](install-deps.sh) to build/install most dependencies requ
* `mtp` modules will not run if a file has existing tags in the db, so clear out the tags with `-e2tsr` the first time you launch with new `mtp` options
## usage with volflags
## usage with volume-flags
instead of affecting all volumes, you can set the options for just one volume like so:
`copyparty -v /mnt/nas/music:/music:r:c,e2dsa:c,e2ts` immediately followed by any combination of these:
* `:c,mtp=key=f,audio-key.py`
* `:c,mtp=.bpm=f,audio-bpm.py`
* `:c,mtp=ahash,vhash=f,media-hash.py`
```
copyparty -v /mnt/nas/music:/music:r:cmtp=key=f,audio-key.py:cmtp=.bpm=f,audio-bpm.py:ce2dsa:ce2ts
```

View File

@@ -19,18 +19,17 @@ dep: ffmpeg
def det(tf):
# fmt: off
sp.check_call([
b"ffmpeg",
b"-nostdin",
b"-hide_banner",
b"-v", b"fatal",
b"-ss", b"13",
b"-y", b"-i", fsenc(sys.argv[1]),
b"-map", b"0:a:0",
b"-ac", b"1",
b"-ar", b"22050",
b"-t", b"300",
b"-f", b"f32le",
fsenc(tf)
"ffmpeg",
"-nostdin",
"-hide_banner",
"-v", "fatal",
"-ss", "13",
"-y", "-i", fsenc(sys.argv[1]),
"-ac", "1",
"-ar", "22050",
"-t", "300",
"-f", "f32le",
tf
])
# fmt: on

View File

@@ -23,15 +23,14 @@ dep: ffmpeg
def det(tf):
# fmt: off
sp.check_call([
b"ffmpeg",
b"-nostdin",
b"-hide_banner",
b"-v", b"fatal",
b"-y", b"-i", fsenc(sys.argv[1]),
b"-map", b"0:a:0",
b"-t", b"300",
b"-sample_fmt", b"s16",
fsenc(tf)
"ffmpeg",
"-nostdin",
"-hide_banner",
"-v", "fatal",
"-y", "-i", fsenc(sys.argv[1]),
"-t", "300",
"-sample_fmt", "s16",
tf
])
# fmt: on

View File

@@ -1,89 +0,0 @@
#!/usr/bin/env python3
import sys
import json
import zlib
import struct
import base64
import hashlib
try:
from copyparty.util import fsenc
except:
def fsenc(p):
return p
"""
calculates various checksums for uploads,
usage: -mtp crc32,md5,sha1,sha256b=ad,bin/mtag/cksum.py
"""
def main():
config = "crc32 md5 md5b sha1 sha1b sha256 sha256b sha512/240 sha512b/240"
# b suffix = base64 encoded
# slash = truncate to n bits
known = {
"md5": hashlib.md5,
"sha1": hashlib.sha1,
"sha256": hashlib.sha256,
"sha512": hashlib.sha512,
}
config = config.split()
hashers = {
k: v()
for k, v in known.items()
if k in [x.split("/")[0].rstrip("b") for x in known]
}
crc32 = 0 if "crc32" in config else None
with open(fsenc(sys.argv[1]), "rb", 512 * 1024) as f:
while True:
buf = f.read(64 * 1024)
if not buf:
break
for x in hashers.values():
x.update(buf)
if crc32 is not None:
crc32 = zlib.crc32(buf, crc32)
ret = {}
for s in config:
alg = s.split("/")[0]
b64 = alg.endswith("b")
alg = alg.rstrip("b")
if alg in hashers:
v = hashers[alg].digest()
elif alg == "crc32":
v = crc32
if v < 0:
v &= 2 ** 32 - 1
v = struct.pack(">L", v)
else:
raise Exception("what is {}".format(s))
if "/" in s:
v = v[: int(int(s.split("/")[1]) / 8)]
if b64:
v = base64.b64encode(v).decode("ascii").rstrip("=")
else:
try:
v = v.hex()
except:
import binascii
v = binascii.hexlify(v)
ret[s] = v
print(json.dumps(ret, indent=4))
if __name__ == "__main__":
main()

View File

@@ -1,61 +0,0 @@
#!/usr/bin/env python3
"""
fetch latest msg from guestbook and return as tag
example copyparty config to use this:
--urlform save,get -vsrv/hello:hello:w:c,e2ts,mtp=guestbook=t10,ad,p,bin/mtag/guestbook-read.py:mte=+guestbook
explained:
for realpath srv/hello (served at /hello), write-only for eveyrone,
enable file analysis on upload (e2ts),
use mtp plugin "bin/mtag/guestbook-read.py" to provide metadata tag "guestbook",
do this on all uploads regardless of extension,
t10 = 10 seconds timeout for each dwonload,
ad = parse file regardless if FFmpeg thinks it is audio or not
p = request upload info as json on stdin (need ip)
mte=+guestbook enabled indexing of that tag for this volume
PS: this requires e2ts to be functional,
meaning you need to do at least one of these:
* apt install ffmpeg
* pip3 install mutagen
"""
import json
import os
import sqlite3
import sys
# set 0 to allow infinite msgs from one IP,
# other values delete older messages to make space,
# so 1 only keeps latest msg
NUM_MSGS_TO_KEEP = 1
def main():
fp = os.path.abspath(sys.argv[1])
fdir = os.path.dirname(fp)
zb = sys.stdin.buffer.read()
zs = zb.decode("utf-8", "replace")
md = json.loads(zs)
ip = md["up_ip"]
# can put the database inside `fdir` if you'd like,
# by default it saves to PWD:
# os.chdir(fdir)
db = sqlite3.connect("guestbook.db3")
with db:
t = "select msg from gb where ip = ? order by ts desc"
r = db.execute(t, (ip,)).fetchone()
if r:
print(r[0])
if __name__ == "__main__":
main()

View File

@@ -1,111 +0,0 @@
#!/usr/bin/env python3
"""
store messages from users in an sqlite database
which can be read from another mtp for example
takes input from application/x-www-form-urlencoded POSTs,
for example using the message/pager function on the website
example copyparty config to use this:
--urlform save,get -vsrv/hello:hello:w:c,e2ts,mtp=xgb=ebin,t10,ad,p,bin/mtag/guestbook.py:mte=+xgb
explained:
for realpath srv/hello (served at /hello),write-only for eveyrone,
enable file analysis on upload (e2ts),
use mtp plugin "bin/mtag/guestbook.py" to provide metadata tag "xgb",
do this on all uploads with the file extension "bin",
t300 = 300 seconds timeout for each dwonload,
ad = parse file regardless if FFmpeg thinks it is audio or not
p = request upload info as json on stdin
mte=+xgb enabled indexing of that tag for this volume
PS: this requires e2ts to be functional,
meaning you need to do at least one of these:
* apt install ffmpeg
* pip3 install mutagen
"""
import json
import os
import sqlite3
import sys
from urllib.parse import unquote_to_bytes as unquote
# set 0 to allow infinite msgs from one IP,
# other values delete older messages to make space,
# so 1 only keeps latest msg
NUM_MSGS_TO_KEEP = 1
def main():
fp = os.path.abspath(sys.argv[1])
fdir = os.path.dirname(fp)
fname = os.path.basename(fp)
if not fname.startswith("put-") or not fname.endswith(".bin"):
raise Exception("not a post file")
zb = sys.stdin.buffer.read()
zs = zb.decode("utf-8", "replace")
md = json.loads(zs)
buf = b""
with open(fp, "rb") as f:
while True:
b = f.read(4096)
buf += b
if len(buf) > 4096:
raise Exception("too big")
if not b:
break
if not buf:
raise Exception("file is empty")
buf = unquote(buf.replace(b"+", b" "))
txt = buf.decode("utf-8")
if not txt.startswith("msg="):
raise Exception("does not start with msg=")
ip = md["up_ip"]
ts = md["up_at"]
txt = txt[4:]
# can put the database inside `fdir` if you'd like,
# by default it saves to PWD:
# os.chdir(fdir)
db = sqlite3.connect("guestbook.db3")
try:
db.execute("select 1 from gb").fetchone()
except:
with db:
db.execute("create table gb (ip text, ts real, msg text)")
db.execute("create index gb_ip on gb(ip)")
with db:
if NUM_MSGS_TO_KEEP == 1:
t = "delete from gb where ip = ?"
db.execute(t, (ip,))
t = "insert into gb values (?,?,?)"
db.execute(t, (ip, ts, txt))
if NUM_MSGS_TO_KEEP > 1:
t = "select ts from gb where ip = ? order by ts desc"
hits = db.execute(t, (ip,)).fetchall()
if len(hits) > NUM_MSGS_TO_KEEP:
lim = hits[NUM_MSGS_TO_KEEP][0]
t = "delete from gb where ip = ? and ts <= ?"
db.execute(t, (ip, lim))
print(txt)
if __name__ == "__main__":
main()

View File

@@ -1,95 +0,0 @@
#!/usr/bin/env python3
"""
remove exif tags from uploaded images
dependencies:
exiftool
about:
creates a "noexif" subfolder and puts exif-stripped copies of each image there,
the reason for the subfolder is to avoid issues with the up2k.db / deduplication:
if the original image is modified in-place, then copyparty will keep the original
hash in up2k.db for a while (until the next volume rescan), so if the image is
reuploaded after a rescan then the upload will be renamed and kept as a dupe
alternatively you could switch the logic around, making a copy of the original
image into a subfolder named "exif" and modify the original in-place, but then
up2k.db will be out of sync until the next rescan, so any additional uploads
of the same image will get symlinked (deduplicated) to the modified copy
instead of the original in "exif"
or maybe delete the original image after processing, that would kinda work too
example copyparty config to use this:
-v/mnt/nas/pics:pics:rwmd,ed:c,e2ts,mte=+noexif:c,mtp=noexif=ejpg,ejpeg,ad,bin/mtag/image-noexif.py
explained:
for realpath /mnt/nas/pics (served at /pics) with read-write-modify-delete for ed,
enable file analysis on upload (e2ts),
append "noexif" to the list of known tags (mtp),
and use mtp plugin "bin/mtag/image-noexif.py" to provide that tag,
do this on all uploads with the file extension "jpg" or "jpeg",
ad = parse file regardless if FFmpeg thinks it is audio or not
PS: this requires e2ts to be functional,
meaning you need to do at least one of these:
* apt install ffmpeg
* pip3 install mutagen
and your python must have sqlite3 support compiled in
"""
import os
import sys
import filecmp
import subprocess as sp
try:
from copyparty.util import fsenc
except:
def fsenc(p):
return p.encode("utf-8")
def main():
cwd, fn = os.path.split(sys.argv[1])
if os.path.basename(cwd) == "noexif":
return
os.chdir(cwd)
f1 = fsenc(fn)
f2 = os.path.join(b"noexif", f1)
cmd = [
b"exiftool",
b"-exif:all=",
b"-iptc:all=",
b"-xmp:all=",
b"-P",
b"-o",
b"noexif/",
b"--",
f1,
]
sp.check_output(cmd)
if not os.path.exists(f2):
print("failed")
return
if filecmp.cmp(f1, f2, shallow=False):
print("clean")
else:
print("exif")
# lastmod = os.path.getmtime(f1)
# times = (int(time.time()), int(lastmod))
# os.utime(f2, times)
if __name__ == "__main__":
try:
main()
except:
pass

View File

@@ -4,9 +4,7 @@ set -e
# install dependencies for audio-*.py
#
# linux/alpine: requires gcc g++ make cmake patchelf {python3,ffmpeg,fftw,libsndfile}-dev py3-{wheel,pip} py3-numpy{,-dev}
# linux/debian: requires libav{codec,device,filter,format,resample,util}-dev {libfftw3,python3,libsndfile1}-dev python3-{numpy,pip} vamp-{plugin-sdk,examples} patchelf cmake
# linux/fedora: requires gcc gcc-c++ make cmake patchelf {python3,ffmpeg,fftw,libsndfile}-devel python3-numpy vamp-plugin-sdk qm-vamp-plugins
# linux: requires {python3,ffmpeg,fftw}-dev py3-{wheel,pip} py3-numpy{,-dev} vamp-sdk-dev patchelf
# win64: requires msys2-mingw64 environment
# macos: requires macports
#
@@ -102,11 +100,8 @@ export -f dl_files
github_tarball() {
rm -rf g
mkdir g
cd g
dl_text "$1" |
tee ../json |
tee json |
(
# prefer jq if available
jq -r '.tarball_url' ||
@@ -115,11 +110,8 @@ github_tarball() {
awk -F\" '/"tarball_url": "/ {print$4}'
) |
tee /dev/stderr |
head -n 1 |
tr -d '\r' | tr '\n' '\0' |
xargs -0 bash -c 'dl_files "$@"' _
mv * ../tgz
cd ..
}
@@ -134,7 +126,6 @@ gitlab_tarball() {
tr \" '\n' | grep -E '\.tar\.gz$' | head -n 1
) |
tee /dev/stderr |
head -n 1 |
tr -d '\r' | tr '\n' '\0' |
tee links |
xargs -0 bash -c 'dl_files "$@"' _
@@ -146,27 +137,20 @@ install_keyfinder() {
# use msys2 in mingw-w64 mode
# pacman -S --needed mingw-w64-x86_64-{ffmpeg,python}
[ -e $HOME/pe/keyfinder ] && {
echo found a keyfinder build in ~/pe, skipping
return
}
cd "$td"
github_tarball https://api.github.com/repos/mixxxdj/libkeyfinder/releases/latest
ls -al
tar -xf tgz
rm tgz
tar -xf mixxxdj-libkeyfinder-*
rm -- *.tar.gz
cd mixxxdj-libkeyfinder*
h="$HOME"
so="lib/libkeyfinder.so"
memes=(-DBUILD_TESTING=OFF)
memes=()
[ $win ] &&
so="bin/libkeyfinder.dll" &&
h="$(printf '%s\n' "$USERPROFILE" | tr '\\' '/')" &&
memes+=(-G "MinGW Makefiles")
memes+=(-G "MinGW Makefiles" -DBUILD_TESTING=OFF)
[ $mac ] &&
so="lib/libkeyfinder.dylib"
@@ -186,7 +170,7 @@ install_keyfinder() {
}
# rm -rf /Users/ed/Library/Python/3.9/lib/python/site-packages/*keyfinder*
CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include -I/usr/include/ffmpeg" \
CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include" \
LDFLAGS="-L$h/pe/keyfinder/lib -L$h/pe/keyfinder/lib64 -L/opt/local/lib" \
PKG_CONFIG_PATH=/c/msys64/mingw64/lib/pkgconfig \
$pybin -m pip install --user keyfinder
@@ -223,22 +207,6 @@ install_vamp() {
$pybin -m pip install --user vamp
cd "$td"
echo '#include <vamp-sdk/Plugin.h>' | gcc -x c -c -o /dev/null - || [ -e ~/pe/vamp-sdk ] || {
printf '\033[33mcould not find the vamp-sdk, building from source\033[0m\n'
(dl_files yolo https://code.soundsoftware.ac.uk/attachments/download/2588/vamp-plugin-sdk-2.9.0.tar.gz)
sha512sum -c <(
echo "7ef7f837d19a08048b059e0da408373a7964ced452b290fae40b85d6d70ca9000bcfb3302cd0b4dc76cf2a848528456f78c1ce1ee0c402228d812bd347b6983b -"
) <vamp-plugin-sdk-2.9.0.tar.gz
tar -xf vamp-plugin-sdk-2.9.0.tar.gz
rm -- *.tar.gz
ls -al
cd vamp-plugin-sdk-*
./configure --prefix=$HOME/pe/vamp-sdk
make -j1 install
}
cd "$td"
have_beatroot || {
printf '\033[33mcould not find the vamp beatroot plugin, building from source\033[0m\n'
(dl_files yolo https://code.soundsoftware.ac.uk/attachments/download/885/beatroot-vamp-v1.0.tar.gz)
@@ -246,11 +214,8 @@ install_vamp() {
echo "1f444d1d58ccf565c0adfe99f1a1aa62789e19f5071e46857e2adfbc9d453037bc1c4dcb039b02c16240e9b97f444aaff3afb625c86aa2470233e711f55b6874 -"
) <beatroot-vamp-v1.0.tar.gz
tar -xf beatroot-vamp-v1.0.tar.gz
rm -- *.tar.gz
cd beatroot-vamp-v1.0
[ -e ~/pe/vamp-sdk ] &&
sed -ri 's`^(CFLAGS :=.*)`\1 -I'$HOME'/pe/vamp-sdk/include`' Makefile.linux
make -f Makefile.linux -j4 LDFLAGS=-L$HOME/pe/vamp-sdk/lib
make -f Makefile.linux -j4
# /home/ed/vamp /home/ed/.vamp /usr/local/lib/vamp
mkdir ~/vamp
cp -pv beatroot-vamp.* ~/vamp/
@@ -264,7 +229,6 @@ install_vamp() {
# not in use because it kinda segfaults, also no windows support
install_soundtouch() {
cd "$td"
gitlab_tarball https://gitlab.com/api/v4/projects/soundtouch%2Fsoundtouch/releases
tar -xvf soundtouch-*

View File

@@ -1,73 +0,0 @@
#!/usr/bin/env python
import re
import sys
import json
import time
import base64
import hashlib
import subprocess as sp
try:
from copyparty.util import fsenc
except:
def fsenc(p):
return p.encode("utf-8")
"""
dep: ffmpeg
"""
def det():
# fmt: off
cmd = [
b"ffmpeg",
b"-nostdin",
b"-hide_banner",
b"-v", b"fatal",
b"-i", fsenc(sys.argv[1]),
b"-f", b"framemd5",
b"-"
]
# fmt: on
p = sp.Popen(cmd, stdout=sp.PIPE)
# ps = io.TextIOWrapper(p.stdout, encoding="utf-8")
ps = p.stdout
chans = {}
for ln in ps:
if ln.startswith(b"#stream#"):
break
m = re.match(r"^#media_type ([0-9]): ([a-zA-Z])", ln.decode("utf-8"))
if m:
chans[m.group(1)] = m.group(2)
hashers = [hashlib.sha512(), hashlib.sha512()]
for ln in ps:
n = int(ln[:1])
v = ln.rsplit(b",", 1)[-1].strip()
hashers[n].update(v)
r = {}
for k, v in chans.items():
dg = hashers[int(k)].digest()[:12]
dg = base64.urlsafe_b64encode(dg).decode("ascii")
r[v[0].lower() + "hash"] = dg
print(json.dumps(r, indent=4))
def main():
try:
det()
except:
pass # mute
if __name__ == "__main__":
main()

View File

@@ -1,38 +0,0 @@
#!/usr/bin/env python3
import os
import sys
import subprocess as sp
"""
mtp test -- opens a texteditor
usage:
-vsrv/v1:v1:r:c,mte=+x1:c,mtp=x1=ad,p,bin/mtag/mousepad.py
explained:
c,mte: list of tags to index in this volume
c,mtp: add new tag provider
x1: dummy tag to provide
ad: dontcare if audio or not
p: priority 1 (run after initial tag-scan with ffprobe or mutagen)
"""
def main():
env = os.environ.copy()
env["DISPLAY"] = ":0.0"
if False:
# open the uploaded file
fp = sys.argv[-1]
else:
# display stdin contents (`oth_tags`)
fp = "/dev/stdin"
p = sp.Popen(["/usr/bin/mousepad", fp])
p.communicate()
main()

View File

@@ -1,76 +0,0 @@
#!/usr/bin/env python
import json
import os
import subprocess as sp
import sys
import time
try:
from copyparty.util import fsenc
except:
def fsenc(p):
return p.encode("utf-8")
_ = r"""
first checks the tag "vidchk" which must be "ok" to continue,
then uploads all files to some cloud storage (RCLONE_REMOTE)
and DELETES THE ORIGINAL FILES if rclone returns 0 ("success")
deps:
rclone
usage:
-mtp x2=t43200,ay,p2,bin/mtag/rclone-upload.py
explained:
t43200: timeout 12h
ay: only process files which contain audio (including video with audio)
p2: set priority 2 (after vidchk's suggested priority of 1),
so the output of vidchk will be passed in here
complete usage example as vflags along with vidchk:
-vsrv/vidchk:vidchk:r:rw,ed:c,e2dsa,e2ts,mtp=vidchk=t600,p,bin/mtag/vidchk.py:c,mtp=rupload=t43200,ay,p2,bin/mtag/rclone-upload.py:c,mte=+vidchk,rupload
setup: see https://rclone.org/drive/
if you wanna use this script standalone / separately from copyparty,
either set CONDITIONAL_UPLOAD False or provide the following stdin:
{"vidchk":"ok"}
"""
RCLONE_REMOTE = "notmybox"
CONDITIONAL_UPLOAD = True
def main():
fp = sys.argv[1]
if CONDITIONAL_UPLOAD:
zb = sys.stdin.buffer.read()
zs = zb.decode("utf-8", "replace")
md = json.loads(zs)
chk = md.get("vidchk", None)
if chk != "ok":
print(f"vidchk={chk}", file=sys.stderr)
sys.exit(1)
dst = f"{RCLONE_REMOTE}:".encode("utf-8")
cmd = [b"rclone", b"copy", b"--", fsenc(fp), dst]
t0 = time.time()
try:
sp.check_call(cmd)
except:
print("rclone failed", file=sys.stderr)
sys.exit(1)
print(f"{time.time() - t0:.1f} sec")
os.unlink(fsenc(fp))
if __name__ == "__main__":
main()

View File

@@ -1,21 +0,0 @@
// ==UserScript==
// @name twitter-unmute
// @namespace http://ocv.me/
// @version 0.1
// @description memes
// @author ed <irc.rizon.net>
// @match https://twitter.com/*
// @icon https://www.google.com/s2/favicons?domain=twitter.com
// @grant GM_addStyle
// ==/UserScript==
function grunnur() {
setInterval(function () {
//document.querySelector('div[aria-label="Unmute"]').click();
document.querySelector('video').muted = false;
}, 200);
}
var scr = document.createElement('script');
scr.textContent = '(' + grunnur.toString() + ')();';
(document.head || document.getElementsByTagName('head')[0]).appendChild(scr);

View File

@@ -1,39 +0,0 @@
# example config file to use copyparty as a youtube manifest collector,
# use with copyparty like: python copyparty.py -c yt-ipr.conf
#
# see docs/example.conf for a better explanation of the syntax, but
# newlines are block separators, so adding blank lines inside a volume definition is bad
# (use comments as separators instead)
# create user ed, password wark
u ed:wark
# create a volume at /ytm which stores files at ./srv/ytm
./srv/ytm
/ytm
# write-only, but read-write for user ed
w
rw ed
# rescan the volume on startup
c e2dsa
# collect tags from all new files since last scan
c e2ts
# optionally enable compression to make the files 50% smaller
c pk
# only allow uploads which are between 16k and 1m large
c sz=16k-1m
# allow up to 10 uploads over 5 minutes from each ip
c maxn=10,300
# move uploads into subfolders: YEAR-MONTH / DAY-HOUR / <upload>
c rotf=%Y-%m/%d-%H
# delete uploads when they are 24 hours old
c lifetime=86400
# add the parser and tell copyparty what tags it can expect from it
c mtp=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires=bin/mtag/yt-ipr.py
# decide which tags we want to index and in what order
c mte=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires
# create any other volumes you'd like down here, or merge this with an existing config file

View File

@@ -1,47 +0,0 @@
// ==UserScript==
// @name youtube-playerdata-hub
// @match https://youtube.com/*
// @match https://*.youtube.com/*
// @version 1.0
// @grant GM_addStyle
// ==/UserScript==
function main() {
var server = 'https://127.0.0.1:3923/ytm?pw=wark',
interval = 60; // sec
var sent = {};
function send(txt, mf_url, desc) {
if (sent[mf_url])
return;
fetch(server + '&_=' + Date.now(), { method: "PUT", body: txt });
console.log('[yt-pdh] yeet %d bytes, %s', txt.length, desc);
sent[mf_url] = 1;
}
function collect() {
try {
var pd = document.querySelector('ytd-watch-flexy');
if (!pd)
return console.log('[yt-pdh] no video found');
pd = pd.playerData;
var mu = pd.streamingData.dashManifestUrl || pd.streamingData.hlsManifestUrl;
if (!mu || !mu.length)
return console.log('[yt-pdh] no manifest found');
var desc = pd.videoDetails.videoId + ', ' + pd.videoDetails.title;
send(JSON.stringify(pd), mu, desc);
}
catch (ex) {
console.log("[yt-pdh]", ex);
}
}
setInterval(collect, interval * 1000);
}
var scr = document.createElement('script');
scr.textContent = '(' + main.toString() + ')();';
(document.head || document.getElementsByTagName('head')[0]).appendChild(scr);
console.log('[yt-pdh] a');

View File

@@ -1,139 +0,0 @@
#!/usr/bin/env python3
"""
use copyparty as a chromecast replacement:
* post a URL and it will open in the default browser
* upload a file and it will open in the default application
* the `key` command simulates keyboard input
* the `x` command executes other xdotool commands
* the `c` command executes arbitrary unix commands
the android app makes it a breeze to post pics and links:
https://github.com/9001/party-up/releases
(iOS devices have to rely on the web-UI)
goes without saying, but this is HELLA DANGEROUS,
GIVES RCE TO ANYONE WHO HAVE UPLOAD PERMISSIONS
example copyparty config to use this:
--urlform save,get -v.::w:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,kn,c0,bin/mtag/very-bad-idea.py
recommended deps:
apt install xdotool libnotify-bin
https://github.com/9001/copyparty/blob/hovudstraum/contrib/plugins/meadup.js
and you probably want `twitter-unmute.user.js` from the res folder
-----------------------------------------------------------------------
-- startup script:
-----------------------------------------------------------------------
#!/bin/bash
set -e
# create qr code
ip=$(ip r | awk '/^default/{print$(NF-2)}'); echo http://$ip:3923/ | qrencode -o - -s 4 >/dev/shm/cpp-qr.png
/usr/bin/feh -x /dev/shm/cpp-qr.png &
# reposition and make topmost (with janky raspbian support)
( sleep 0.5
xdotool search --name cpp-qr.png windowactivate --sync windowmove 1780 0
wmctrl -r :ACTIVE: -b toggle,above || true
ps aux | grep -E 'sleep[ ]7\.27' ||
while true; do
w=$(xdotool getactivewindow)
xdotool search --name cpp-qr.png windowactivate windowraise windowfocus
xdotool windowactivate $w
xdotool windowfocus $w
sleep 7.27 || break
done &
xeyes # distraction window to prevent ^w from closing the qr-code
) &
# bail if copyparty is already running
ps aux | grep -E '[3] copy[p]arty' && exit 0
# dumb chrome wrapper to allow autoplay
cat >/usr/local/bin/chromium-browser <<'EOF'
#!/bin/bash
set -e
/usr/bin/chromium-browser --autoplay-policy=no-user-gesture-required "$@"
EOF
chmod 755 /usr/local/bin/chromium-browser
# start the server (note: replace `-v.::rw:` with `-v.::w:` to disallow retrieving uploaded stuff)
cd ~/Downloads; python3 copyparty-sfx.py --urlform save,get -v.::rw:c,e2d,e2t,mte=+a1:c,mtp=a1=ad,kn,very-bad-idea.py
"""
import os
import sys
import time
import subprocess as sp
from urllib.parse import unquote_to_bytes as unquote
def main():
fp = os.path.abspath(sys.argv[1])
with open(fp, "rb") as f:
txt = f.read(4096)
if txt.startswith(b"msg="):
open_post(txt)
else:
open_url(fp)
def open_post(txt):
txt = unquote(txt.replace(b"+", b" ")).decode("utf-8")[4:]
try:
k, v = txt.split(" ", 1)
except:
open_url(txt)
if k == "key":
sp.call(["xdotool", "key"] + v.split(" "))
elif k == "x":
sp.call(["xdotool"] + v.split(" "))
elif k == "c":
env = os.environ.copy()
while " " in v:
v1, v2 = v.split(" ", 1)
if "=" not in v1:
break
ek, ev = v1.split("=", 1)
env[ek] = ev
v = v2
sp.call(v.split(" "), env=env)
else:
open_url(txt)
def open_url(txt):
ext = txt.rsplit(".")[-1].lower()
sp.call(["notify-send", "--", txt])
if ext not in ["jpg", "jpeg", "png", "gif", "webp"]:
# sp.call(["wmctrl", "-c", ":ACTIVE:"]) # closes the active window correctly
sp.call(["killall", "vlc"])
sp.call(["killall", "mpv"])
sp.call(["killall", "feh"])
time.sleep(0.5)
for _ in range(20):
sp.call(["xdotool", "key", "ctrl+w"]) # closes the open tab correctly
# else:
# sp.call(["xdotool", "getactivewindow", "windowminimize"]) # minimizes the focused windo
# close any error messages:
sp.call(["xdotool", "search", "--name", "Error", "windowclose"])
# sp.call(["xdotool", "key", "ctrl+alt+d"]) # doesnt work at all
# sp.call(["xdotool", "keydown", "--delay", "100", "ctrl+alt+d"])
# sp.call(["xdotool", "keyup", "ctrl+alt+d"])
sp.call(["xdg-open", txt])
main()

View File

@@ -1,131 +0,0 @@
#!/usr/bin/env python3
import json
import re
import os
import sys
import subprocess as sp
try:
from copyparty.util import fsenc
except:
def fsenc(p):
return p.encode("utf-8")
_ = r"""
inspects video files for errors and such
plus stores a bunch of metadata to filename.ff.json
usage:
-mtp vidchk=t600,ay,p,bin/mtag/vidchk.py
explained:
t600: timeout 10min
ay: only process files which contain audio (including video with audio)
p: set priority 1 (lowest priority after initial ffprobe/mutagen for base tags),
makes copyparty feed base tags into this script as json
if you wanna use this script standalone / separately from copyparty,
provide the video resolution on stdin as json: {"res":"1920x1080"}
"""
FAST = True # parse entire file at container level
# FAST = False # fully decode audio and video streams
# warnings to ignore
harmless = re.compile(
r"Unsupported codec with id |Could not find codec parameters.*Attachment:|analyzeduration"
+ r"|timescale not set"
)
def wfilter(lines):
return [x for x in lines if x.strip() and not harmless.search(x)]
def errchk(so, se, rc, dbg):
if dbg:
with open(dbg, "wb") as f:
f.write(b"so:\n" + so + b"\nse:\n" + se + b"\n")
if rc:
err = (so + se).decode("utf-8", "replace").split("\n", 1)
err = wfilter(err) or err
return f"ERROR {rc}: {err[0]}"
if se:
err = se.decode("utf-8", "replace").split("\n", 1)
err = wfilter(err)
if err:
return f"Warning: {err[0]}"
return None
def main():
fp = sys.argv[1]
zb = sys.stdin.buffer.read()
zs = zb.decode("utf-8", "replace")
md = json.loads(zs)
fdir = os.path.dirname(os.path.realpath(fp))
flag = os.path.join(fdir, ".processed")
if os.path.exists(flag):
return "already processed"
try:
w, h = [int(x) for x in md["res"].split("x")]
if not w + h:
raise Exception()
except:
return "could not determine resolution"
# grab streams/format metadata + 2 seconds of frames at the start and end
zs = "ffprobe -hide_banner -v warning -of json -show_streams -show_format -show_packets -show_data_hash crc32 -read_intervals %+2,999999%+2"
cmd = zs.encode("ascii").split(b" ") + [fsenc(fp)]
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
so, se = p.communicate()
# spaces to tabs, drops filesize from 69k to 48k
so = b"\n".join(
[
b"\t" * int((len(x) - len(x.lstrip())) / 4) + x.lstrip()
for x in (so or b"").split(b"\n")
]
)
with open(fsenc(f"{fp}.ff.json"), "wb") as f:
f.write(so)
err = errchk(so, se, p.returncode, f"{fp}.vidchk")
if err:
return err
if max(w, h) < 1280 and min(w, h) < 720:
return "resolution too small"
zs = (
"ffmpeg -y -hide_banner -nostdin -v warning"
+ " -err_detect +crccheck+bitstream+buffer+careful+compliant+aggressive+explode"
+ " -xerror -i"
)
cmd = zs.encode("ascii").split(b" ") + [fsenc(fp)]
if FAST:
zs = "-c copy -f null -"
else:
zs = "-vcodec rawvideo -acodec pcm_s16le -f null -"
cmd += zs.encode("ascii").split(b" ")
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
so, se = p.communicate()
return errchk(so, se, p.returncode, f"{fp}.vidchk")
if __name__ == "__main__":
print(main() or "ok")

View File

@@ -1,90 +0,0 @@
#!/usr/bin/env python3
"""
DEPRECATED -- replaced by event hooks;
https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/wget.py
---
use copyparty as a file downloader by POSTing URLs as
application/x-www-form-urlencoded (for example using the
message/pager function on the website)
example copyparty config to use this:
--urlform save,get -vsrv/wget:wget:rwmd,ed:c,e2ts,mtp=title=ebin,t300,ad,bin/mtag/wget.py
explained:
for realpath srv/wget (served at /wget) with read-write-modify-delete for ed,
enable file analysis on upload (e2ts),
use mtp plugin "bin/mtag/wget.py" to provide metadata tag "title",
do this on all uploads with the file extension "bin",
t300 = 300 seconds timeout for each dwonload,
ad = parse file regardless if FFmpeg thinks it is audio or not
PS: this requires e2ts to be functional,
meaning you need to do at least one of these:
* apt install ffmpeg
* pip3 install mutagen
"""
import os
import sys
import subprocess as sp
from urllib.parse import unquote_to_bytes as unquote
def main():
fp = os.path.abspath(sys.argv[1])
fdir = os.path.dirname(fp)
fname = os.path.basename(fp)
if not fname.startswith("put-") or not fname.endswith(".bin"):
raise Exception("not a post file")
buf = b""
with open(fp, "rb") as f:
while True:
b = f.read(4096)
buf += b
if len(buf) > 4096:
raise Exception("too big")
if not b:
break
if not buf:
raise Exception("file is empty")
buf = unquote(buf.replace(b"+", b" "))
url = buf.decode("utf-8")
if not url.startswith("msg="):
raise Exception("does not start with msg=")
url = url[4:]
if "://" not in url:
url = "https://" + url
os.chdir(fdir)
name = url.split("?")[0].split("/")[-1]
tfn = "-- DOWNLOADING " + name
open(tfn, "wb").close()
cmd = ["wget", "--trust-server-names", "--", url]
try:
sp.check_call(cmd)
# OPTIONAL:
# on success, delete the .bin file which contains the URL
os.unlink(fp)
except:
open("-- FAILED TO DONWLOAD " + name, "wb").close()
os.unlink(tfn)
print(url)
if __name__ == "__main__":
main()

View File

@@ -1,198 +0,0 @@
#!/usr/bin/env python
import re
import os
import sys
import gzip
import json
import base64
import string
import urllib.request
from datetime import datetime
"""
youtube initial player response
it's probably best to use this through a config file; see res/yt-ipr.conf
but if you want to use plain arguments instead then:
-v srv/ytm:ytm:w:rw,ed
:c,e2ts,e2dsa
:c,sz=16k-1m:c,maxn=10,300:c,rotf=%Y-%m/%d-%H
:c,mtp=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires=bin/mtag/yt-ipr.py
:c,mte=yt-id,yt-title,yt-author,yt-channel,yt-views,yt-private,yt-manifest,yt-expires
see res/yt-ipr.user.js for the example userscript to go with this
"""
def main():
try:
with gzip.open(sys.argv[1], "rt", encoding="utf-8", errors="replace") as f:
txt = f.read()
except:
with open(sys.argv[1], "r", encoding="utf-8", errors="replace") as f:
txt = f.read()
txt = "{" + txt.split("{", 1)[1]
try:
pd = json.loads(txt)
except json.decoder.JSONDecodeError as ex:
pd = json.loads(txt[: ex.pos])
# print(json.dumps(pd, indent=2))
if "videoDetails" in pd:
parse_youtube(pd)
else:
parse_freg(pd)
def get_expiration(url):
et = re.search(r"[?&]expire=([0-9]+)", url).group(1)
et = datetime.utcfromtimestamp(int(et))
return et.strftime("%Y-%m-%d, %H:%M")
def parse_youtube(pd):
vd = pd["videoDetails"]
sd = pd["streamingData"]
et = sd["adaptiveFormats"][0]["url"]
et = get_expiration(et)
mf = []
if "dashManifestUrl" in sd:
mf.append("dash")
if "hlsManifestUrl" in sd:
mf.append("hls")
r = {
"yt-id": vd["videoId"],
"yt-title": vd["title"],
"yt-author": vd["author"],
"yt-channel": vd["channelId"],
"yt-views": vd["viewCount"],
"yt-private": vd["isPrivate"],
# "yt-expires": sd["expiresInSeconds"],
"yt-manifest": ",".join(mf),
"yt-expires": et,
}
print(json.dumps(r))
freg_conv(pd)
def parse_freg(pd):
md = pd["metadata"]
r = {
"yt-id": md["id"],
"yt-title": md["title"],
"yt-author": md["channelName"],
"yt-channel": md["channelURL"].strip("/").split("/")[-1],
"yt-expires": get_expiration(list(pd["video"].values())[0]),
}
print(json.dumps(r))
def freg_conv(pd):
# based on getURLs.js v1.5 (2021-08-07)
# fmt: off
priority = {
"video": [
337, 315, 266, 138, # 2160p60
313, 336, # 2160p
308, # 1440p60
271, 264, # 1440p
335, 303, 299, # 1080p60
248, 169, 137, # 1080p
334, 302, 298, # 720p60
247, 136 # 720p
],
"audio": [
251, 141, 171, 140, 250, 249, 139
]
}
vid_id = pd["videoDetails"]["videoId"]
chan_id = pd["videoDetails"]["channelId"]
try:
thumb_url = pd["microformat"]["playerMicroformatRenderer"]["thumbnail"]["thumbnails"][0]["url"]
start_ts = pd["microformat"]["playerMicroformatRenderer"]["liveBroadcastDetails"]["startTimestamp"]
except:
thumb_url = f"https://img.youtube.com/vi/{vid_id}/maxresdefault.jpg"
start_ts = ""
# fmt: on
metadata = {
"title": pd["videoDetails"]["title"],
"id": vid_id,
"channelName": pd["videoDetails"]["author"],
"channelURL": "https://www.youtube.com/channel/" + chan_id,
"description": pd["videoDetails"]["shortDescription"],
"thumbnailUrl": thumb_url,
"startTimestamp": start_ts,
}
if [x for x in vid_id if x not in string.ascii_letters + string.digits + "_-"]:
print(f"malicious json", file=sys.stderr)
return
basepath = os.path.dirname(sys.argv[1])
thumb_fn = f"{basepath}/{vid_id}.jpg"
tmp_fn = f"{thumb_fn}.{os.getpid()}"
if not os.path.exists(thumb_fn) and (
thumb_url.startswith("https://img.youtube.com/vi/")
or thumb_url.startswith("https://i.ytimg.com/vi/")
):
try:
with urllib.request.urlopen(thumb_url) as fi:
with open(tmp_fn, "wb") as fo:
fo.write(fi.read())
os.rename(tmp_fn, thumb_fn)
except:
if os.path.exists(tmp_fn):
os.unlink(tmp_fn)
try:
with open(thumb_fn, "rb") as f:
thumb = base64.b64encode(f.read()).decode("ascii")
except:
thumb = "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/yQALCAABAAEBAREA/8wABgAQEAX/2gAIAQEAAD8A0s8g/9k="
metadata["thumbnail"] = "data:image/jpeg;base64," + thumb
ret = {
"metadata": metadata,
"version": "1.5",
"createTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
}
for stream, itags in priority.items():
for itag in itags:
url = None
for afmt in pd["streamingData"]["adaptiveFormats"]:
if itag == afmt["itag"]:
url = afmt["url"]
break
if url:
ret[stream] = {itag: url}
break
fn = f"{basepath}/{vid_id}.urls.json"
with open(fn, "w", encoding="utf-8", errors="replace") as f:
f.write(json.dumps(ret, indent=4))
if __name__ == "__main__":
try:
main()
except:
# raise
pass

View File

@@ -1,177 +0,0 @@
#!/usr/bin/env python3
"""
partyjournal.py: chronological history of uploads
2021-12-31, v0.1, ed <irc.rizon.net>, MIT-Licensed
https://github.com/9001/copyparty/blob/hovudstraum/bin/partyjournal.py
produces a chronological list of all uploads,
by collecting info from up2k databases and the filesystem
specify subnet `192.168.1.*` with argument `.=192.168.1.`,
affecting all successive mappings
usage:
./partyjournal.py > partyjournal.html .=192.168.1. cart=125 steen=114 steen=131 sleepy=121 fscarlet=144 ed=101 ed=123
"""
import sys
import base64
import sqlite3
import argparse
from datetime import datetime
from urllib.parse import quote_from_bytes as quote
from urllib.parse import unquote_to_bytes as unquote
FS_ENCODING = sys.getfilesystemencoding()
class APF(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
pass
##
## snibbed from copyparty
def s3dec(v):
if not v.startswith("//"):
return v
v = base64.urlsafe_b64decode(v.encode("ascii")[2:])
return v.decode(FS_ENCODING, "replace")
def quotep(txt):
btxt = txt.encode("utf-8", "replace")
quot1 = quote(btxt, safe=b"/")
quot1 = quot1.encode("ascii")
quot2 = quot1.replace(b" ", b"+")
return quot2.decode("utf-8", "replace")
def html_escape(s, quote=False, crlf=False):
"""html.escape but also newlines"""
s = s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
if quote:
s = s.replace('"', "&quot;").replace("'", "&#x27;")
if crlf:
s = s.replace("\r", "&#13;").replace("\n", "&#10;")
return s
## end snibs
##
def main():
ap = argparse.ArgumentParser(formatter_class=APF)
ap.add_argument("who", nargs="*")
ar = ap.parse_args()
imap = {}
subnet = ""
for v in ar.who:
if "=" not in v:
raise Exception("bad who: " + v)
k, v = v.split("=")
if k == ".":
subnet = v
continue
imap["{}{}".format(subnet, v)] = k
print(repr(imap), file=sys.stderr)
print(
"""\
<!DOCTYPE html>
<html lang="en">
<head><meta charset="utf-8"><style>
html, body {
color: #ccc;
background: #222;
font-family: sans-serif;
}
a {
color: #fc5;
}
td, th {
padding: .2em .5em;
border: 1px solid #999;
border-width: 0 1px 1px 0;
white-space: nowrap;
}
td:nth-child(1),
td:nth-child(2),
td:nth-child(3) {
font-family: monospace, monospace;
text-align: right;
}
tr:first-child {
position: sticky;
top: -1px;
}
th {
background: #222;
text-align: left;
}
</style></head><body><table><tr>
<th>wark</th>
<th>time</th>
<th>size</th>
<th>who</th>
<th>link</th>
</tr>"""
)
db_path = ".hist/up2k.db"
conn = sqlite3.connect(db_path)
q = r"pragma table_info(up)"
inf = conn.execute(q).fetchall()
cols = [x[1] for x in inf]
print("<!-- " + str(cols) + " -->")
# ['w', 'mt', 'sz', 'rd', 'fn', 'ip', 'at']
q = r"select * from up order by case when at > 0 then at else mt end"
for w, mt, sz, rd, fn, ip, at in conn.execute(q):
link = "/".join([s3dec(x) for x in [rd, fn] if x])
if fn.startswith("put-") and sz < 4096:
try:
with open(link, "rb") as f:
txt = f.read().decode("utf-8", "replace")
except:
continue
if txt.startswith("msg="):
txt = txt.encode("utf-8", "replace")
txt = unquote(txt.replace(b"+", b" "))
link = txt.decode("utf-8")[4:]
sz = "{:,}".format(sz)
v = [
w[:16],
datetime.utcfromtimestamp(at if at > 0 else mt).strftime(
"%Y-%m-%d %H:%M:%S"
),
sz,
imap.get(ip, ip),
]
row = "<tr>\n "
row += "\n ".join(["<td>{}</th>".format(x) for x in v])
row += '\n <td><a href="{}">{}</a></td>'.format(link, html_escape(link))
row += "\n</tr>"
print(row)
print("</table></body></html>")
if __name__ == "__main__":
main()

View File

@@ -1,128 +0,0 @@
#!/bin/bash
set -e
# runs copyparty (or any other program really) in a chroot
#
# assumption: these directories, and everything within, are owned by root
sysdirs=( /bin /lib /lib32 /lib64 /sbin /usr /etc/alternatives )
# error-handler
help() { cat <<'EOF'
usage:
./prisonparty.sh <ROOTDIR> <UID> <GID> [VOLDIR [VOLDIR...]] -- python3 copyparty-sfx.py [...]
example:
./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 copyparty-sfx.py -v /mnt/nas/music::rwmd
example for running straight from source (instead of using an sfx):
PYTHONPATH=$PWD ./prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt/nas/music -- python3 -um copyparty -v /mnt/nas/music::rwmd
note that if you have python modules installed as --user (such as bpm/key detectors),
you should add /home/foo/.local as a VOLDIR
EOF
exit 1
}
# read arguments
trap help EXIT
jail="$(realpath "$1")"; shift
uid="$1"; shift
gid="$1"; shift
vols=()
while true; do
v="$1"; shift
[ "$v" = -- ] && break # end of volumes
[ "$#" -eq 0 ] && break # invalid usage
vols+=( "$(realpath "$v")" )
done
pybin="$1"; shift
pybin="$(command -v "$pybin")"
pyarg=
while true; do
v="$1"
[ "${v:0:1}" = - ] || break
pyarg="$pyarg $v"
shift
done
cpp="$1"; shift
[ -d "$cpp" ] && cppdir="$PWD" || {
# sfx, not module
cpp="$(realpath "$cpp")"
cppdir="$(dirname "$cpp")"
}
trap - EXIT
# debug/vis
echo
echo "chroot-dir = $jail"
echo "user:group = $uid:$gid"
echo " copyparty = $cpp"
echo
printf '\033[33m%s\033[0m\n' "copyparty can access these folders and all their subdirectories:"
for v in "${vols[@]}"; do
printf '\033[36m ├─\033[0m %s \033[36m ── added by (You)\033[0m\n' "$v"
done
printf '\033[36m ├─\033[0m %s \033[36m ── where the copyparty binary is\033[0m\n' "$cppdir"
printf '\033[36m ╰─\033[0m %s \033[36m ── the folder you are currently in\033[0m\n' "$PWD"
vols+=("$cppdir" "$PWD")
echo
# remove any trailing slashes
jail="${jail%/}"
# bind-mount system directories and volumes
printf '%s\n' "${sysdirs[@]}" "${vols[@]}" | sed -r 's`/$``' | LC_ALL=C sort | uniq |
while IFS= read -r v; do
[ -e "$v" ] || {
# printf '\033[1;31mfolder does not exist:\033[0m %s\n' "/$v"
continue
}
i1=$(stat -c%D.%i "$v" 2>/dev/null || echo a)
i2=$(stat -c%D.%i "$jail$v" 2>/dev/null || echo b)
# echo "v [$v] i1 [$i1] i2 [$i2]"
[ $i1 = $i2 ] && continue
mkdir -p "$jail$v"
mount --bind "$v" "$jail$v"
done
cln() {
rv=$?
# cleanup if not in use
lsof "$jail" | grep -qF "$jail" &&
echo "chroot is in use, will not cleanup" ||
{
mount | grep -F " on $jail" |
awk '{sub(/ type .*/,"");sub(/.* on /,"");print}' |
LC_ALL=C sort -r | tee /dev/stderr | tr '\n' '\0' | xargs -r0 umount
}
exit $rv
}
trap cln EXIT
# create a tmp
mkdir -p "$jail/tmp"
chmod 777 "$jail/tmp"
# run copyparty
export HOME=$(getent passwd $uid | cut -d: -f6)
export USER=$(getent passwd $uid | cut -d: -f1)
export LOGNAME="$USER"
#echo "pybin [$pybin]"
#echo "pyarg [$pyarg]"
#echo "cpp [$cpp]"
chroot --userspec=$uid:$gid "$jail" "$pybin" $pyarg "$cpp" "$@" &
p=$!
trap 'kill $p' INT TERM
wait

View File

@@ -1,99 +0,0 @@
#!/usr/bin/env python3
"""
unforget.py: rebuild db from logfiles
2022-09-07, v0.1, ed <irc.rizon.net>, MIT-Licensed
https://github.com/9001/copyparty/blob/hovudstraum/bin/unforget.py
only makes sense if running copyparty with --no-forget
(e.g. immediately shifting uploads to other storage)
usage:
xz -d < log | ./unforget.py .hist/up2k.db
"""
import re
import sys
import json
import base64
import sqlite3
import argparse
FS_ENCODING = sys.getfilesystemencoding()
class APF(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
pass
mem_cur = sqlite3.connect(":memory:").cursor()
mem_cur.execute(r"create table a (b text)")
def s3enc(rd: str, fn: str) -> tuple[str, str]:
ret: list[str] = []
for v in [rd, fn]:
try:
mem_cur.execute("select * from a where b = ?", (v,))
ret.append(v)
except:
wtf8 = v.encode(FS_ENCODING, "surrogateescape")
ret.append("//" + base64.urlsafe_b64encode(wtf8).decode("ascii"))
return ret[0], ret[1]
def main():
ap = argparse.ArgumentParser()
ap.add_argument("db")
ar = ap.parse_args()
db = sqlite3.connect(ar.db).cursor()
ptn_times = re.compile(r"no more chunks, setting times \(([0-9]+)")
at = 0
ctr = 0
for ln in [x.decode("utf-8", "replace").rstrip() for x in sys.stdin.buffer]:
if "no more chunks, setting times (" in ln:
m = ptn_times.search(ln)
if m:
at = int(m.group(1))
if '"hash": []' in ln:
try:
ofs = ln.find("{")
j = json.loads(ln[ofs:])
except:
pass
w = j["wark"]
if db.execute("select w from up where w = ?", (w,)).fetchone():
continue
# PYTHONPATH=/home/ed/dev/copyparty/ python3 -m copyparty -e2dsa -v foo:foo:rwmd,ed -aed:wark --no-forget
# 05:34:43.845 127.0.0.1 42496 no more chunks, setting times (1662528883, 1658001882)
# 05:34:43.863 127.0.0.1 42496 {"name": "f\"2", "purl": "/foo/bar/baz/", "size": 1674, "lmod": 1658001882, "sprs": true, "hash": [], "wark": "LKIWpp2jEAh9dH3fu-DobuURFGEKlODXDGTpZ1otMhUg"}
# | w | mt | sz | rd | fn | ip | at |
# | LKIWpp2jEAh9dH3fu-DobuURFGEKlODXDGTpZ1otMhUg | 1658001882 | 1674 | bar/baz | f"2 | 127.0.0.1 | 1662528883 |
rd, fn = s3enc(j["purl"].strip("/"), j["name"])
ip = ln.split(" ")[1].split("m")[-1]
q = "insert into up values (?,?,?,?,?,?,?)"
v = (w, int(j["lmod"]), int(j["size"]), rd, fn, ip, at)
db.execute(q, v)
ctr += 1
if ctr % 1024 == 1023:
print(f"{ctr} commit...")
db.connection.commit()
if ctr:
db.connection.commit()
print(f"unforgot {ctr} files")
if __name__ == "__main__":
main()

File diff suppressed because it is too large Load Diff

24
bin/up2k.sh Normal file → Executable file
View File

@@ -8,7 +8,7 @@ set -e
##
## config
datalen=$((128*1024*1024))
datalen=$((2*1024*1024*1024))
target=127.0.0.1
posturl=/inc
passwd=wark
@@ -37,10 +37,10 @@ gendata() {
# pipe a chunk, get the base64 checksum
gethash() {
printf $(
sha512sum | cut -c-66 |
sha512sum | cut -c-64 |
sed -r 's/ .*//;s/(..)/\\x\1/g'
) |
base64 -w0 | cut -c-44 |
base64 -w0 | cut -c-43 |
tr '+/' '-_'
}
@@ -123,7 +123,7 @@ printf '\033[36m'
{
{
cat <<EOF
POST $posturl/ HTTP/1.1
POST $posturl/handshake.php HTTP/1.1
Connection: Close
Cookie: cppwd=$passwd
Content-Type: text/plain;charset=UTF-8
@@ -145,16 +145,14 @@ printf '\033[0m\nwark: %s\n' $wark
##
## wait for signal to continue
true || {
w8=/dev/shm/$salt.w8
touch $w8
w8=/dev/shm/$salt.w8
touch $w8
echo "ready; rm -f $w8"
echo "ready; rm -f $w8"
while [ -e $w8 ]; do
sleep 0.2
done
}
while [ -e $w8 ]; do
sleep 0.2
done
##
@@ -177,7 +175,7 @@ while [ $remains -gt 0 ]; do
{
cat <<EOF
POST $posturl/ HTTP/1.1
POST $posturl/chunkpit.php HTTP/1.1
Connection: Keep-Alive
Cookie: cppwd=$passwd
Content-Type: application/octet-stream

View File

@@ -1,6 +1,3 @@
### [`plugins/`](plugins/)
* example extensions
### [`copyparty.bat`](copyparty.bat)
* launches copyparty with no arguments (anon read+write within same folder)
* intended for windows machines with no python.exe in PATH
@@ -22,29 +19,17 @@ however if your copyparty is behind a reverse-proxy, you may want to use [`share
* `URL`: full URL to the root folder (with trailing slash) followed by `$regex:1|1$`
* `pw`: password (remove `Parameters` if anon-write)
### [`media-osd-bgone.ps1`](media-osd-bgone.ps1)
* disables the [windows OSD popup](https://user-images.githubusercontent.com/241032/122821375-0e08df80-d2dd-11eb-9fd9-184e8aacf1d0.png) (the thing on the left) which appears every time you hit media hotkeys to adjust volume or change song while playing music with the copyparty web-ui, or most other audio players really
### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
* disables thumbnails and folder-type detection in windows explorer
* makes it way faster (especially for slow/networked locations (such as partyfuse))
### [`webdav-cfg.reg`](webdav-cfg.bat)
* improves the native webdav support in windows;
* removes the 47.6 MiB filesize limit when downloading from webdav
* optionally enables webdav basic-auth over plaintext http
* optionally helps disable wpad, removing the 10sec latency
* makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
### [`cfssl.sh`](cfssl.sh)
* creates CA and server certificates using cfssl
* give a 3rd argument to install it to your copyparty config
* systemd service at [`systemd/cfssl.service`](systemd/cfssl.service)
# OS integration
init-scripts to start copyparty as a service
* [`systemd/copyparty.service`](systemd/copyparty.service) runs the sfx normally
* [`rc/copyparty`](rc/copyparty) runs sfx normally on freebsd, create a `copyparty` user
* [`systemd/prisonparty.service`](systemd/prisonparty.service) runs the sfx in a chroot
* [`systemd/copyparty.service`](systemd/copyparty.service)
* [`openrc/copyparty`](openrc/copyparty)
# Reverse-proxy

View File

@@ -1,15 +0,0 @@
# when running copyparty behind a reverse proxy,
# the following arguments are recommended:
#
# --http-only lower latency on initial connection
# -i 127.0.0.1 only accept connections from nginx
#
# if you are doing location-based proxying (such as `/stuff` below)
# you must run copyparty with --rp-loc=stuff
#
# on fedora/rhel, remember to setsebool -P httpd_can_network_connect 1
LoadModule proxy_module modules/mod_proxy.so
ProxyPass "/stuff" "http://127.0.0.1:3923/stuff"
# do not specify ProxyPassReverse
RequestHeader set "X-Forwarded-Proto" expr=%{REQUEST_SCHEME}

View File

@@ -1,14 +1,13 @@
#!/bin/bash
set -e
# ca-name and server-fqdn
# ca-name and server-name
ca_name="$1"
srv_fqdn="$2"
srv_name="$2"
[ -z "$srv_fqdn" ] && {
[ -z "$srv_name" ] && {
echo "need arg 1: ca name"
echo "need arg 2: server fqdn and/or IPs, comma-separated"
echo "optional arg 3: if set, write cert into copyparty cfg"
echo "need arg 2: server name"
exit 1
}
@@ -32,15 +31,15 @@ EOF
gen_srv() {
(tee /dev/stderr <<EOF
{"key": {"algo":"rsa", "size":4096},
"names": [{"O":"$ca_name - $srv_fqdn"}]}
"names": [{"O":"$ca_name - $srv_name"}]}
EOF
)|
cfssl gencert -ca ca.pem -ca-key ca.key \
-profile=www -hostname="$srv_fqdn" - |
cfssljson -bare "$srv_fqdn"
-profile=www -hostname="$srv_name.$ca_name" - |
cfssljson -bare "$srv_name"
mv "$srv_fqdn-key.pem" "$srv_fqdn.key"
rm "$srv_fqdn.csr"
mv "$srv_name-key.pem" "$srv_name.key"
rm "$srv_name.csr"
}
@@ -58,13 +57,13 @@ show() {
awk '!o; {o=0} /[0-9a-f:]{16}/{o=1}'
}
show ca.pem
show "$srv_fqdn.pem"
show "$srv_name.pem"
# write cert into copyparty config
[ -z "$3" ] || {
mkdir -p ~/.config/copyparty
cat "$srv_fqdn".{key,pem} ca.pem >~/.config/copyparty/cert.pem
cat "$srv_name".{key,pem} ca.pem >~/.config/copyparty/cert.pem
}

View File

@@ -1,104 +0,0 @@
# media-osd-bgone.ps1: disable media-control OSD on win10do
# v1.1, 2021-06-25, ed <irc.rizon.net>, MIT-licensed
# https://github.com/9001/copyparty/blob/hovudstraum/contrib/media-osd-bgone.ps1
#
# locates the first window that looks like the media OSD and minimizes it;
# doing this once after each reboot should do the trick
# (adjust the width/height filter if it doesn't work)
#
# ---------------------------------------------------------------------
#
# tip: save the following as "media-osd-bgone.bat" next to this script:
# start cmd /c "powershell -command ""set-executionpolicy -scope process bypass; .\media-osd-bgone.ps1"" & ping -n 2 127.1 >nul"
#
# then create a shortcut to that bat-file and move the shortcut here:
# %appdata%\Microsoft\Windows\Start Menu\Programs\Startup
#
# and now this will autorun on bootup
Add-Type -TypeDefinition @"
using System;
using System.IO;
using System.Threading;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Windows.Forms;
namespace A {
public class B : Control {
[DllImport("user32.dll")]
static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, int dwExtraInfo);
[DllImport("user32.dll", SetLastError = true)]
static extern IntPtr FindWindowEx(IntPtr hwndParent, IntPtr hwndChildAfter, string lpszClass, string lpszWindow);
[DllImport("user32.dll", SetLastError=true)]
static extern bool GetWindowRect(IntPtr hwnd, out RECT lpRect);
[DllImport("user32.dll")]
static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
[StructLayout(LayoutKind.Sequential)]
public struct RECT {
public int x;
public int y;
public int x2;
public int y2;
}
bool fa() {
RECT r;
IntPtr it = IntPtr.Zero;
while ((it = FindWindowEx(IntPtr.Zero, it, "NativeHWNDHost", "")) != IntPtr.Zero) {
if (FindWindowEx(it, IntPtr.Zero, "DirectUIHWND", "") == IntPtr.Zero)
continue;
if (!GetWindowRect(it, out r))
continue;
int w = r.x2 - r.x + 1;
int h = r.y2 - r.y + 1;
Console.WriteLine("[*] hwnd {0:x} @ {1}x{2} sz {3}x{4}", it, r.x, r.y, w, h);
if (h != 141)
continue;
ShowWindow(it, 6);
Console.WriteLine("[+] poof");
return true;
}
return false;
}
void fb() {
keybd_event((byte)Keys.VolumeMute, 0, 0, 0);
keybd_event((byte)Keys.VolumeMute, 0, 2, 0);
Thread.Sleep(500);
keybd_event((byte)Keys.VolumeMute, 0, 0, 0);
keybd_event((byte)Keys.VolumeMute, 0, 2, 0);
while (true) {
if (fa()) {
break;
}
Console.WriteLine("[!] not found");
Thread.Sleep(1000);
}
this.Invoke((MethodInvoker)delegate {
Application.Exit();
});
}
public void Run() {
Console.WriteLine("[+] hi");
new Thread(new ThreadStart(fb)).Start();
Application.Run();
Console.WriteLine("[+] bye");
}
}
}
"@ -ReferencedAssemblies System.Windows.Forms
(New-Object -TypeName A.B).Run()

View File

@@ -1,21 +1,11 @@
# when running copyparty behind a reverse proxy,
# the following arguments are recommended:
#
# --http-only lower latency on initial connection
# -i 127.0.0.1 only accept connections from nginx
#
# -nc must match or exceed the webserver's max number of concurrent clients;
# copyparty default is 1024 if OS permits it (see "max clients:" on startup),
# nginx default is 512 (worker_processes 1, worker_connections 512)
#
# you may also consider adding -j0 for CPU-intensive configurations
# (not that i can really think of any good examples)
#
# on fedora/rhel, remember to setsebool -P httpd_can_network_connect 1
# when running copyparty behind a reverse-proxy,
# make sure that copyparty allows at least as many clients as the proxy does,
# so run copyparty with -nc 512 if your nginx has the default limits
# (worker_processes 1, worker_connections 512)
upstream cpp {
server 127.0.0.1:3923;
keepalive 1;
keepalive 120;
}
server {
listen 443 ssl;

View File

@@ -8,11 +8,11 @@
#
# you may want to:
# change '/usr/bin/python' to another interpreter
# change '/mnt::rw' to another location or permission-set
# change '/mnt::a' to another location or permission-set
name="$SVCNAME"
command_background=true
pidfile="/var/run/$SVCNAME.pid"
command="/usr/bin/python3 /usr/local/bin/copyparty-sfx.py"
command_args="-q -v /mnt::rw"
command="/usr/bin/python /usr/local/bin/copyparty-sfx.py"
command_args="-q -v /mnt::a"

View File

@@ -1,33 +0,0 @@
# example resource files
can be provided to copyparty to tweak things
## example `.epilogue.html`
save one of these as `.epilogue.html` inside a folder to customize it:
* [`minimal-up2k.html`](minimal-up2k.html) will [simplify the upload ui](https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png)
## example browser-js
point `--js-browser` to one of these by URL:
* [`minimal-up2k.js`](minimal-up2k.js) is similar to the above `minimal-up2k.html` except it applies globally to all write-only folders
* [`up2k-hooks.js`](up2k-hooks.js) lets you specify a ruleset for files to skip uploading
* [`up2k-hook-ytid.js`](up2k-hook-ytid.js) is a more specific example checking youtube-IDs against some API
## example browser-css
point `--css-browser` to one of these by URL:
* [`browser-icons.css`](browser-icons.css) adds filetype icons
## meadup.js
* turns copyparty into chromecast just more flexible (and probably way more buggy)
* usage: put the js somewhere in the webroot and `--js-browser /memes/meadup.js`

View File

@@ -1,71 +0,0 @@
/* video, alternative 1:
top-left icon, just like the other formats
=======================================================================
#ggrid>a:is(
[href$=".mkv"i],
[href$=".mp4"i],
[href$=".webm"i],
):before {
content: '📺';
}
*/
/* video, alternative 2:
play-icon in the middle of the thumbnail
=======================================================================
*/
#ggrid>a:is(
[href$=".mkv"i],
[href$=".mp4"i],
[href$=".webm"i],
) {
position: relative;
overflow: hidden;
}
#ggrid>a:is(
[href$=".mkv"i],
[href$=".mp4"i],
[href$=".webm"i],
):before {
content: '▶';
opacity: .8;
margin: 0;
padding: 1em .5em 1em .7em;
border-radius: 9em;
line-height: 0;
color: #fff;
text-shadow: none;
background: rgba(0, 0, 0, 0.7);
left: calc(50% - 1em);
top: calc(50% - 1.4em);
}
/* audio */
#ggrid>a:is(
[href$=".mp3"i],
[href$=".ogg"i],
[href$=".opus"i],
[href$=".flac"i],
[href$=".m4a"i],
[href$=".aac"i],
):before {
content: '🎵';
}
/* image */
#ggrid>a:is(
[href$=".jpg"i],
[href$=".jpeg"i],
[href$=".png"i],
[href$=".gif"i],
[href$=".webp"i],
):before {
content: '🎨';
}

View File

@@ -1,506 +0,0 @@
// USAGE:
// place this file somewhere in the webroot and then
// python3 -m copyparty --js-browser /memes/meadup.js
//
// FEATURES:
// * adds an onscreen keyboard for operating a media center remotely,
// relies on https://github.com/9001/copyparty/blob/hovudstraum/bin/mtag/very-bad-idea.py
// * adds an interactive anime girl (if you can find the dependencies)
var hambagas = [
"https://www.youtube.com/watch?v=pFA3KGp4GuU"
];
// keybaord,
// onscreen keyboard by @steinuil
function initKeybaord(BASE_URL, HAMBAGA, consoleLog, consoleError) {
document.querySelector('.keybaord-container').innerHTML = `
<div class="keybaord-body">
<div class="keybaord-row keybaord-row-1">
<div class="keybaord-key" data-keybaord-key="Escape">
esc
</div>
<div class="keybaord-key" data-keybaord-key="F1">
F1
</div>
<div class="keybaord-key" data-keybaord-key="F2">
F2
</div>
<div class="keybaord-key" data-keybaord-key="F3">
F3
</div>
<div class="keybaord-key" data-keybaord-key="F4">
F4
</div>
<div class="keybaord-key" data-keybaord-key="F5">
F5
</div>
<div class="keybaord-key" data-keybaord-key="F6">
F6
</div>
<div class="keybaord-key" data-keybaord-key="F7">
F7
</div>
<div class="keybaord-key" data-keybaord-key="F8">
F8
</div>
<div class="keybaord-key" data-keybaord-key="F9">
F9
</div>
<div class="keybaord-key" data-keybaord-key="F10">
F10
</div>
<div class="keybaord-key" data-keybaord-key="F11">
F11
</div>
<div class="keybaord-key" data-keybaord-key="F12">
F12
</div>
<div class="keybaord-key" data-keybaord-key="Insert">
ins
</div>
<div class="keybaord-key" data-keybaord-key="Delete">
del
</div>
</div>
<div class="keybaord-row keybaord-row-2">
<div class="keybaord-key" data-keybaord-key="\`">
\`
</div>
<div class="keybaord-key" data-keybaord-key="1">
1
</div>
<div class="keybaord-key" data-keybaord-key="2">
2
</div>
<div class="keybaord-key" data-keybaord-key="3">
3
</div>
<div class="keybaord-key" data-keybaord-key="4">
4
</div>
<div class="keybaord-key" data-keybaord-key="5">
5
</div>
<div class="keybaord-key" data-keybaord-key="6">
6
</div>
<div class="keybaord-key" data-keybaord-key="7">
7
</div>
<div class="keybaord-key" data-keybaord-key="8">
8
</div>
<div class="keybaord-key" data-keybaord-key="9">
9
</div>
<div class="keybaord-key" data-keybaord-key="0">
0
</div>
<div class="keybaord-key" data-keybaord-key="-">
-
</div>
<div class="keybaord-key" data-keybaord-key="=">
=
</div>
<div class="keybaord-key keybaord-backspace" data-keybaord-key="BackSpace">
backspace
</div>
</div>
<div class="keybaord-row keybaord-row-3">
<div class="keybaord-key keybaord-tab" data-keybaord-key="Tab">
tab
</div>
<div class="keybaord-key" data-keybaord-key="q">
q
</div>
<div class="keybaord-key" data-keybaord-key="w">
w
</div>
<div class="keybaord-key" data-keybaord-key="e">
e
</div>
<div class="keybaord-key" data-keybaord-key="r">
r
</div>
<div class="keybaord-key" data-keybaord-key="t">
t
</div>
<div class="keybaord-key" data-keybaord-key="y">
y
</div>
<div class="keybaord-key" data-keybaord-key="u">
u
</div>
<div class="keybaord-key" data-keybaord-key="i">
i
</div>
<div class="keybaord-key" data-keybaord-key="o">
o
</div>
<div class="keybaord-key" data-keybaord-key="p">
p
</div>
<div class="keybaord-key" data-keybaord-key="[">
[
</div>
<div class="keybaord-key" data-keybaord-key="]">
]
</div>
<div class="keybaord-key keybaord-enter" data-keybaord-key="Return">
enter
</div>
</div>
<div class="keybaord-row keybaord-row-4">
<div class="keybaord-key keybaord-capslock" data-keybaord-key="HAMBAGA">
🍔
</div>
<div class="keybaord-key" data-keybaord-key="a">
a
</div>
<div class="keybaord-key" data-keybaord-key="s">
s
</div>
<div class="keybaord-key" data-keybaord-key="d">
d
</div>
<div class="keybaord-key" data-keybaord-key="f">
f
</div>
<div class="keybaord-key" data-keybaord-key="g">
g
</div>
<div class="keybaord-key" data-keybaord-key="h">
h
</div>
<div class="keybaord-key" data-keybaord-key="j">
j
</div>
<div class="keybaord-key" data-keybaord-key="k">
k
</div>
<div class="keybaord-key" data-keybaord-key="l">
l
</div>
<div class="keybaord-key" data-keybaord-key=";">
;
</div>
<div class="keybaord-key" data-keybaord-key="'">
'
</div>
<div class="keybaord-key keybaord-backslash" data-keybaord-key="\\">
\\
</div>
</div>
<div class="keybaord-row keybaord-row-5">
<div class="keybaord-key keybaord-lshift" data-keybaord-key="Shift_L">
shift
</div>
<div class="keybaord-key" data-keybaord-key="\\">
\\
</div>
<div class="keybaord-key" data-keybaord-key="z">
z
</div>
<div class="keybaord-key" data-keybaord-key="x">
x
</div>
<div class="keybaord-key" data-keybaord-key="c">
c
</div>
<div class="keybaord-key" data-keybaord-key="v">
v
</div>
<div class="keybaord-key" data-keybaord-key="b">
b
</div>
<div class="keybaord-key" data-keybaord-key="n">
n
</div>
<div class="keybaord-key" data-keybaord-key="m">
m
</div>
<div class="keybaord-key" data-keybaord-key=",">
,
</div>
<div class="keybaord-key" data-keybaord-key=".">
.
</div>
<div class="keybaord-key" data-keybaord-key="/">
/
</div>
<div class="keybaord-key keybaord-rshift" data-keybaord-key="Shift_R">
shift
</div>
</div>
<div class="keybaord-row keybaord-row-6">
<div class="keybaord-key keybaord-lctrl" data-keybaord-key="Control_L">
ctrl
</div>
<div class="keybaord-key keybaord-super" data-keybaord-key="Meta_L">
win
</div>
<div class="keybaord-key keybaord-alt" data-keybaord-key="Alt_L">
alt
</div>
<div class="keybaord-key keybaord-spacebar" data-keybaord-key="space">
space
</div>
<div class="keybaord-key keybaord-altgr" data-keybaord-key="Alt_R">
altgr
</div>
<div class="keybaord-key keybaord-what" data-keybaord-key="Menu">
menu
</div>
<div class="keybaord-key keybaord-rctrl" data-keybaord-key="Control_R">
ctrl
</div>
</div>
<div class="keybaord-row">
<div class="keybaord-key" data-keybaord-key="XF86AudioLowerVolume">
🔉
</div>
<div class="keybaord-key" data-keybaord-key="XF86AudioRaiseVolume">
🔊
</div>
<div class="keybaord-key" data-keybaord-key="Left">
⬅️
</div>
<div class="keybaord-key" data-keybaord-key="Down">
⬇️
</div>
<div class="keybaord-key" data-keybaord-key="Up">
⬆️
</div>
<div class="keybaord-key" data-keybaord-key="Right">
➡️
</div>
<div class="keybaord-key" data-keybaord-key="Page_Up">
PgUp
</div>
<div class="keybaord-key" data-keybaord-key="Page_Down">
PgDn
</div>
<div class="keybaord-key" data-keybaord-key="Home">
🏠
</div>
<div class="keybaord-key" data-keybaord-key="End">
End
</div>
</div>
<div>
`;
function arraySample(array) {
return array[Math.floor(Math.random() * array.length)];
}
function sendMessage(msg) {
return fetch(BASE_URL, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
},
body: "msg=" + encodeURIComponent(msg),
}).then(
(r) => r.text(), // so the response body shows up in network tab
(err) => consoleError(err)
);
}
const MODIFIER_ON_CLASS = "keybaord-modifier-on";
const KEY_DATASET = "data-keybaord-key";
const KEY_CLASS = "keybaord-key";
const modifiers = new Set()
function toggleModifier(button, key) {
button.classList.toggle(MODIFIER_ON_CLASS);
if (modifiers.has(key)) {
modifiers.delete(key);
} else {
modifiers.add(key);
}
}
function popModifiers() {
let modifierString = "";
modifiers.forEach((mod) => {
document.querySelector("[" + KEY_DATASET + "='" + mod + "']")
.classList.remove(MODIFIER_ON_CLASS);
modifierString += mod + "+";
});
modifiers.clear();
return modifierString;
}
Array.from(document.querySelectorAll("." + KEY_CLASS)).forEach((button) => {
const key = button.dataset.keybaordKey;
button.addEventListener("click", (ev) => {
switch (key) {
case "HAMBAGA":
sendMessage(arraySample(HAMBAGA));
break;
case "Shift_L":
case "Shift_R":
case "Control_L":
case "Control_R":
case "Meta_L":
case "Alt_L":
case "Alt_R":
toggleModifier(button, key);
break;
default: {
const keyWithModifiers = popModifiers() + key;
consoleLog(keyWithModifiers);
sendMessage("key " + keyWithModifiers)
.then(() => consoleLog(keyWithModifiers + " OK"));
}
}
});
});
}
// keybaord integration
(function () {
var o = mknod('div');
clmod(o, 'keybaord-container', 1);
ebi('op_msg').appendChild(o);
o = mknod('style');
o.innerHTML = `
.keybaord-body {
display: flex;
flex-flow: column nowrap;
margin: .6em 0;
}
.keybaord-row {
display: flex;
}
.keybaord-key {
border: 1px solid rgba(128,128,128,0.2);
width: 41px;
height: 40px;
display: flex;
justify-content: center;
align-items: center;
}
.keybaord-key:active {
background-color: lightgrey;
}
.keybaord-key.keybaord-modifier-on {
background-color: lightblue;
}
.keybaord-key.keybaord-backspace {
width: 82px;
}
.keybaord-key.keybaord-tab {
width: 55px;
}
.keybaord-key.keybaord-enter {
width: 69px;
}
.keybaord-key.keybaord-capslock {
width: 80px;
}
.keybaord-key.keybaord-backslash {
width: 88px;
}
.keybaord-key.keybaord-lshift {
width: 65px;
}
.keybaord-key.keybaord-rshift {
width: 103px;
}
.keybaord-key.keybaord-lctrl {
width: 55px;
}
.keybaord-key.keybaord-super {
width: 55px;
}
.keybaord-key.keybaord-alt {
width: 55px;
}
.keybaord-key.keybaord-altgr {
width: 55px;
}
.keybaord-key.keybaord-what {
width: 55px;
}
.keybaord-key.keybaord-rctrl {
width: 55px;
}
.keybaord-key.keybaord-spacebar {
width: 302px;
}
`;
document.head.appendChild(o);
initKeybaord('/', hambagas,
(msg) => { toast.inf(2, msg.toString()) },
(msg) => { toast.err(30, msg.toString()) });
})();
// live2d (dumb pointless meme)
// dependencies for this part are not tracked in git
// so delete this section if you wanna use this file
// (or supply your own l2d model and js)
(function () {
var o = mknod('link');
o.setAttribute('rel', 'stylesheet');
o.setAttribute('href', "/bad-memes/pio.css");
document.head.appendChild(o);
o = mknod('style');
o.innerHTML = '.pio-container{text-shadow:none;z-index:1}';
document.head.appendChild(o);
o = mknod('div');
clmod(o, 'pio-container', 1);
o.innerHTML = '<div class="pio-action"></div><canvas id="pio" width="280" height="500"></canvas>';
document.body.appendChild(o);
var remaining = 3;
for (var a of ['pio', 'l2d', 'fireworks']) {
import_js(`/bad-memes/${a}.js`, function () {
if (remaining --> 1)
return;
o = mknod('script');
o.innerHTML = 'var pio = new Paul_Pio({"selector":[],"mode":"fixed","hidden":false,"content":{"close":"ok bye"},"model":["/bad-memes/sagiri/model.json"]});';
document.body.appendChild(o);
});
}
})();

View File

@@ -1,46 +0,0 @@
<!--
NOTE: DEPRECATED; please use the javascript version instead:
https://github.com/9001/copyparty/blob/hovudstraum/contrib/plugins/minimal-up2k.js
----
save this as .epilogue.html inside a write-only folder to declutter the UI, makes it look like
https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png
only works if you disable the prologue/epilogue sandbox with --no-sb-lg
which should probably be combined with --no-dot-ren to prevent damage
(`no_sb_lg` can also be set per-volume with volflags)
-->
<style>
/* make the up2k ui REALLY minimal by hiding a bunch of stuff: */
#ops, #tree, #path, #wfp, /* main tabs and navigators (tree/breadcrumbs) */
#u2conf tr:first-child>td[rowspan]:not(#u2btn_cw), /* most of the config options */
#srch_dz, #srch_zd, /* the filesearch dropzone */
#u2cards, #u2etaw /* and the upload progress tabs */
{display: none !important} /* do it! */
/* add some margins because now it's weird */
.opview {margin-top: 2.5em}
#op_up2k {margin-top: 6em}
/* and embiggen the upload button */
#u2conf #u2btn, #u2btn {padding:1.5em 0}
/* adjust the button area a bit */
#u2conf.w, #u2conf.ww {width: 35em !important; margin: 5em auto}
/* a */
#op_up2k {min-height: 0}
</style>
<a href="#" onclick="this.parentNode.innerHTML='';">show advanced options</a>

View File

@@ -1,59 +0,0 @@
/*
makes the up2k ui REALLY minimal by hiding a bunch of stuff
almost the same as minimal-up2k.html except this one...:
-- applies to every write-only folder when used with --js-browser
-- only applies if javascript is enabled
-- doesn't hide the total upload ETA display
-- looks slightly better
*/
var u2min = `
<style>
#ops, #path, #tree, #files, #wfp,
#u2conf td.c+.c, #u2cards, #srch_dz, #srch_zd {
display: none !important;
}
#u2conf {margin:5em auto 0 auto !important}
#u2conf.ww {width:70em}
#u2conf.w {width:50em}
#u2conf.w .c,
#u2conf.w #u2btn_cw {text-align:left}
#u2conf.w #u2btn_cw {width:70%}
#u2etaw {margin:3em auto}
#u2etaw.w {
text-align: center;
margin: -3.5em auto 5em auto;
}
#u2etaw.w #u2etas {margin-right:-37em}
#u2etaw.w #u2etas.o {margin-top:-2.2em}
#u2etaw.ww {margin:-1em auto}
#u2etaw.ww #u2etas {padding-left:4em}
#u2etas {
background: none !important;
border: none !important;
}
#wrap {margin-left:2em !important}
.logue {
border: none !important;
margin: 2em auto !important;
}
.logue:before {content:'' !important}
</style>
<a href="#" onclick="this.parentNode.innerHTML='';">show advanced options</a>
`;
if (!has(perms, 'read')) {
var e2 = mknod('div');
e2.innerHTML = u2min;
ebi('wrap').insertBefore(e2, QS('#wfp'));
}

View File

@@ -1,297 +0,0 @@
// way more specific example --
// assumes all files dropped into the uploader have a youtube-id somewhere in the filename,
// locates the youtube-ids and passes them to an API which returns a list of IDs which should be uploaded
//
// also tries to find the youtube-id in the embedded metadata
//
// assumes copyparty is behind nginx as /ytq is a standalone service which must be rproxied in place
function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
var passthru = up2k.uc.fsearch;
if (passthru)
return hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
a_up2k_namefilter(good_files, nil_files, bad_files, hooks).then(() => { });
}
// ebi('op_up2k').appendChild(mknod('input','unick'));
function bstrpos(buf, ptn) {
var ofs = 0,
ch0 = ptn[0],
sz = buf.byteLength;
while (true) {
ofs = buf.indexOf(ch0, ofs);
if (ofs < 0 || ofs >= sz)
return -1;
for (var a = 1; a < ptn.length; a++)
if (buf[ofs + a] !== ptn[a])
break;
if (a === ptn.length)
return ofs;
++ofs;
}
}
async function a_up2k_namefilter(good_files, nil_files, bad_files, hooks) {
var t0 = Date.now(),
yt_ids = new Set(),
textdec = new TextDecoder('latin1'),
md_ptn = new TextEncoder().encode('youtube.com/watch?v='),
file_ids = [], // all IDs found for each good_files
md_only = [], // `${id} ${fn}` where ID was only found in metadata
mofs = 0,
mnchk = 0,
mfile = '',
myid = localStorage.getItem('ytid_t0');
if (!myid)
localStorage.setItem('ytid_t0', myid = Date.now());
for (var a = 0; a < good_files.length; a++) {
var [fobj, name] = good_files[a],
cname = name, // will clobber
sz = fobj.size,
ids = [],
fn_ids = [],
md_ids = [],
id_ok = false,
m;
// all IDs found in this file
file_ids.push(ids);
// look for ID in filename; reduce the
// metadata-scan intensity if the id looks safe
m = /[\[(-]([\w-]{11})[\])]?\.(?:mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name);
id_ok = !!m;
while (true) {
// fuzzy catch-all;
// some ytdl fork did %(title)-%(id).%(ext) ...
m = /(?:^|[^\w])([\w-]{11})(?:$|[^\w-])/.exec(cname);
if (!m)
break;
cname = cname.replace(m[1], '');
yt_ids.add(m[1]);
fn_ids.unshift(m[1]);
}
// look for IDs in video metadata,
if (/\.(mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name)) {
toast.show('inf r', 0, `analyzing file ${a + 1} / ${good_files.length} :\n${name}\n\nhave analysed ${++mnchk} files in ${(Date.now() - t0) / 1000} seconds, ${humantime((good_files.length - (a + 1)) * (((Date.now() - t0) / 1000) / mnchk))} remaining,\n\nbiggest offset so far is ${mofs}, in this file:\n\n${mfile}`);
// check first and last 128 MiB;
// pWxOroN5WCo.mkv @ 6edb98 (6.92M)
// Nf-nN1wF5Xo.mp4 @ 4a98034 (74.6M)
var chunksz = 1024 * 1024 * 2, // byte
aspan = id_ok ? 128 : 512; // MiB
aspan = parseInt(Math.min(sz / 2, aspan * 1024 * 1024) / chunksz) * chunksz;
if (!aspan)
aspan = Math.min(sz, chunksz);
for (var side = 0; side < 2; side++) {
var ofs = side ? Math.max(0, sz - aspan) : 0,
nchunks = aspan / chunksz;
for (var chunk = 0; chunk < nchunks; chunk++) {
var bchunk = await fobj.slice(ofs, ofs + chunksz + 16).arrayBuffer(),
uchunk = new Uint8Array(bchunk, 0, bchunk.byteLength),
bofs = bstrpos(uchunk, md_ptn),
absofs = Math.min(ofs + bofs, (sz - ofs) + bofs),
txt = bofs < 0 ? '' : textdec.decode(uchunk.subarray(bofs)),
m;
//console.log(`side ${ side }, chunk ${ chunk }, ofs ${ ofs }, bchunk ${ bchunk.byteLength }, txt ${ txt.length }`);
while (true) {
// mkv/webm have [a-z] immediately after url
m = /(youtube\.com\/watch\?v=[\w-]{11})/.exec(txt);
if (!m)
break;
txt = txt.replace(m[1], '');
m = m[1].slice(-11);
console.log(`found ${m} @${bofs}, ${name} `);
yt_ids.add(m);
if (!has(fn_ids, m) && !has(md_ids, m)) {
md_ids.push(m);
md_only.push(`${m} ${name}`);
}
else
// id appears several times; make it preferred
md_ids.unshift(m);
// bail after next iteration
chunk = nchunks - 1;
side = 9;
if (mofs < absofs) {
mofs = absofs;
mfile = name;
}
}
ofs += chunksz;
if (ofs >= sz)
break;
}
}
}
for (var yi of md_ids)
ids.push(yi);
for (var yi of fn_ids)
if (!has(ids, yi))
ids.push(yi);
}
if (md_only.length)
console.log('recovered the following youtube-IDs by inspecting metadata:\n\n' + md_only.join('\n'));
else if (yt_ids.size)
console.log('did not discover any additional youtube-IDs by inspecting metadata; all the IDs also existed in the filenames');
else
console.log('failed to find any youtube-IDs at all, sorry');
if (false) {
var msg = `finished analysing ${mnchk} files in ${(Date.now() - t0) / 1000} seconds,\n\nbiggest offset was ${mofs} in this file:\n\n${mfile}`,
mfun = function () { toast.ok(0, msg); };
mfun();
setTimeout(mfun, 200);
return hooks[0]([], [], [], hooks.slice(1));
}
var el = ebi('unick'), unick = el ? el.value : '';
if (unick) {
console.log(`sending uploader nickname [${unick}]`);
fetch(document.location, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' },
body: 'msg=' + encodeURIComponent(unick)
});
}
toast.inf(5, `running query for ${yt_ids.size} youtube-IDs...`);
var xhr = new XHR();
xhr.open('POST', '/ytq', true);
xhr.setRequestHeader('Content-Type', 'text/plain');
xhr.onload = xhr.onerror = function () {
if (this.status != 200)
return toast.err(0, `sorry, database query failed ;_;\n\nplease let us know so we can look at it, thx!!\n\nerror ${this.status}: ${(this.response && this.response.err) || this.responseText}`);
process_id_list(this.responseText);
};
xhr.send(Array.from(yt_ids).join('\n'));
function process_id_list(txt) {
var wanted_ids = new Set(txt.trim().split('\n')),
name_id = {},
wanted_names = new Set(), // basenames with a wanted ID -- not including relpath
wanted_names_scoped = {}, // basenames with a wanted ID -> list of dirs to search under
wanted_files = new Set(); // filedrops
for (var a = 0; a < good_files.length; a++) {
var name = good_files[a][1];
for (var b = 0; b < file_ids[a].length; b++)
if (wanted_ids.has(file_ids[a][b])) {
// let the next stage handle this to prevent dupes
//wanted_files.add(good_files[a]);
var m = /(.*)\.(mp4|webm|mkv|flv|opus|ogg|mp3|m4a|aac)$/i.exec(name);
if (!m)
continue;
var [rd, fn] = vsplit(m[1]);
if (fn in wanted_names_scoped)
wanted_names_scoped[fn].push(rd);
else
wanted_names_scoped[fn] = [rd];
wanted_names.add(fn);
name_id[m[1]] = file_ids[a][b];
break;
}
}
// add all files with the same basename as each explicitly wanted file
// (infojson/chatlog/etc when ID was discovered from metadata)
for (var a = 0; a < good_files.length; a++) {
var [rd, name] = vsplit(good_files[a][1]);
for (var b = 0; b < 3; b++) {
name = name.replace(/\.[^\.]+$/, '');
if (!wanted_names.has(name))
continue;
var vid_fp = false;
for (var c of wanted_names_scoped[name])
if (rd.startsWith(c))
vid_fp = c + name;
if (!vid_fp)
continue;
var subdir = name_id[vid_fp];
subdir = `v${subdir.slice(0, 1)}/${subdir}-${myid}`;
var newpath = subdir + '/' + good_files[a][1].split(/\//g).pop();
// check if this file is a dupe
for (var c of good_files)
if (c[1] == newpath)
newpath = null;
if (!newpath)
break;
good_files[a][1] = newpath;
wanted_files.add(good_files[a]);
break;
}
}
function upload_filtered() {
if (!wanted_files.size)
return modal.alert('Good news -- turns out we already have all those.\n\nBut thank you for checking in!');
hooks[0](Array.from(wanted_files), nil_files, bad_files, hooks.slice(1));
}
function upload_all() {
hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
}
var n_skip = good_files.length - wanted_files.size,
msg = `you added ${good_files.length} files; ${good_files.length == n_skip ? 'all' : n_skip} of them were skipped --\neither because we already have them,\nor because there is no youtube-ID in your filenames.\n\n<code>OK</code> / <code>Enter</code> = continue uploading just the ${wanted_files.size} files we definitely need\n\n<code>Cancel</code> / <code>ESC</code> = override the filter; upload ALL the files you added`;
if (!n_skip)
upload_filtered();
else
modal.confirm(msg, upload_filtered, upload_all);
};
}
up2k_hooks.push(function () {
up2k.gotallfiles.unshift(up2k_namefilter);
});
// persist/restore nickname field if present
setInterval(function () {
var o = ebi('unick');
if (!o || document.activeElement == o)
return;
o.oninput = function () {
localStorage.setItem('unick', o.value);
};
o.value = localStorage.getItem('unick') || '';
}, 1000);

View File

@@ -1,45 +0,0 @@
// hooks into up2k
function up2k_namefilter(good_files, nil_files, bad_files, hooks) {
// is called when stuff is dropped into the browser,
// after iterating through the directory tree and discovering all files,
// before the upload confirmation dialogue is shown
// good_files will successfully upload
// nil_files are empty files and will show an alert in the final hook
// bad_files are unreadable and cannot be uploaded
var file_lists = [good_files, nil_files, bad_files];
// build a list of filenames
var filenames = [];
for (var lst of file_lists)
for (var ent of lst)
filenames.push(ent[1]);
toast.inf(5, "running database query...");
// simulate delay while passing the list to some api for checking
setTimeout(function () {
// only keep webm files as an example
var new_lists = [];
for (var lst of file_lists) {
var keep = [];
new_lists.push(keep);
for (var ent of lst)
if (/\.webm$/.test(ent[1]))
keep.push(ent);
}
// finally, call the next hook in the chain
[good_files, nil_files, bad_files] = new_lists;
hooks[0](good_files, nil_files, bad_files, hooks.slice(1));
}, 1000);
}
// register
up2k_hooks.push(function () {
up2k.gotallfiles.unshift(up2k_namefilter);
});

View File

@@ -1,31 +0,0 @@
#!/bin/sh
#
# PROVIDE: copyparty
# REQUIRE: networking
# KEYWORD:
. /etc/rc.subr
name="copyparty"
rcvar="copyparty_enable"
copyparty_user="copyparty"
copyparty_args="-e2dsa -v /storage:/storage:r" # change as you see fit
copyparty_command="/usr/local/bin/python3.8 /usr/local/copyparty/copyparty-sfx.py ${copyparty_args}"
pidfile="/var/run/copyparty/${name}.pid"
command="/usr/sbin/daemon"
command_args="-P ${pidfile} -r -f ${copyparty_command}"
stop_postcmd="copyparty_shutdown"
copyparty_shutdown()
{
if [ -e "${pidfile}" ]; then
echo "Stopping supervising daemon."
kill -s TERM `cat ${pidfile}`
fi
}
load_rc_config $name
: ${copyparty_enable:=no}
run_rc_command "$1"

View File

@@ -1,23 +0,0 @@
# systemd service which generates a new TLS certificate on each boot,
# that way the one-year expiry time won't cause any issues --
# just have everyone trust the ca.pem once every 10 years
#
# assumptions/placeholder values:
# * this script and copyparty runs as user "cpp"
# * copyparty repo is at ~cpp/dev/copyparty
# * CA is named partylan
# * server IPs = 10.1.2.3 and 192.168.123.1
# * server hostname = party.lan
[Unit]
Description=copyparty certificate generator
Before=copyparty.service
[Service]
User=cpp
Type=oneshot
SyslogIdentifier=cpp-cert
ExecStart=/bin/bash -c 'cd ~/dev/copyparty/contrib && ./cfssl.sh partylan 10.1.2.3,192.168.123.1,party.lan y'
[Install]
WantedBy=multi-user.target

View File

@@ -2,60 +2,18 @@
# and share '/mnt' with anonymous read+write
#
# installation:
# cp -pv copyparty.service /etc/systemd/system
# restorecon -vr /etc/systemd/system/copyparty.service
# firewall-cmd --permanent --add-port={80,443,3923}/tcp # --zone=libvirt
# firewall-cmd --reload
# systemctl daemon-reload && systemctl enable --now copyparty
# cp -pv copyparty.service /etc/systemd/system && systemctl enable --now copyparty
#
# you may want to:
# change "User=cpp" and "/home/cpp/" to another user
# remove the nft lines to only listen on port 3923
# and in the ExecStart= line:
# change '/usr/bin/python3' to another interpreter
# change '/mnt::rw' to another location or permission-set
# add '-q' to disable logging on busy servers
# add '-i 127.0.0.1' to only allow local connections
# add '-e2dsa' to enable filesystem scanning + indexing
# add '-e2ts' to enable metadata indexing
#
# with `Type=notify`, copyparty will signal systemd when it is ready to
# accept connections; correctly delaying units depending on copyparty.
# But note that journalctl will get the timestamps wrong due to
# python disabling line-buffering, so messages are out-of-order:
# https://user-images.githubusercontent.com/241032/126040249-cb535cc7-c599-4931-a796-a5d9af691bad.png
#
# unless you add -q to disable logging, you may want to remove the
# following line to allow buffering (slightly better performance):
# Environment=PYTHONUNBUFFERED=x
#
# keep ExecStartPre before ExecStart, at least on rhel8
# change '/usr/bin/python' to another interpreter
# change '/mnt::a' to another location or permission-set
[Unit]
Description=copyparty file server
[Service]
Type=notify
SyslogIdentifier=copyparty
Environment=PYTHONUNBUFFERED=x
ExecReload=/bin/kill -s USR1 $MAINPID
# user to run as + where the TLS certificate is (if any)
User=cpp
Environment=XDG_CONFIG_HOME=/home/cpp/.config
# setup forwarding from ports 80 and 443 to port 3923
ExecStartPre=+/bin/bash -c 'nft -n -a list table nat | awk "/ to :3923 /{print\$NF}" | xargs -rL1 nft delete rule nat prerouting handle; true'
ExecStartPre=+nft add table ip nat
ExecStartPre=+nft -- add chain ip nat prerouting { type nat hook prerouting priority -100 \; }
ExecStartPre=+nft add rule ip nat prerouting tcp dport 80 redirect to :3923
ExecStartPre=+nft add rule ip nat prerouting tcp dport 443 redirect to :3923
# stop systemd-tmpfiles-clean.timer from deleting copyparty while it's running
ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
# copyparty settings
ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -e2d -v /mnt::rw
ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
[Install]
WantedBy=multi-user.target

View File

@@ -1,27 +0,0 @@
# this will start `/usr/local/bin/copyparty-sfx.py`
# in a chroot, preventing accidental access elsewhere
# and share '/mnt' with anonymous read+write
#
# installation:
# 1) put copyparty-sfx.py and prisonparty.sh in /usr/local/bin
# 2) cp -pv prisonparty.service /etc/systemd/system && systemctl enable --now prisonparty
#
# you may want to:
# change '/mnt::rw' to another location or permission-set
# (remember to change the '/mnt' chroot arg too)
#
# enable line-buffering for realtime logging (slight performance cost):
# inside the [Service] block, add the following line:
# Environment=PYTHONUNBUFFERED=x
[Unit]
Description=copyparty file server
[Service]
SyslogIdentifier=prisonparty
WorkingDirectory=/usr/local/bin
ExecStart=/bin/bash /usr/local/bin/prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt -- \
/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::rw
[Install]
WantedBy=multi-user.target

View File

@@ -1,51 +0,0 @@
@echo off
rem removes the 47.6 MiB filesize limit when downloading from webdav
rem + optionally allows/enables password-auth over plaintext http
rem + optionally helps disable wpad, removing the 10sec latency
setlocal enabledelayedexpansion
net session >nul 2>&1
if %errorlevel% neq 0 (
echo sorry, you must run this as administrator
pause
exit /b
)
reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\WebClient\Parameters /v FileSizeLimitInBytes /t REG_DWORD /d 0xffffffff /f
reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\WebClient\Parameters /v FsCtlRequestTimeoutInSec /t REG_DWORD /d 0xffffffff /f
echo(
echo OK;
echo allow webdav basic-auth over plaintext http?
echo Y: login works, but the password will be visible in wireshark etc
echo N: login will NOT work unless you use https and valid certificates
set c=.
set /p "c=(Y/N): "
echo(
if /i not "!c!"=="y" goto :g1
reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\WebClient\Parameters /v BasicAuthLevel /t REG_DWORD /d 0x2 /f
rem default is 1 (require tls)
:g1
echo(
echo OK;
echo do you want to disable wpad?
echo can give a HUGE speed boost depending on network settings
set c=.
set /p "c=(Y/N): "
echo(
if /i not "!c!"=="y" goto :g2
echo(
echo i'm about to open the [Connections] tab in [Internet Properties] for you;
echo please click [LAN settings] and disable [Automatically detect settings]
echo(
pause
control inetcpl.cpl,,4
:g2
net stop webclient
net start webclient
echo(
echo OK; all done
pause

View File

@@ -1,51 +1,50 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import os
import platform
import sys
import time
import sys
import os
try:
from typing import TYPE_CHECKING
except:
TYPE_CHECKING = False
if True:
from typing import Any, Callable
PY2 = sys.version_info < (3,)
if not PY2:
unicode: Callable[[Any], str] = str
else:
PY2 = sys.version_info[0] == 2
if PY2:
sys.dont_write_bytecode = True
unicode = unicode # noqa: F821 # pylint: disable=undefined-variable,self-assigning-variable
WINDOWS: Any = (
[int(x) for x in platform.version().split(".")]
if platform.system() == "Windows"
else False
)
WINDOWS = False
if platform.system() == "Windows":
WINDOWS = [int(x) for x in platform.version().split(".")]
VT100 = not WINDOWS or WINDOWS >= [10, 0, 14393]
# introduced in anniversary update
ANYWIN = WINDOWS or sys.platform in ["msys", "cygwin"]
ANYWIN = WINDOWS or sys.platform in ["msys"]
MACOS = platform.system() == "Darwin"
try:
CORES = len(os.sched_getaffinity(0))
except:
CORES = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 2
class EnvParams(object):
def __init__(self) -> None:
def __init__(self):
self.t0 = time.time()
self.mod = ""
self.cfg = ""
self.ox = getattr(sys, "oxidized", None)
self.mod = os.path.dirname(os.path.realpath(__file__))
if self.mod.endswith("__init__"):
self.mod = os.path.dirname(self.mod)
if sys.platform == "win32":
self.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
elif sys.platform == "darwin":
self.cfg = os.path.expanduser("~/Library/Preferences/copyparty")
else:
self.cfg = os.path.normpath(
os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config"))
+ "/copyparty"
)
self.cfg = self.cfg.replace("\\", "/")
try:
os.makedirs(self.cfg)
except:
if not os.path.isdir(self.cfg):
raise
E = EnvParams()

1195
copyparty/__main__.py Executable file → Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -1,8 +1,8 @@
# coding: utf-8
VERSION = (1, 6, 3)
CODENAME = "cors k"
BUILD_DT = (2023, 1, 31)
VERSION = (0, 11, 29)
CODENAME = "the grid"
BUILD_DT = (2021, 6, 30)
S_VERSION = ".".join(map(str, VERSION))
S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

File diff suppressed because it is too large Load Diff

View File

@@ -1,81 +0,0 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import os
from ..util import SYMTIME, fsdec, fsenc
from . import path as path
if True: # pylint: disable=using-constant-test
from typing import Any, Optional
_ = (path,)
__all__ = ["path"]
# grep -hRiE '(^|[^a-zA-Z_\.-])os\.' . | gsed -r 's/ /\n/g;s/\(/(\n/g' | grep -hRiE '(^|[^a-zA-Z_\.-])os\.' | sort | uniq -c
# printf 'os\.(%s)' "$(grep ^def bos/__init__.py | gsed -r 's/^def //;s/\(.*//' | tr '\n' '|' | gsed -r 's/.$//')"
def chmod(p: str, mode: int) -> None:
return os.chmod(fsenc(p), mode)
def listdir(p: str = ".") -> list[str]:
return [fsdec(x) for x in os.listdir(fsenc(p))]
def makedirs(name: str, mode: int = 0o755, exist_ok: bool = True) -> bool:
bname = fsenc(name)
try:
os.makedirs(bname, mode)
return True
except:
if not exist_ok or not os.path.isdir(bname):
raise
return False
def mkdir(p: str, mode: int = 0o755) -> None:
return os.mkdir(fsenc(p), mode)
def open(p: str, *a, **ka) -> int:
return os.open(fsenc(p), *a, **ka)
def rename(src: str, dst: str) -> None:
return os.rename(fsenc(src), fsenc(dst))
def replace(src: str, dst: str) -> None:
return os.replace(fsenc(src), fsenc(dst))
def rmdir(p: str) -> None:
return os.rmdir(fsenc(p))
def stat(p: str) -> os.stat_result:
return os.stat(fsenc(p))
def unlink(p: str) -> None:
return os.unlink(fsenc(p))
def utime(
p: str, times: Optional[tuple[float, float]] = None, follow_symlinks: bool = True
) -> None:
if SYMTIME:
return os.utime(fsenc(p), times, follow_symlinks=follow_symlinks)
else:
return os.utime(fsenc(p), times)
if hasattr(os, "lstat"):
def lstat(p: str) -> os.stat_result:
return os.lstat(fsenc(p))
else:
lstat = stat

View File

@@ -1,45 +0,0 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import os
from ..util import SYMTIME, fsdec, fsenc
def abspath(p: str) -> str:
return fsdec(os.path.abspath(fsenc(p)))
def exists(p: str) -> bool:
return os.path.exists(fsenc(p))
def getmtime(p: str, follow_symlinks: bool = True) -> float:
if not follow_symlinks and SYMTIME:
return os.lstat(fsenc(p)).st_mtime
else:
return os.path.getmtime(fsenc(p))
def getsize(p: str) -> int:
return os.path.getsize(fsenc(p))
def isfile(p: str) -> bool:
return os.path.isfile(fsenc(p))
def isdir(p: str) -> bool:
return os.path.isdir(fsenc(p))
def islink(p: str) -> bool:
return os.path.islink(fsenc(p))
def lexists(p: str) -> bool:
return os.path.lexists(fsenc(p))
def realpath(p: str) -> str:
return fsdec(os.path.realpath(fsenc(p)))

View File

@@ -1,64 +1,70 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import threading
import time
import traceback
import threading
import queue
from .__init__ import CORES, TYPE_CHECKING
from .broker_mpw import MpWorker
from .__init__ import PY2, WINDOWS, VT100
from .broker_util import try_exec
from .util import Daemon, mp
if TYPE_CHECKING:
from .svchub import SvcHub
if True: # pylint: disable=using-constant-test
from typing import Any
from .broker_mpw import MpWorker
from .util import mp
class MProcess(mp.Process):
def __init__(
self,
q_pend: queue.Queue[tuple[int, str, list[Any]]],
q_yield: queue.Queue[tuple[int, str, list[Any]]],
target: Any,
args: Any,
) -> None:
super(MProcess, self).__init__(target=target, args=args)
self.q_pend = q_pend
self.q_yield = q_yield
if PY2 and not WINDOWS:
from multiprocessing.reduction import ForkingPickler
from StringIO import StringIO as MemesIO # pylint: disable=import-error
class BrokerMp(object):
"""external api; manages MpWorkers"""
def __init__(self, hub: "SvcHub") -> None:
def __init__(self, hub):
self.hub = hub
self.log = hub.log
self.args = hub.args
self.procs = []
self.retpend = {}
self.retpend_mutex = threading.Lock()
self.mutex = threading.Lock()
self.num_workers = self.args.j or CORES
self.log("broker", "booting {} subprocesses".format(self.num_workers))
for n in range(1, self.num_workers + 1):
q_pend: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(1)
q_yield: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(64)
cores = self.args.j
if not cores:
cores = mp.cpu_count()
self.log("broker", "booting {} subprocesses".format(cores))
for n in range(cores):
q_pend = mp.Queue(1)
q_yield = mp.Queue(64)
proc = mp.Process(target=MpWorker, args=(q_pend, q_yield, self.args, n))
proc.q_pend = q_pend
proc.q_yield = q_yield
proc.nid = n
proc.clients = {}
proc.workload = 0
thr = threading.Thread(
target=self.collector, args=(proc,), name="mp-collector"
)
thr.daemon = True
thr.start()
proc = MProcess(q_pend, q_yield, MpWorker, (q_pend, q_yield, self.args, n))
Daemon(self.collector, "mp-sink-{}".format(n), (proc,))
self.procs.append(proc)
proc.start()
def shutdown(self) -> None:
if not self.args.q:
thr = threading.Thread(
target=self.debug_load_balancer, name="mp-dbg-loadbalancer"
)
thr.daemon = True
thr.start()
def shutdown(self):
self.log("broker", "shutting down")
for n, proc in enumerate(self.procs):
thr = threading.Thread(
target=proc.q_pend.put((0, "shutdown", [])),
target=proc.q_pend.put([0, "shutdown", []]),
name="mp-shutdown-{}-{}".format(n, len(self.procs)),
)
thr.start()
@@ -74,12 +80,7 @@ class BrokerMp(object):
procs.pop()
def reload(self) -> None:
self.log("broker", "reloading")
for _, proc in enumerate(self.procs):
proc.q_pend.put((0, "reload", []))
def collector(self, proc: MProcess) -> None:
def collector(self, proc):
"""receive message from hub in other process"""
while True:
msg = proc.q_yield.get()
@@ -88,41 +89,77 @@ class BrokerMp(object):
if dest == "log":
self.log(*args)
elif dest == "workload":
with self.mutex:
proc.workload = args[0]
elif dest == "httpdrop":
addr = args[0]
with self.mutex:
del proc.clients[addr]
if not proc.clients:
proc.workload = 0
self.hub.tcpsrv.num_clients.add(-1)
elif dest == "retq":
# response from previous ipc call
raise Exception("invalid broker_mp usage")
with self.retpend_mutex:
retq = self.retpend.pop(retq_id)
retq.put(args)
else:
# new ipc invoking managed service in hub
try:
obj = self.hub
for node in dest.split("."):
obj = getattr(obj, node)
obj = self.hub
for node in dest.split("."):
obj = getattr(obj, node)
# TODO will deadlock if dest performs another ipc
rv = try_exec(retq_id, obj, *args)
except:
rv = ["exception", "stack", traceback.format_exc()]
# TODO will deadlock if dest performs another ipc
rv = try_exec(retq_id, obj, *args)
if retq_id:
proc.q_pend.put((retq_id, "retq", rv))
proc.q_pend.put([retq_id, "retq", rv])
def say(self, dest: str, *args: Any) -> None:
def put(self, want_retval, dest, *args):
"""
send message to non-hub component in other process,
returns a Queue object which eventually contains the response if want_retval
(not-impl here since nothing uses it yet)
"""
if dest == "listen":
for p in self.procs:
p.q_pend.put((0, dest, [args[0], len(self.procs)]))
if dest == "httpconn":
sck, addr = args
sck2 = sck
if PY2:
buf = MemesIO()
ForkingPickler(buf).dump(sck)
sck2 = buf.getvalue()
elif dest == "set_netdevs":
for p in self.procs:
p.q_pend.put((0, dest, list(args)))
proc = sorted(self.procs, key=lambda x: x.workload)[0]
proc.q_pend.put([0, dest, [sck2, addr]])
elif dest == "cb_httpsrv_up":
self.hub.cb_httpsrv_up()
with self.mutex:
proc.clients[addr] = 50
proc.workload += 50
else:
raise Exception("what is " + str(dest))
def debug_load_balancer(self):
fmt = "\033[1m{}\033[0;36m{:4}\033[0m "
if not VT100:
fmt = "({}{:4})"
last = ""
while self.procs:
msg = ""
for proc in self.procs:
msg += fmt.format(len(proc.clients), proc.workload)
if msg != last:
last = msg
with self.hub.log_mutex:
print(msg)
time.sleep(0.1)

View File

@@ -1,84 +1,68 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
from copyparty.authsrv import AuthSrv
import argparse
import os
import signal
import sys
import time
import signal
import threading
import queue
from .__init__ import ANYWIN
from .authsrv import AuthSrv
from .broker_util import BrokerCli, ExceptionalQueue
from .__init__ import PY2, WINDOWS
from .broker_util import ExceptionalQueue
from .httpsrv import HttpSrv
from .util import FAKE_MP, Daemon, HMaccas
from .util import FAKE_MP
if True: # pylint: disable=using-constant-test
from types import FrameType
from typing import Any, Optional, Union
if PY2 and not WINDOWS:
import pickle # nosec
class MpWorker(BrokerCli):
class MpWorker(object):
"""one single mp instance"""
def __init__(
self,
q_pend: queue.Queue[tuple[int, str, list[Any]]],
q_yield: queue.Queue[tuple[int, str, list[Any]]],
args: argparse.Namespace,
n: int,
) -> None:
super(MpWorker, self).__init__()
def __init__(self, q_pend, q_yield, args, n):
self.q_pend = q_pend
self.q_yield = q_yield
self.args = args
self.n = n
self.log = self._log_disabled if args.q and not args.lo else self._log_enabled
self.retpend: dict[int, Any] = {}
self.retpend = {}
self.retpend_mutex = threading.Lock()
self.mutex = threading.Lock()
self.workload_thr_alive = False
# we inherited signal_handler from parent,
# replace it with something harmless
if not FAKE_MP:
sigs = [signal.SIGINT, signal.SIGTERM]
if not ANYWIN:
sigs.append(signal.SIGUSR1)
for sig in sigs:
signal.signal(sig, self.signal_handler)
signal.signal(signal.SIGINT, self.signal_handler)
# starting to look like a good idea
self.asrv = AuthSrv(args, None, False)
# instantiate all services here (TODO: inheritance?)
self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
self.httpsrv = HttpSrv(self, n)
self.httpsrv = HttpSrv(self, True)
self.httpsrv.disconnect_func = self.httpdrop
# on winxp and some other platforms,
# use thr.join() to block all signals
Daemon(self.main, "mpw-main").join()
thr = threading.Thread(target=self.main, name="mpw-main")
thr.daemon = True
thr.start()
thr.join()
def signal_handler(self, sig: Optional[int], frame: Optional[FrameType]) -> None:
def signal_handler(self, signal, frame):
# print('k')
pass
def _log_enabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
self.q_yield.put((0, "log", [src, msg, c]))
def log(self, src, msg, c=0):
self.q_yield.put([0, "log", [src, msg, c]])
def _log_disabled(self, src: str, msg: str, c: Union[int, str] = 0) -> None:
pass
def logw(self, msg: str, c: Union[int, str] = 0) -> None:
def logw(self, msg, c=0):
self.log("mp{}".format(self.n), msg, c)
def main(self) -> None:
def httpdrop(self, addr):
self.q_yield.put([0, "httpdrop", [addr]])
def main(self):
while True:
retq_id, dest, args = self.q_pend.get()
@@ -89,16 +73,24 @@ class MpWorker(BrokerCli):
sys.exit(0)
return
elif dest == "reload":
self.logw("mpw.asrv reloading")
self.asrv.reload()
self.logw("mpw.asrv reloaded")
elif dest == "httpconn":
sck, addr = args
if PY2:
sck = pickle.loads(sck) # nosec
elif dest == "listen":
self.httpsrv.listen(args[0], args[1])
if self.args.log_conn:
self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
elif dest == "set_netdevs":
self.httpsrv.set_netdevs(args[0])
self.httpsrv.accept(sck, addr)
with self.mutex:
if not self.workload_thr_alive:
self.workload_thr_alive = True
thr = threading.Thread(
target=self.thr_workload, name="mpw-workload"
)
thr.daemon = True
thr.start()
elif dest == "retq":
# response from previous ipc call
@@ -110,14 +102,28 @@ class MpWorker(BrokerCli):
else:
raise Exception("what is " + str(dest))
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
retq = ExceptionalQueue(1)
retq_id = id(retq)
with self.retpend_mutex:
self.retpend[retq_id] = retq
def put(self, want_retval, dest, *args):
if want_retval:
retq = ExceptionalQueue(1)
retq_id = id(retq)
with self.retpend_mutex:
self.retpend[retq_id] = retq
else:
retq = None
retq_id = 0
self.q_yield.put((retq_id, dest, list(args)))
self.q_yield.put([retq_id, dest, args])
return retq
def say(self, dest: str, *args: Any) -> None:
self.q_yield.put((0, dest, list(args)))
def thr_workload(self):
"""announce workloads to MpSrv (the mp controller / loadbalancer)"""
# avoid locking in extract_filedata by tracking difference here
while True:
time.sleep(0.2)
with self.mutex:
if self.httpsrv.num_clients() == 0:
# no clients rn, termiante thread
self.workload_thr_alive = False
return
self.q_yield.put([0, "workload", [self.httpsrv.workload]])

View File

@@ -1,73 +1,56 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import os
import threading
from .__init__ import TYPE_CHECKING
from .broker_util import BrokerCli, ExceptionalQueue, try_exec
from .authsrv import AuthSrv
from .httpsrv import HttpSrv
from .util import HMaccas
if TYPE_CHECKING:
from .svchub import SvcHub
if True: # pylint: disable=using-constant-test
from typing import Any
from .broker_util import ExceptionalQueue, try_exec
class BrokerThr(BrokerCli):
class BrokerThr(object):
"""external api; behaves like BrokerMP but using plain threads"""
def __init__(self, hub: "SvcHub") -> None:
super(BrokerThr, self).__init__()
def __init__(self, hub):
self.hub = hub
self.log = hub.log
self.args = hub.args
self.asrv = hub.asrv
self.mutex = threading.Lock()
self.num_workers = 1
# instantiate all services here (TODO: inheritance?)
self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
self.httpsrv = HttpSrv(self, None)
self.reload = self.noop
self.httpsrv = HttpSrv(self)
self.httpsrv.disconnect_func = self.httpdrop
def shutdown(self) -> None:
def shutdown(self):
# self.log("broker", "shutting down")
self.httpsrv.shutdown()
def noop(self) -> None:
pass
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
def put(self, want_retval, dest, *args):
if dest == "httpconn":
sck, addr = args
if self.args.log_conn:
self.log("%s %s" % addr, "|%sC-qpop" % ("-" * 4,), c="1;30")
# new ipc invoking managed service in hub
obj = self.hub
for node in dest.split("."):
obj = getattr(obj, node)
self.httpsrv.accept(sck, addr)
rv = try_exec(True, obj, *args)
else:
# new ipc invoking managed service in hub
obj = self.hub
for node in dest.split("."):
obj = getattr(obj, node)
# pretend we're broker_mp
retq = ExceptionalQueue(1)
retq.put(rv)
return retq
# TODO will deadlock if dest performs another ipc
rv = try_exec(want_retval, obj, *args)
if not want_retval:
return
def say(self, dest: str, *args: Any) -> None:
if dest == "listen":
self.httpsrv.listen(args[0], 1)
return
# pretend we're broker_mp
retq = ExceptionalQueue(1)
retq.put(rv)
return retq
if dest == "set_netdevs":
self.httpsrv.set_netdevs(args[0])
return
# new ipc invoking managed service in hub
obj = self.hub
for node in dest.split("."):
obj = getattr(obj, node)
try_exec(False, obj, *args)
def httpdrop(self, addr):
self.hub.tcpsrv.num_clients.add(-1)

View File

@@ -1,28 +1,17 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import argparse
import traceback
from queue import Queue
from .__init__ import TYPE_CHECKING
from .authsrv import AuthSrv
from .util import HMaccas, Pebkac
if True: # pylint: disable=using-constant-test
from typing import Any, Optional, Union
from .util import RootLogger
if TYPE_CHECKING:
from .httpsrv import HttpSrv
from .util import Pebkac, Queue
class ExceptionalQueue(Queue, object):
def get(self, block: bool = True, timeout: Optional[float] = None) -> Any:
def get(self, block=True, timeout=None):
rv = super(ExceptionalQueue, self).get(block, timeout)
# TODO: how expensive is this?
if isinstance(rv, list):
if rv[0] == "exception":
if rv[1] == "pebkac":
@@ -33,29 +22,7 @@ class ExceptionalQueue(Queue, object):
return rv
class BrokerCli(object):
"""
helps mypy understand httpsrv.broker but still fails a few levels deeper,
for example resolving httpconn.* in httpcli -- see lines tagged #mypy404
"""
log: "RootLogger"
args: argparse.Namespace
asrv: AuthSrv
httpsrv: "HttpSrv"
iphash: HMaccas
def __init__(self) -> None:
pass
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
return ExceptionalQueue(1)
def say(self, dest: str, *args: Any) -> None:
pass
def try_exec(want_retval: Union[bool, int], func: Any, *args: list[Any]) -> Any:
def try_exec(want_retval, func, *args):
try:
return func(*args)

View File

@@ -1,72 +0,0 @@
import importlib
import sys
import xml.etree.ElementTree as ET
from .__init__ import PY2
if True: # pylint: disable=using-constant-test
from typing import Any, Optional
def get_ET() -> ET.XMLParser:
pn = "xml.etree.ElementTree"
cn = "_elementtree"
cmod = sys.modules.pop(cn, None)
if not cmod:
return ET.XMLParser # type: ignore
pmod = sys.modules.pop(pn)
sys.modules[cn] = None # type: ignore
ret = importlib.import_module(pn)
for name, mod in ((pn, pmod), (cn, cmod)):
if mod:
sys.modules[name] = mod
else:
sys.modules.pop(name, None)
sys.modules["xml.etree"].ElementTree = pmod # type: ignore
ret.ParseError = ET.ParseError # type: ignore
return ret.XMLParser # type: ignore
XMLParser: ET.XMLParser = get_ET()
class DXMLParser(XMLParser): # type: ignore
def __init__(self) -> None:
tb = ET.TreeBuilder()
super(DXMLParser, self).__init__(target=tb)
p = self._parser if PY2 else self.parser
p.StartDoctypeDeclHandler = self.nope
p.EntityDeclHandler = self.nope
p.UnparsedEntityDeclHandler = self.nope
p.ExternalEntityRefHandler = self.nope
def nope(self, *a: Any, **ka: Any) -> None:
raise BadXML("{}, {}".format(a, ka))
class BadXML(Exception):
pass
def parse_xml(txt: str) -> ET.Element:
parser = DXMLParser()
parser.feed(txt)
return parser.close() # type: ignore
def mktnod(name: str, text: str) -> ET.Element:
el = ET.Element(name)
el.text = text
return el
def mkenod(name: str, sub_el: Optional[ET.Element] = None) -> ET.Element:
el = ET.Element(name)
if sub_el is not None:
el.append(sub_el)
return el

View File

@@ -1,152 +0,0 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import os
import re
import time
from .__init__ import ANYWIN, MACOS
from .authsrv import AXS, VFS
from .bos import bos
from .util import chkcmd, min_ex
if True: # pylint: disable=using-constant-test
from typing import Optional, Union
from .util import RootLogger
class Fstab(object):
def __init__(self, log: "RootLogger"):
self.log_func = log
self.trusted = False
self.tab: Optional[VFS] = None
self.cache: dict[str, str] = {}
self.age = 0.0
def log(self, msg: str, c: Union[int, str] = 0) -> None:
self.log_func("fstab", msg, c)
def get(self, path: str) -> str:
if len(self.cache) > 9000:
self.age = time.time()
self.tab = None
self.cache = {}
fs = "ext4"
msg = "failed to determine filesystem at [{}]; assuming {}\n{}"
if ANYWIN:
fs = "vfat"
try:
path = self._winpath(path)
except:
self.log(msg.format(path, fs, min_ex()), 3)
return fs
path = path.lstrip("/")
try:
return self.cache[path]
except:
pass
try:
fs = self.get_w32(path) if ANYWIN else self.get_unix(path)
except:
self.log(msg.format(path, fs, min_ex()), 3)
fs = fs.lower()
self.cache[path] = fs
self.log("found {} at {}".format(fs, path))
return fs
def _winpath(self, path: str) -> str:
# try to combine volume-label + st_dev (vsn)
path = path.replace("/", "\\")
vid = path.split(":", 1)[0].strip("\\").split("\\", 1)[0]
try:
return "{}*{}".format(vid, bos.stat(path).st_dev)
except:
return vid
def build_fallback(self) -> None:
self.tab = VFS(self.log_func, "idk", "/", AXS(), {})
self.trusted = False
def build_tab(self) -> None:
self.log("building tab")
sptn = r"^.*? on (.*) type ([^ ]+) \(.*"
if MACOS:
sptn = r"^.*? on (.*) \(([^ ]+), .*"
ptn = re.compile(sptn)
so, _ = chkcmd(["mount"])
tab1: list[tuple[str, str]] = []
for ln in so.split("\n"):
m = ptn.match(ln)
if not m:
continue
zs1, zs2 = m.groups()
tab1.append((str(zs1), str(zs2)))
tab1.sort(key=lambda x: (len(x[0]), x[0]))
path1, fs1 = tab1[0]
tab = VFS(self.log_func, fs1, path1, AXS(), {})
for path, fs in tab1[1:]:
tab.add(fs, path.lstrip("/"))
self.tab = tab
def relabel(self, path: str, nval: str) -> None:
assert self.tab
self.cache = {}
if ANYWIN:
path = self._winpath(path)
path = path.lstrip("/")
ptn = re.compile(r"^[^\\/]*")
vn, rem = self.tab._find(path)
if not self.trusted:
# no mtab access; have to build as we go
if "/" in rem:
self.tab.add("idk", os.path.join(vn.vpath, rem.split("/")[0]))
if rem:
self.tab.add(nval, path)
else:
vn.realpath = nval
return
visit = [vn]
while visit:
vn = visit.pop()
vn.realpath = ptn.sub(nval, vn.realpath)
visit.extend(list(vn.nodes.values()))
def get_unix(self, path: str) -> str:
if not self.tab:
try:
self.build_tab()
self.trusted = True
except:
# prisonparty or other restrictive environment
self.log("failed to build tab:\n{}".format(min_ex()), 3)
self.build_fallback()
assert self.tab
ret = self.tab._find(path)[0]
if self.trusted or path == ret.vpath:
return ret.realpath.split("/")[0]
else:
return "idk"
def get_w32(self, path: str) -> str:
if not self.tab:
self.build_fallback()
assert self.tab
ret = self.tab._find(path)[0]
return ret.realpath

View File

@@ -1,463 +0,0 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import argparse
import logging
import os
import stat
import sys
import time
from pyftpdlib.authorizers import AuthenticationFailed, DummyAuthorizer
from pyftpdlib.filesystems import AbstractedFS, FilesystemError
from pyftpdlib.handlers import FTPHandler
from pyftpdlib.servers import FTPServer
from .__init__ import ANYWIN, PY2, TYPE_CHECKING, E
from .bos import bos
from .util import (
Daemon,
Pebkac,
exclude_dotfiles,
fsenc,
ipnorm,
relchk,
sanitize_fn,
vjoin,
)
try:
from pyftpdlib.ioloop import IOLoop
except ImportError:
p = os.path.join(E.mod, "vend")
print("loading asynchat from " + p)
sys.path.append(p)
from pyftpdlib.ioloop import IOLoop
if TYPE_CHECKING:
from .svchub import SvcHub
if True: # pylint: disable=using-constant-test
import typing
from typing import Any, Optional
class FtpAuth(DummyAuthorizer):
def __init__(self, hub: "SvcHub") -> None:
super(FtpAuth, self).__init__()
self.hub = hub
def validate_authentication(
self, username: str, password: str, handler: Any
) -> None:
handler.username = "{}:{}".format(username, password)
ip = handler.addr[0]
if ip.startswith("::ffff:"):
ip = ip[7:]
ip = ipnorm(ip)
bans = self.hub.bans
if ip in bans:
rt = bans[ip] - time.time()
if rt < 0:
logging.info("client unbanned")
del bans[ip]
else:
raise AuthenticationFailed("banned")
asrv = self.hub.asrv
if username == "anonymous":
uname = "*"
else:
uname = asrv.iacct.get(password, "") or asrv.iacct.get(username, "") or "*"
if not uname or not (asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)):
g = self.hub.gpwd
if g.lim:
bonk, ip = g.bonk(ip, handler.username)
if bonk:
logging.warning("client banned: invalid passwords")
bans[ip] = bonk
raise AuthenticationFailed("Authentication failed.")
handler.username = uname
def get_home_dir(self, username: str) -> str:
return "/"
def has_user(self, username: str) -> bool:
asrv = self.hub.asrv
return username in asrv.acct
def has_perm(self, username: str, perm: int, path: Optional[str] = None) -> bool:
return True # handled at filesystem layer
def get_perms(self, username: str) -> str:
return "elradfmwMT"
def get_msg_login(self, username: str) -> str:
return "sup {}".format(username)
def get_msg_quit(self, username: str) -> str:
return "cya"
class FtpFs(AbstractedFS):
def __init__(
self, root: str, cmd_channel: Any
) -> None: # pylint: disable=super-init-not-called
self.h = self.cmd_channel = cmd_channel # type: FTPHandler
self.hub: "SvcHub" = cmd_channel.hub
self.args = cmd_channel.args
self.uname = self.hub.asrv.iacct.get(cmd_channel.password, "*")
self.cwd = "/" # pyftpdlib convention of leading slash
self.root = "/var/lib/empty"
self.can_read = self.can_write = self.can_move = False
self.can_delete = self.can_get = self.can_upget = False
self.listdirinfo = self.listdir
self.chdir(".")
def v2a(
self,
vpath: str,
r: bool = False,
w: bool = False,
m: bool = False,
d: bool = False,
) -> str:
try:
vpath = vpath.replace("\\", "/").lstrip("/")
rd, fn = os.path.split(vpath)
if ANYWIN and not relchk(rd):
raise FilesystemError("unsupported characters in filepath")
fn = sanitize_fn(fn or "", "", [".prologue.html", ".epilogue.html"])
vpath = vjoin(rd, fn)
vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, r, w, m, d)
if not vfs.realpath:
raise FilesystemError("no filesystem mounted at this path")
return os.path.join(vfs.realpath, rem)
except Pebkac as ex:
raise FilesystemError(str(ex))
def rv2a(
self,
vpath: str,
r: bool = False,
w: bool = False,
m: bool = False,
d: bool = False,
) -> str:
return self.v2a(os.path.join(self.cwd, vpath), r, w, m, d)
def ftp2fs(self, ftppath: str) -> str:
# return self.v2a(ftppath)
return ftppath # self.cwd must be vpath
def fs2ftp(self, fspath: str) -> str:
# raise NotImplementedError()
return fspath
def validpath(self, path: str) -> bool:
if "/.hist/" in path:
if "/up2k." in path or path.endswith("/dir.txt"):
raise FilesystemError("access to this file is forbidden")
return True
def open(self, filename: str, mode: str) -> typing.IO[Any]:
r = "r" in mode
w = "w" in mode or "a" in mode or "+" in mode
ap = self.rv2a(filename, r, w)
if w:
try:
st = bos.stat(ap)
td = time.time() - st.st_mtime
except:
td = 0
if td < -1 or td > self.args.ftp_wt:
raise FilesystemError("cannot open existing file for writing")
self.validpath(ap)
return open(fsenc(ap), mode)
def chdir(self, path: str) -> None:
nwd = join(self.cwd, path)
vfs, rem = self.hub.asrv.vfs.get(nwd, self.uname, False, False)
ap = vfs.canonical(rem)
if not bos.path.isdir(ap):
# returning 550 is library-default and suitable
raise FilesystemError("Failed to change directory")
self.cwd = nwd
(
self.can_read,
self.can_write,
self.can_move,
self.can_delete,
self.can_get,
self.can_upget,
) = self.hub.asrv.vfs.can_access(self.cwd.lstrip("/"), self.h.username)
def mkdir(self, path: str) -> None:
ap = self.rv2a(path, w=True)
bos.mkdir(ap)
def listdir(self, path: str) -> list[str]:
vpath = join(self.cwd, path).lstrip("/")
try:
vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, True, False)
fsroot, vfs_ls1, vfs_virt = vfs.ls(
rem,
self.uname,
not self.args.no_scandir,
[[True, False], [False, True]],
)
vfs_ls = [x[0] for x in vfs_ls1]
vfs_ls.extend(vfs_virt.keys())
if not self.args.ed:
vfs_ls = exclude_dotfiles(vfs_ls)
vfs_ls.sort()
return vfs_ls
except:
if vpath:
# display write-only folders as empty
return []
# return list of volumes
r = {x.split("/")[0]: 1 for x in self.hub.asrv.vfs.all_vols.keys()}
return list(sorted(list(r.keys())))
def rmdir(self, path: str) -> None:
ap = self.rv2a(path, d=True)
bos.rmdir(ap)
def remove(self, path: str) -> None:
if self.args.no_del:
raise FilesystemError("the delete feature is disabled in server config")
vp = join(self.cwd, path).lstrip("/")
try:
self.hub.up2k.handle_rm(self.uname, self.h.remote_ip, [vp], [])
except Exception as ex:
raise FilesystemError(str(ex))
def rename(self, src: str, dst: str) -> None:
if not self.can_move:
raise FilesystemError("not allowed for user " + self.h.username)
if self.args.no_mv:
t = "the rename/move feature is disabled in server config"
raise FilesystemError(t)
svp = join(self.cwd, src).lstrip("/")
dvp = join(self.cwd, dst).lstrip("/")
try:
self.hub.up2k.handle_mv(self.uname, svp, dvp)
except Exception as ex:
raise FilesystemError(str(ex))
def chmod(self, path: str, mode: str) -> None:
pass
def stat(self, path: str) -> os.stat_result:
try:
ap = self.rv2a(path, r=True)
return bos.stat(ap)
except:
ap = self.rv2a(path)
st = bos.stat(ap)
if not stat.S_ISDIR(st.st_mode):
raise
return st
def utime(self, path: str, timeval: float) -> None:
ap = self.rv2a(path, w=True)
return bos.utime(ap, (timeval, timeval))
def lstat(self, path: str) -> os.stat_result:
ap = self.rv2a(path)
return bos.stat(ap)
def isfile(self, path: str) -> bool:
try:
st = self.stat(path)
return stat.S_ISREG(st.st_mode)
except:
return False # expected for mojibake in ftp_SIZE()
def islink(self, path: str) -> bool:
ap = self.rv2a(path)
return bos.path.islink(ap)
def isdir(self, path: str) -> bool:
try:
st = self.stat(path)
return stat.S_ISDIR(st.st_mode)
except:
return True
def getsize(self, path: str) -> int:
ap = self.rv2a(path)
return bos.path.getsize(ap)
def getmtime(self, path: str) -> float:
ap = self.rv2a(path)
return bos.path.getmtime(ap)
def realpath(self, path: str) -> str:
return path
def lexists(self, path: str) -> bool:
ap = self.rv2a(path)
return bos.path.lexists(ap)
def get_user_by_uid(self, uid: int) -> str:
return "root"
def get_group_by_uid(self, gid: int) -> str:
return "root"
class FtpHandler(FTPHandler):
abstracted_fs = FtpFs
hub: "SvcHub"
args: argparse.Namespace
def __init__(self, conn: Any, server: Any, ioloop: Any = None) -> None:
self.hub: "SvcHub" = FtpHandler.hub
self.args: argparse.Namespace = FtpHandler.args
if PY2:
FTPHandler.__init__(self, conn, server, ioloop)
else:
super(FtpHandler, self).__init__(conn, server, ioloop)
# abspath->vpath mapping to resolve log_transfer paths
self.vfs_map: dict[str, str] = {}
# reduce non-debug logging
self.log_cmds_list = [x for x in self.log_cmds_list if x not in ("CWD", "XCWD")]
def ftp_STOR(self, file: str, mode: str = "w") -> Any:
# Optional[str]
vp = join(self.fs.cwd, file).lstrip("/")
ap = self.fs.v2a(vp)
self.vfs_map[ap] = vp
# print("ftp_STOR: {} {} => {}".format(vp, mode, ap))
ret = FTPHandler.ftp_STOR(self, file, mode)
# print("ftp_STOR: {} {} OK".format(vp, mode))
return ret
def log_transfer(
self,
cmd: str,
filename: bytes,
receive: bool,
completed: bool,
elapsed: float,
bytes: int,
) -> Any:
# None
ap = filename.decode("utf-8", "replace")
vp = self.vfs_map.pop(ap, None)
# print("xfer_end: {} => {}".format(ap, vp))
if vp:
vp, fn = os.path.split(vp)
vfs, rem = self.hub.asrv.vfs.get(vp, self.username, False, True)
vfs, rem = vfs.get_dbv(rem)
self.hub.up2k.hash_file(
vfs.realpath,
vfs.flags,
rem,
fn,
self.remote_ip,
time.time(),
)
return FTPHandler.log_transfer(
self, cmd, filename, receive, completed, elapsed, bytes
)
try:
from pyftpdlib.handlers import TLS_FTPHandler
class SftpHandler(FtpHandler, TLS_FTPHandler):
pass
except:
pass
class Ftpd(object):
def __init__(self, hub: "SvcHub") -> None:
self.hub = hub
self.args = hub.args
hs = []
if self.args.ftp:
hs.append([FtpHandler, self.args.ftp])
if self.args.ftps:
try:
h1 = SftpHandler
except:
t = "\nftps requires pyopenssl;\nplease run the following:\n\n {} -m pip install --user pyopenssl\n"
print(t.format(sys.executable))
sys.exit(1)
h1.certfile = os.path.join(self.args.E.cfg, "cert.pem")
h1.tls_control_required = True
h1.tls_data_required = True
hs.append([h1, self.args.ftps])
for h_lp in hs:
h2, lp = h_lp
h2.hub = hub
h2.args = hub.args
h2.authorizer = FtpAuth(hub)
if self.args.ftp_pr:
p1, p2 = [int(x) for x in self.args.ftp_pr.split("-")]
if self.args.ftp and self.args.ftps:
# divide port range in half
d = int((p2 - p1) / 2)
if lp == self.args.ftp:
p2 = p1 + d
else:
p1 += d + 1
h2.passive_ports = list(range(p1, p2 + 1))
if self.args.ftp_nat:
h2.masquerade_address = self.args.ftp_nat
lgr = logging.getLogger("pyftpdlib")
lgr.setLevel(logging.DEBUG if self.args.ftpv else logging.INFO)
ioloop = IOLoop()
for ip in self.args.i:
for h, lp in hs:
FTPServer((ip, int(lp)), h, ioloop)
Daemon(ioloop.loop, "ftp")
def join(p1: str, p2: str) -> str:
w = os.path.join(p1, p2.replace("\\", "/"))
return os.path.normpath(w).replace("\\", "/")

File diff suppressed because it is too large Load Diff

View File

@@ -1,38 +1,24 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import argparse # typechk
import os
import re
import socket
import threading # typechk
import os
import time
import socket
HAVE_SSL = True
try:
HAVE_SSL = True
import ssl
except:
HAVE_SSL = False
from . import util as Util
from .__init__ import TYPE_CHECKING, EnvParams
from .authsrv import AuthSrv # typechk
from .__init__ import E
from .util import Unrecv
from .httpcli import HttpCli
from .ico import Ico
from .mtag import HAVE_FFMPEG
from .th_cli import ThumbCli
from .th_srv import HAVE_PIL, HAVE_VIPS
from .u2idx import U2idx
from .util import HMaccas, shut_socket
if True: # pylint: disable=using-constant-test
from typing import Optional, Pattern, Union
if TYPE_CHECKING:
from .httpsrv import HttpSrv
PTN_HTTP = re.compile(br"[A-Z]{3}[A-Z ]")
from .th_cli import ThumbCli
from .th_srv import HAVE_PIL
from .ico import Ico
class HttpConn(object):
@@ -41,50 +27,39 @@ class HttpConn(object):
creates an HttpCli for each request (Connection: Keep-Alive)
"""
def __init__(
self, sck: socket.socket, addr: tuple[str, int], hsrv: "HttpSrv"
) -> None:
def __init__(self, sck, addr, hsrv):
self.s = sck
self.sr: Optional[Util._Unrecv] = None
self.cli: Optional[HttpCli] = None
self.addr = addr
self.hsrv = hsrv
self.mutex: threading.Lock = hsrv.mutex # mypy404
self.args: argparse.Namespace = hsrv.args # mypy404
self.E: EnvParams = self.args.E
self.asrv: AuthSrv = hsrv.asrv # mypy404
self.args = hsrv.args
self.asrv = hsrv.asrv
self.is_mp = hsrv.is_mp
self.cert_path = hsrv.cert_path
self.u2fh: Util.FHC = hsrv.u2fh # mypy404
self.iphash: HMaccas = hsrv.broker.iphash
self.bans: dict[str, int] = hsrv.bans
self.aclose: dict[str, int] = hsrv.aclose
enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb
self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None # mypy404
self.ico: Ico = Ico(self.args) # mypy404
enth = HAVE_PIL and not self.args.no_thumb
self.thumbcli = ThumbCli(hsrv.broker) if enth else None
self.ico = Ico(self.args)
self.t0: float = time.time() # mypy404
self.freshen_pwd: float = 0.0
self.t0 = time.time()
self.stopping = False
self.nreq: int = -1 # mypy404
self.nbyte: int = 0 # mypy404
self.u2idx: Optional[U2idx] = None
self.log_func: "Util.RootLogger" = hsrv.log # mypy404
self.log_src: str = "httpconn" # mypy404
self.lf_url: Optional[Pattern[str]] = (
re.compile(self.args.lf_url) if self.args.lf_url else None
) # mypy404
self.nreq = 0
self.nbyte = 0
self.workload = 0
self.u2idx = None
self.log_func = hsrv.log
self.lf_url = re.compile(self.args.lf_url) if self.args.lf_url else None
self.set_rproxy()
def shutdown(self) -> None:
def shutdown(self):
self.stopping = True
try:
shut_socket(self.log, self.s, 1)
self.s.shutdown(socket.SHUT_RDWR)
self.s.close()
except:
pass
def set_rproxy(self, ip: Optional[str] = None) -> str:
def set_rproxy(self, ip=None):
if ip is None:
color = 36
ip = self.addr[0]
@@ -97,37 +72,35 @@ class HttpConn(object):
self.log_src = "{} \033[{}m{}".format(ip, color, self.addr[1]).ljust(26)
return self.log_src
def respath(self, res_name: str) -> str:
return os.path.join(self.E.mod, "web", res_name)
def respath(self, res_name):
return os.path.join(E.mod, "web", res_name)
def log(self, msg: str, c: Union[int, str] = 0) -> None:
def log(self, msg, c=0):
self.log_func(self.log_src, msg, c)
def get_u2idx(self) -> U2idx:
# one u2idx per tcp connection;
# sqlite3 fully parallelizes under python threads
def get_u2idx(self):
if not self.u2idx:
self.u2idx = U2idx(self)
return self.u2idx
def _detect_https(self) -> bool:
def _detect_https(self):
method = None
if self.cert_path:
try:
method = self.s.recv(4, socket.MSG_PEEK)
except socket.timeout:
return False
return
except AttributeError:
# jython does not support msg_peek; forget about https
method = self.s.recv(4)
self.sr = Util.Unrecv(self.s, self.log)
self.sr = Unrecv(self.s)
self.sr.buf = method
# jython used to do this, they stopped since it's broken
# but reimplementing sendall is out of scope for now
if not getattr(self.s, "sendall", None):
self.s.sendall = self.s.send # type: ignore
self.s.sendall = self.s.send
if len(method) != 4:
err = "need at least 4 bytes in the first packet; got {}".format(
@@ -137,20 +110,17 @@ class HttpConn(object):
self.log(err)
self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
return False
return
return not method or not bool(PTN_HTTP.match(method))
def run(self) -> None:
self.s.settimeout(10)
return method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]
def run(self):
self.sr = None
if self.args.https_only:
is_https = True
elif self.args.http_only or not HAVE_SSL:
is_https = False
else:
# raise Exception("asdf")
is_https = self._detect_https()
if is_https:
@@ -179,15 +149,14 @@ class HttpConn(object):
self.s = ctx.wrap_socket(self.s, server_side=True)
msg = [
"\033[1;3{:d}m{}".format(c, s)
for c, s in zip([0, 5, 0], self.s.cipher()) # type: ignore
for c, s in zip([0, 5, 0], self.s.cipher())
]
self.log(" ".join(msg) + "\033[0m")
if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
ciphers = self.s.shared_ciphers()
assert ciphers
overlap = [str(y[::-1]) for y in ciphers]
self.log("TLS cipher overlap:" + "\n".join(overlap))
overlap = [y[::-1] for y in self.s.shared_ciphers()]
lines = [str(x) for x in (["TLS cipher overlap:"] + overlap)]
self.log("\n".join(lines))
for k, v in [
["compression", self.s.compression()],
["ALPN proto", self.s.selected_alpn_protocol()],
@@ -198,7 +167,11 @@ class HttpConn(object):
except Exception as ex:
em = str(ex)
if "ALERT_CERTIFICATE_UNKNOWN" in em:
if "ALERT_BAD_CERTIFICATE" in em:
# firefox-linux if there is no exception yet
self.log("client rejected our certificate (nice)")
elif "ALERT_CERTIFICATE_UNKNOWN" in em:
# android-chrome keeps doing this
pass
@@ -208,10 +181,15 @@ class HttpConn(object):
return
if not self.sr:
self.sr = Util.Unrecv(self.s, self.log)
self.sr = Unrecv(self.s)
while not self.stopping:
if self.is_mp:
self.workload += 50
if self.workload >= 2 ** 31:
self.workload = 100
self.nreq += 1
self.cli = HttpCli(self)
if not self.cli.run():
cli = HttpCli(self)
if not cli.run():
return

View File

@@ -1,15 +1,13 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import base64
import math
import os
import socket
import sys
import threading
import time
import queue
import base64
import struct
import socket
import threading
try:
import jinja2
@@ -28,31 +26,8 @@ except ImportError:
)
sys.exit(1)
from .__init__ import ANYWIN, MACOS, TYPE_CHECKING, EnvParams
from .bos import bos
from .__init__ import E, MACOS
from .httpconn import HttpConn
from .util import (
E_SCK,
FHC,
Daemon,
Garda,
Magician,
Netdev,
NetMap,
ipnorm,
min_ex,
shut_socket,
spack,
start_log_thrs,
start_stackmon,
)
if TYPE_CHECKING:
from .broker_util import BrokerCli
from .ssdp import SSDPr
if True: # pylint: disable=using-constant-test
from typing import Any, Optional
class HttpSrv(object):
@@ -61,353 +36,89 @@ class HttpSrv(object):
relying on MpSrv for performance (HttpSrv is just plain threads)
"""
def __init__(self, broker: "BrokerCli", nid: Optional[int]) -> None:
def __init__(self, broker, is_mp=False):
self.broker = broker
self.nid = nid
self.is_mp = is_mp
self.args = broker.args
self.E: EnvParams = self.args.E
self.log = broker.log
self.asrv = broker.asrv
# redefine in case of multiprocessing
socket.setdefaulttimeout(120)
nsuf = "-n{}-i{:x}".format(nid, os.getpid()) if nid else ""
self.magician = Magician()
self.nm = NetMap([], {})
self.ssdp: Optional["SSDPr"] = None
self.gpwd = Garda(self.args.ban_pw)
self.g404 = Garda(self.args.ban_404)
self.bans: dict[str, int] = {}
self.aclose: dict[str, int] = {}
self.bound: set[tuple[str, int]] = set()
self.name = "hsrv" + nsuf
self.disconnect_func = None
self.mutex = threading.Lock()
self.stopping = False
self.tp_nthr = 0 # actual
self.tp_ncli = 0 # fading
self.tp_time = 0.0 # latest worker collect
self.tp_q: Optional[queue.LifoQueue[Any]] = (
None if self.args.no_htp else queue.LifoQueue()
)
self.t_periodic: Optional[threading.Thread] = None
self.u2fh = FHC()
self.srvs: list[socket.socket] = []
self.ncli = 0 # exact
self.clients: set[HttpConn] = set() # laggy
self.nclimax = 0
self.cb_ts = 0.0
self.cb_v = ""
self.clients = {}
self.workload = 0
self.workload_thr_alive = False
self.cb_ts = 0
self.cb_v = 0
env = jinja2.Environment()
env.loader = jinja2.FileSystemLoader(os.path.join(self.E.mod, "web"))
jn = ["splash", "svcs", "browser", "browser2", "msg", "md", "mde", "cf"]
self.j2 = {x: env.get_template(x + ".html") for x in jn}
zs = os.path.join(self.E.mod, "web", "deps", "prism.js.gz")
self.prism = os.path.exists(zs)
env.loader = jinja2.FileSystemLoader(os.path.join(E.mod, "web"))
self.j2 = {
x: env.get_template(x + ".html")
for x in ["splash", "browser", "browser2", "msg", "md", "mde"]
}
self.mallow = "GET HEAD POST PUT DELETE OPTIONS".split()
if not self.args.no_dav:
zs = "PROPFIND PROPPATCH LOCK UNLOCK MKCOL COPY MOVE"
self.mallow += zs.split()
if self.args.zs:
from .ssdp import SSDPr
self.ssdp = SSDPr(broker)
cert_path = os.path.join(self.E.cfg, "cert.pem")
if bos.path.exists(cert_path):
cert_path = os.path.join(E.cfg, "cert.pem")
if os.path.exists(cert_path):
self.cert_path = cert_path
else:
self.cert_path = ""
self.cert_path = None
if self.tp_q:
self.start_threads(4)
if nid:
if self.args.stackmon:
start_stackmon(self.args.stackmon, nid)
if self.args.log_thrs:
start_log_thrs(self.log, self.args.log_thrs, nid)
self.th_cfg: dict[str, Any] = {}
Daemon(self.post_init, "hsrv-init2")
def post_init(self) -> None:
try:
x = self.broker.ask("thumbsrv.getcfg")
self.th_cfg = x.get()
except:
pass
def set_netdevs(self, netdevs: dict[str, Netdev]) -> None:
ips = set()
for ip, _ in self.bound:
ips.add(ip)
self.nm = NetMap(list(ips), netdevs)
def start_threads(self, n: int) -> None:
self.tp_nthr += n
if self.args.log_htp:
self.log(self.name, "workers += {} = {}".format(n, self.tp_nthr), 6)
for _ in range(n):
Daemon(self.thr_poolw, self.name + "-poolw")
def stop_threads(self, n: int) -> None:
self.tp_nthr -= n
if self.args.log_htp:
self.log(self.name, "workers -= {} = {}".format(n, self.tp_nthr), 6)
assert self.tp_q
for _ in range(n):
self.tp_q.put(None)
def periodic(self) -> None:
while True:
time.sleep(2 if self.tp_ncli or self.ncli else 10)
with self.mutex:
self.u2fh.clean()
if self.tp_q:
self.tp_ncli = max(self.ncli, self.tp_ncli - 2)
if self.tp_nthr > self.tp_ncli + 8:
self.stop_threads(4)
if not self.ncli and not self.u2fh.cache and self.tp_nthr <= 8:
self.t_periodic = None
return
def listen(self, sck: socket.socket, nlisteners: int) -> None:
if self.args.j != 1:
# lost in the pickle; redefine
if not ANYWIN or self.args.reuseaddr:
sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sck.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
sck.settimeout(None) # < does not inherit, ^ opts above do
ip, port = sck.getsockname()[:2]
self.srvs.append(sck)
self.bound.add((ip, port))
self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
Daemon(
self.thr_listen,
"httpsrv-n{}-listen-{}-{}".format(self.nid or "0", ip, port),
(sck,),
)
def thr_listen(self, srv_sck: socket.socket) -> None:
"""listens on a shared tcp server"""
ip, port = srv_sck.getsockname()[:2]
fno = srv_sck.fileno()
hip = "[{}]".format(ip) if ":" in ip else ip
msg = "subscribed @ {}:{} f{} p{}".format(hip, port, fno, os.getpid())
self.log(self.name, msg)
def fun() -> None:
self.broker.say("cb_httpsrv_up")
threading.Thread(target=fun, name="sig-hsrv-up1").start()
while not self.stopping:
if self.args.log_conn:
self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="90")
spins = 0
while self.ncli >= self.nclimax:
if not spins:
self.log(self.name, "at connection limit; waiting", 3)
spins += 1
time.sleep(0.1)
if spins != 50 or not self.args.aclose:
continue
ipfreq: dict[str, int] = {}
with self.mutex:
for c in self.clients:
ip = ipnorm(c.ip)
try:
ipfreq[ip] += 1
except:
ipfreq[ip] = 1
ip, n = sorted(ipfreq.items(), key=lambda x: x[1], reverse=True)[0]
if n < self.nclimax / 2:
continue
self.aclose[ip] = int(time.time() + self.args.aclose * 60)
nclose = 0
nloris = 0
nconn = 0
with self.mutex:
for c in self.clients:
cip = ipnorm(c.ip)
if ip != cip:
continue
nconn += 1
try:
if (
c.nreq >= 1
or not c.cli
or c.cli.in_hdr_recv
or c.cli.keepalive
):
Daemon(c.shutdown)
nclose += 1
if c.nreq <= 0 and (not c.cli or c.cli.in_hdr_recv):
nloris += 1
except:
pass
t = "{} downgraded to connection:close for {} min; dropped {}/{} connections"
self.log(self.name, t.format(ip, self.args.aclose, nclose, nconn), 1)
if nloris < nconn / 2:
continue
t = "slowloris (idle-conn): {} banned for {} min"
self.log(self.name, t.format(ip, self.args.loris, nclose), 1)
self.bans[ip] = int(time.time() + self.args.loris * 60)
if self.args.log_conn:
self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="90")
try:
sck, saddr = srv_sck.accept()
cip, cport = saddr[:2]
if cip.startswith("::ffff:"):
cip = cip[7:]
addr = (cip, cport)
except (OSError, socket.error) as ex:
if self.stopping:
break
self.log(self.name, "accept({}): {}".format(fno, ex), c=6)
time.sleep(0.02)
continue
if self.args.log_conn:
t = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
"-" * 3, ip, port % 8, port
)
self.log("%s %s" % addr, t, c="90")
self.accept(sck, addr)
def accept(self, sck: socket.socket, addr: tuple[str, int]) -> None:
def accept(self, sck, addr):
"""takes an incoming tcp connection and creates a thread to handle it"""
now = time.time()
if self.args.log_conn:
self.log("%s %s" % addr, "|%sC-cthr" % ("-" * 5,), c="1;30")
if now - (self.tp_time or now) > 300:
t = "httpserver threadpool died: tpt {:.2f}, now {:.2f}, nthr {}, ncli {}"
self.log(self.name, t.format(self.tp_time, now, self.tp_nthr, self.ncli), 1)
self.tp_time = 0
self.tp_q = None
with self.mutex:
self.ncli += 1
if not self.t_periodic:
name = "hsrv-pt"
if self.nid:
name += "-{}".format(self.nid)
self.t_periodic = Daemon(self.periodic, name)
if self.tp_q:
self.tp_time = self.tp_time or now
self.tp_ncli = max(self.tp_ncli, self.ncli)
if self.tp_nthr < self.ncli + 4:
self.start_threads(8)
self.tp_q.put((sck, addr))
return
if not self.args.no_htp:
t = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
self.log(self.name, t, 1)
Daemon(
self.thr_client,
"httpconn-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
(sck, addr),
thr = threading.Thread(
target=self.thr_client,
args=(sck, addr),
name="httpsrv-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
)
thr.daemon = True
thr.start()
def thr_poolw(self) -> None:
assert self.tp_q
while True:
task = self.tp_q.get()
if not task:
break
with self.mutex:
self.tp_time = 0
def num_clients(self):
with self.mutex:
return len(self.clients)
def shutdown(self):
clients = list(self.clients.keys())
for cli in clients:
try:
sck, addr = task
me = threading.current_thread()
me.name = "httpconn-{}-{}".format(
addr[0].split(".", 2)[-1][-6:], addr[1]
)
self.thr_client(sck, addr)
me.name = self.name + "-poolw"
except Exception as ex:
if str(ex).startswith("client d/c "):
self.log(self.name, "thr_client: " + str(ex), 6)
else:
self.log(self.name, "thr_client: " + min_ex(), 3)
def shutdown(self) -> None:
self.stopping = True
for srv in self.srvs:
try:
srv.close()
cli.shutdown()
except:
pass
thrs = []
clients = list(self.clients)
for cli in clients:
t = threading.Thread(target=cli.shutdown)
thrs.append(t)
t.start()
self.log("httpsrv-n", "ok bye")
if self.tp_q:
self.stop_threads(self.tp_nthr)
for _ in range(10):
time.sleep(0.05)
if self.tp_q.empty():
break
for t in thrs:
t.join()
self.log(self.name, "ok bye")
def thr_client(self, sck: socket.socket, addr: tuple[str, int]) -> None:
def thr_client(self, sck, addr):
"""thread managing one tcp client"""
sck.settimeout(120)
cli = HttpConn(sck, addr, self)
with self.mutex:
self.clients.add(cli)
self.clients[cli] = 0
if self.is_mp:
self.workload += 50
if not self.workload_thr_alive:
self.workload_thr_alive = True
thr = threading.Thread(
target=self.thr_workload, name="httpsrv-workload"
)
thr.daemon = True
thr.start()
# print("{}\n".format(len(self.clients)), end="")
fno = sck.fileno()
try:
if self.args.log_conn:
self.log("%s %s" % addr, "|%sC-crun" % ("-" * 4,), c="90")
self.log("%s %s" % addr, "|%sC-crun" % ("-" * 6,), c="1;30")
cli.run()
except (OSError, socket.error) as ex:
if ex.errno not in E_SCK:
if ex.errno not in [10038, 10054, 107, 57, 9]:
self.log(
"%s %s" % addr,
"run({}): {}".format(fno, ex),
@@ -417,26 +128,61 @@ class HttpSrv(object):
finally:
sck = cli.s
if self.args.log_conn:
self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 5,), c="90")
self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 7,), c="1;30")
try:
fno = sck.fileno()
shut_socket(cli.log, sck)
sck.shutdown(socket.SHUT_RDWR)
sck.close()
except (OSError, socket.error) as ex:
if not MACOS:
self.log(
"%s %s" % addr,
"shut({}): {}".format(fno, ex),
c="90",
c="1;30",
)
if ex.errno not in E_SCK:
if ex.errno not in [10038, 10054, 107, 57, 49, 9]:
# 10038 No longer considered a socket
# 10054 Foribly closed by remote
# 107 Transport endpoint not connected
# 57 Socket is not connected
# 49 Can't assign requested address (wifi down)
# 9 Bad file descriptor
raise
finally:
with self.mutex:
self.clients.remove(cli)
self.ncli -= 1
del self.clients[cli]
def cachebuster(self) -> str:
if self.disconnect_func:
self.disconnect_func(addr) # pylint: disable=not-callable
def thr_workload(self):
"""indicates the python interpreter workload caused by this HttpSrv"""
# avoid locking in extract_filedata by tracking difference here
while True:
time.sleep(0.2)
with self.mutex:
if not self.clients:
# no clients rn, termiante thread
self.workload_thr_alive = False
self.workload = 0
return
total = 0
with self.mutex:
for cli in self.clients.keys():
now = cli.workload
delta = now - self.clients[cli]
if delta < 0:
# was reset in HttpCli to prevent overflow
delta = now
total += delta
self.clients[cli] = now
self.workload = total
def cachebuster(self):
if time.time() - self.cb_ts < 1:
return self.cb_v
@@ -444,16 +190,16 @@ class HttpSrv(object):
if time.time() - self.cb_ts < 1:
return self.cb_v
v = self.E.t0
v = E.t0
try:
with os.scandir(os.path.join(self.E.mod, "web")) as dh:
with os.scandir(os.path.join(E.mod, "web")) as dh:
for fh in dh:
inf = fh.stat()
inf = fh.stat(follow_symlinks=False)
v = max(v, inf.st_mtime)
except:
pass
v = base64.urlsafe_b64encode(spack(b">xxL", int(v)))
v = base64.urlsafe_b64encode(struct.pack(">xxL", int(v)))
self.cb_v = v.decode("ascii")[-4:]
self.cb_ts = time.time()
return self.cb_v

View File

@@ -1,69 +1,33 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import argparse # typechk
import colorsys
import hashlib
import colorsys
from .__init__ import PY2
from .th_srv import HAVE_PIL
from .util import BytesIO
class Ico(object):
def __init__(self, args: argparse.Namespace) -> None:
def __init__(self, args):
self.args = args
def get(self, ext: str, as_thumb: bool, chrome: bool) -> tuple[str, bytes]:
def get(self, ext, as_thumb):
"""placeholder to make thumbnails not break"""
zb = hashlib.sha1(ext.encode("utf-8")).digest()[2:4]
h = hashlib.md5(ext.encode("utf-8")).digest()[:2]
if PY2:
zb = [ord(x) for x in zb]
h = [ord(x) for x in h]
c1 = colorsys.hsv_to_rgb(zb[0] / 256.0, 1, 0.3)
c2 = colorsys.hsv_to_rgb(zb[0] / 256.0, 1, 1)
ci = [int(x * 255) for x in list(c1) + list(c2)]
c = "".join(["{:02x}".format(x) for x in ci])
c1 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 0.3)
c2 = colorsys.hsv_to_rgb(h[0] / 256.0, 1, 1)
c = list(c1) + list(c2)
c = [int(x * 255) for x in c]
c = "".join(["{:02x}".format(x) for x in c])
w = 100
h = 30
if not self.args.th_no_crop and as_thumb:
sw, sh = self.args.th_size.split("x")
h = int(100 / (float(sw) / float(sh)))
w = 100
if chrome and as_thumb:
# cannot handle more than ~2000 unique SVGs
if HAVE_PIL:
# svg: 3s, cache: 6s, this: 8s
from PIL import Image, ImageDraw
h = int(64 * h / w)
w = 64
img = Image.new("RGB", (w, h), "#" + c[:6])
pb = ImageDraw.Draw(img)
tw, th = pb.textsize(ext)
pb.text(((w - tw) // 2, (h - th) // 2), ext, fill="#" + c[6:])
img = img.resize((w * 3, h * 3), Image.NEAREST)
buf = BytesIO()
img.save(buf, format="PNG", compress_level=1)
return "image/png", buf.getvalue()
elif False:
# 48s, too slow
import pyvips
h = int(192 * h / w)
w = 192
img = pyvips.Image.text(
ext, width=w, height=h, dpi=192, align=pyvips.Align.CENTRE
)
img = img.ifthenelse(ci[3:], ci[:3], blend=True)
# i = i.resize(3, kernel=pyvips.Kernel.NEAREST)
buf = img.write_to_buffer(".png[compression=1]")
return "image/png", buf
w, h = self.args.th_size.split("x")
h = int(100 / (float(w) / float(h)))
svg = """\
<?xml version="1.0" encoding="UTF-8"?>
@@ -73,6 +37,6 @@ class Ico(object):
fill="#{}" font-family="monospace" font-size="14px" style="letter-spacing:.5px">{}</text>
</g></svg>
"""
svg = svg.format(h, c[:6], c[6:], ext)
svg = svg.format(h, c[:6], c[6:], ext).encode("utf-8")
return "image/svg+xml", svg.encode("utf-8")
return ["image/svg+xml", svg]

View File

@@ -1,538 +0,0 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import random
import select
import socket
import time
from ipaddress import IPv4Network, IPv6Network
from .__init__ import TYPE_CHECKING
from .__init__ import unicode as U
from .multicast import MC_Sck, MCast
from .stolen.dnslib import CLASS as DC
from .stolen.dnslib import (
NSEC,
PTR,
QTYPE,
RR,
SRV,
TXT,
A,
AAAA,
DNSHeader,
DNSQuestion,
DNSRecord,
)
from .util import CachedSet, Daemon, Netdev, list_ips, min_ex
if TYPE_CHECKING:
from .svchub import SvcHub
if True: # pylint: disable=using-constant-test
from typing import Any, Optional, Union
MDNS4 = "224.0.0.251"
MDNS6 = "ff02::fb"
class MDNS_Sck(MC_Sck):
def __init__(
self,
sck: socket.socket,
nd: Netdev,
grp: str,
ip: str,
net: Union[IPv4Network, IPv6Network],
):
super(MDNS_Sck, self).__init__(sck, nd, grp, ip, net)
self.bp_probe = b""
self.bp_ip = b""
self.bp_svc = b""
self.bp_bye = b""
self.last_tx = 0.0
self.tx_ex = False
class MDNS(MCast):
def __init__(self, hub: "SvcHub", ngen: int) -> None:
al = hub.args
grp4 = "" if al.zm6 else MDNS4
grp6 = "" if al.zm4 else MDNS6
super(MDNS, self).__init__(
hub, MDNS_Sck, al.zm_on, al.zm_off, grp4, grp6, 5353, hub.args.zmv
)
self.srv: dict[socket.socket, MDNS_Sck] = {}
self.logsrc = "mDNS-{}".format(ngen)
self.ngen = ngen
self.ttl = 300
zs = self.args.name + ".local."
zs = zs.encode("ascii", "replace").decode("ascii", "replace")
self.hn = "-".join(x for x in zs.split("?") if x) or (
"vault-{}".format(random.randint(1, 255))
)
self.lhn = self.hn.lower()
# requester ip -> (response deadline, srv, body):
self.q: dict[str, tuple[float, MDNS_Sck, bytes]] = {}
self.rx4 = CachedSet(0.42) # 3 probes @ 250..500..750 => 500ms span
self.rx6 = CachedSet(0.42)
self.svcs, self.sfqdns = self.build_svcs()
self.lsvcs = {k.lower(): v for k, v in self.svcs.items()}
self.lsfqdns = set([x.lower() for x in self.sfqdns])
self.probing = 0.0
self.unsolicited: list[float] = [] # scheduled announces on all nics
self.defend: dict[MDNS_Sck, float] = {} # server -> deadline
def log(self, msg: str, c: Union[int, str] = 0) -> None:
self.log_func(self.logsrc, msg, c)
def build_svcs(self) -> tuple[dict[str, dict[str, Any]], set[str]]:
zms = self.args.zms
http = {"port": 80 if 80 in self.args.p else self.args.p[0]}
https = {"port": 443 if 443 in self.args.p else self.args.p[0]}
webdav = http.copy()
webdavs = https.copy()
webdav["u"] = webdavs["u"] = "u" # KDE requires username
ftp = {"port": (self.args.ftp if "f" in zms else self.args.ftps)}
smb = {"port": self.args.smb_port}
# some gvfs require path
zs = self.args.zm_ld or "/"
if zs:
webdav["path"] = zs
webdavs["path"] = zs
if self.args.zm_lh:
http["path"] = self.args.zm_lh
https["path"] = self.args.zm_lh
if self.args.zm_lf:
ftp["path"] = self.args.zm_lf
if self.args.zm_ls:
smb["path"] = self.args.zm_ls
svcs: dict[str, dict[str, Any]] = {}
if "d" in zms:
svcs["_webdav._tcp.local."] = webdav
if "D" in zms:
svcs["_webdavs._tcp.local."] = webdavs
if "h" in zms:
svcs["_http._tcp.local."] = http
if "H" in zms:
svcs["_https._tcp.local."] = https
if "f" in zms.lower():
svcs["_ftp._tcp.local."] = ftp
if "s" in zms.lower():
svcs["_smb._tcp.local."] = smb
sfqdns: set[str] = set()
for k, v in svcs.items():
name = "{}-c-{}".format(self.args.name, k.split(".")[0][1:])
v["name"] = name
sfqdns.add("{}.{}".format(name, k))
return svcs, sfqdns
def build_replies(self) -> None:
for srv in self.srv.values():
probe = DNSRecord(DNSHeader(0, 0), q=DNSQuestion(self.hn, QTYPE.ANY))
areply = DNSRecord(DNSHeader(0, 0x8400))
sreply = DNSRecord(DNSHeader(0, 0x8400))
bye = DNSRecord(DNSHeader(0, 0x8400))
have4 = have6 = False
for s2 in self.srv.values():
if srv.idx != s2.idx:
continue
if s2.v6:
have6 = True
else:
have4 = True
for ip in srv.ips:
if ":" in ip:
qt = QTYPE.AAAA
ar = {"rclass": DC.F_IN, "rdata": AAAA(ip)}
else:
qt = QTYPE.A
ar = {"rclass": DC.F_IN, "rdata": A(ip)}
r0 = RR(self.hn, qt, ttl=0, **ar)
r120 = RR(self.hn, qt, ttl=120, **ar)
# rfc-10:
# SHOULD rr ttl 120sec for A/AAAA/SRV
# (and recommend 75min for all others)
probe.add_auth(r120)
areply.add_answer(r120)
sreply.add_answer(r120)
bye.add_answer(r0)
for sclass, props in self.svcs.items():
sname = props["name"]
sport = props["port"]
sfqdn = sname + "." + sclass
k = "_services._dns-sd._udp.local."
r = RR(k, QTYPE.PTR, DC.IN, 4500, PTR(sclass))
sreply.add_answer(r)
r = RR(sclass, QTYPE.PTR, DC.IN, 4500, PTR(sfqdn))
sreply.add_answer(r)
r = RR(sfqdn, QTYPE.SRV, DC.F_IN, 120, SRV(0, 0, sport, self.hn))
sreply.add_answer(r)
areply.add_answer(r)
r = RR(sfqdn, QTYPE.SRV, DC.F_IN, 0, SRV(0, 0, sport, self.hn))
bye.add_answer(r)
txts = []
for k in ("u", "path"):
if k not in props:
continue
zb = "{}={}".format(k, props[k]).encode("utf-8")
if len(zb) > 255:
t = "value too long for mdns: [{}]"
raise Exception(t.format(props[k]))
txts.append(zb)
# gvfs really wants txt even if they're empty
r = RR(sfqdn, QTYPE.TXT, DC.F_IN, 4500, TXT(txts))
sreply.add_answer(r)
if not (have4 and have6) and not self.args.zm_noneg:
ns = NSEC(self.hn, ["AAAA" if have6 else "A"])
r = RR(self.hn, QTYPE.NSEC, DC.F_IN, 120, ns)
areply.add_ar(r)
if len(sreply.pack()) < 1400:
sreply.add_ar(r)
srv.bp_probe = probe.pack()
srv.bp_ip = areply.pack()
srv.bp_svc = sreply.pack()
srv.bp_bye = bye.pack()
# since all replies are small enough to fit in one packet,
# always send full replies rather than just a/aaaa records
srv.bp_ip = srv.bp_svc
def send_probes(self) -> None:
slp = random.random() * 0.25
for _ in range(3):
time.sleep(slp)
slp = 0.25
if not self.running:
break
if self.args.zmv:
self.log("sending hostname probe...")
# ipv4: need to probe each ip (each server)
# ipv6: only need to probe each set of looped nics
probed6: set[str] = set()
for srv in self.srv.values():
if srv.ip in probed6:
continue
try:
srv.sck.sendto(srv.bp_probe, (srv.grp, 5353))
if srv.v6:
for ip in srv.ips:
probed6.add(ip)
except Exception as ex:
self.log("sendto failed: {} ({})".format(srv.ip, ex), "90")
def run(self) -> None:
try:
bound = self.create_servers()
except:
t = "no server IP matches the mdns config\n{}"
self.log(t.format(min_ex()), 1)
bound = []
if not bound:
self.log("failed to announce copyparty services on the network", 3)
return
self.build_replies()
Daemon(self.send_probes)
zf = time.time() + 2
self.probing = zf # cant unicast so give everyone an extra sec
self.unsolicited = [zf, zf + 1, zf + 3, zf + 7] # rfc-8.3
last_hop = time.time()
ihop = self.args.mc_hop
while self.running:
timeout = (
0.02 + random.random() * 0.07
if self.probing or self.q or self.defend or self.unsolicited
else (last_hop + ihop if ihop else 180)
)
rdy = select.select(self.srv, [], [], timeout)
rx: list[socket.socket] = rdy[0] # type: ignore
self.rx4.cln()
self.rx6.cln()
buf = b""
addr = ("0", 0)
for sck in rx:
try:
buf, addr = sck.recvfrom(4096)
self.eat(buf, addr, sck)
except:
if not self.running:
self.log("stopped", 2)
return
t = "{} {} \033[33m|{}| {}\n{}".format(
self.srv[sck].name, addr, len(buf), repr(buf)[2:-1], min_ex()
)
self.log(t, 6)
if not self.probing:
self.process()
continue
if self.probing < time.time():
t = "probe ok; announcing [{}]"
self.log(t.format(self.hn[:-1]), 2)
self.probing = 0
self.log("stopped", 2)
def stop(self, panic=False) -> None:
self.running = False
for srv in self.srv.values():
try:
if panic:
srv.sck.close()
else:
srv.sck.sendto(srv.bp_bye, (srv.grp, 5353))
except:
pass
self.srv = {}
def eat(self, buf: bytes, addr: tuple[str, int], sck: socket.socket) -> None:
cip = addr[0]
v6 = ":" in cip
if (cip.startswith("169.254") and not self.ll_ok) or (
v6 and not cip.startswith("fe80")
):
return
cache = self.rx6 if v6 else self.rx4
if buf in cache.c:
return
srv: Optional[MDNS_Sck] = self.srv[sck] if v6 else self.map_client(cip) # type: ignore
if not srv:
return
cache.add(buf)
now = time.time()
if self.args.zmv and cip != srv.ip and cip not in srv.ips:
t = "{} [{}] \033[36m{} \033[0m|{}|"
self.log(t.format(srv.name, srv.ip, cip, len(buf)), "90")
p = DNSRecord.parse(buf)
if self.args.zmvv:
self.log(str(p))
# check for incoming probes for our hostname
cips = [U(x.rdata) for x in p.auth if U(x.rname).lower() == self.lhn]
if cips and self.sips.isdisjoint(cips):
if not [x for x in cips if x not in ("::1", "127.0.0.1")]:
# avahi broadcasting 127.0.0.1-only packets
return
self.log("someone trying to steal our hostname: {}".format(cips), 3)
# immediately unicast
if not self.probing:
srv.sck.sendto(srv.bp_ip, (cip, 5353))
# and schedule multicast
self.defend[srv] = self.defend.get(srv, now + 0.1)
return
# check for someone rejecting our probe / hijacking our hostname
cips = [
U(x.rdata)
for x in p.rr
if U(x.rname).lower() == self.lhn and x.rclass == DC.F_IN
]
if cips and self.sips.isdisjoint(cips):
if not [x for x in cips if x not in ("::1", "127.0.0.1")]:
# avahi broadcasting 127.0.0.1-only packets
return
# check if we've been given additional IPs
for ip in list_ips():
if ip in cips:
self.sips.add(ip)
if not self.sips.isdisjoint(cips):
return
t = "mdns zeroconf: "
if self.probing:
t += "Cannot start; hostname '{}' is occupied"
else:
t += "Emergency stop; hostname '{}' got stolen"
t += " on {}! Use --name to set another hostname.\n\nName taken by {}\n\nYour IPs: {}\n"
self.log(t.format(self.args.name, srv.name, cips, list(self.sips)), 1)
self.stop(True)
return
# then rfc-6.7; dns pretending to be mdns (android...)
if p.header.id or addr[1] != 5353:
rsp: Optional[DNSRecord] = None
for r in p.questions:
try:
lhn = U(r.qname).lower()
except:
self.log("invalid question: {}".format(r))
continue
if lhn != self.lhn:
continue
if p.header.id and r.qtype in (QTYPE.A, QTYPE.AAAA):
rsp = rsp or DNSRecord(DNSHeader(p.header.id, 0x8400))
rsp.add_question(r)
for ip in srv.ips:
qt = r.qtype
v6 = ":" in ip
if v6 == (qt == QTYPE.AAAA):
rd = AAAA(ip) if v6 else A(ip)
rr = RR(self.hn, qt, DC.IN, 10, rd)
rsp.add_answer(rr)
if rsp:
srv.sck.sendto(rsp.pack(), addr[:2])
# but don't return in case it's a differently broken client
# then a/aaaa records
for r in p.questions:
try:
lhn = U(r.qname).lower()
except:
self.log("invalid question: {}".format(r))
continue
if lhn != self.lhn:
continue
# gvfs keeps repeating itself
found = False
unicast = False
for rr in p.rr:
try:
rname = U(rr.rname).lower()
except:
self.log("invalid rr: {}".format(rr))
continue
if rname == self.lhn:
if rr.ttl > 60:
found = True
if rr.rclass == DC.F_IN:
unicast = True
if unicast:
# spec-compliant mDNS-over-unicast
srv.sck.sendto(srv.bp_ip, (cip, 5353))
elif addr[1] != 5353:
# just in case some clients use (and want us to use) invalid ports
srv.sck.sendto(srv.bp_ip, addr[:2])
if not found:
self.q[cip] = (0, srv, srv.bp_ip)
return
deadline = now + (0.5 if p.header.tc else 0.02) # rfc-7.2
# and service queries
for r in p.questions:
if not r or not r.qname:
continue
qname = U(r.qname).lower()
if qname in self.lsvcs or qname == "_services._dns-sd._udp.local.":
self.q[cip] = (deadline, srv, srv.bp_svc)
break
# heed rfc-7.1 if there was an announce in the past 12sec
# (workaround gvfs race-condition where it occasionally
# doesn't read/decode the full response...)
if now < srv.last_tx + 12:
for rr in p.rr:
if not rr.rdata:
continue
rdata = U(rr.rdata).lower()
if rdata in self.lsfqdns:
if rr.ttl > 2250:
self.q.pop(cip, None)
break
def process(self) -> None:
tx = set()
now = time.time()
cooldown = 0.9 # rfc-6: 1
if self.unsolicited and self.unsolicited[0] < now:
self.unsolicited.pop(0)
cooldown = 0.1
for srv in self.srv.values():
tx.add(srv)
for srv, deadline in list(self.defend.items()):
if now < deadline:
continue
if self._tx(srv, srv.bp_ip, 0.02): # rfc-6: 0.25
self.defend.pop(srv)
for cip, (deadline, srv, msg) in list(self.q.items()):
if now < deadline:
continue
self.q.pop(cip)
self._tx(srv, msg, cooldown)
for srv in tx:
self._tx(srv, srv.bp_svc, cooldown)
def _tx(self, srv: MDNS_Sck, msg: bytes, cooldown: float) -> bool:
now = time.time()
if now < srv.last_tx + cooldown:
return False
try:
srv.sck.sendto(msg, (srv.grp, 5353))
srv.last_tx = now
except Exception as ex:
if srv.tx_ex:
return True
srv.tx_ex = True
t = "tx({},|{}|,{}): {}"
self.log(t.format(srv.ip, len(msg), cooldown, ex), 3)
return True

View File

@@ -1,34 +1,30 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import argparse
import json
import os
import sys
import json
import shutil
import subprocess as sp
import sys
from .__init__ import PY2, WINDOWS, E, unicode
from .bos import bos
from .util import REKOBO_LKEY, fsenc, min_ex, retchk, runcmd, uncyg
from .__init__ import PY2, WINDOWS
from .util import fsenc, fsdec, uncyg, REKOBO_LKEY
if True: # pylint: disable=using-constant-test
from typing import Any, Union
from .util import RootLogger
if not PY2:
unicode = str
def have_ff(scmd: str) -> bool:
def have_ff(cmd):
if PY2:
print("# checking {}".format(scmd))
acmd = (scmd + " -version").encode("ascii").split(b" ")
print("# checking {}".format(cmd))
cmd = (cmd + " -version").encode("ascii").split(b" ")
try:
sp.Popen(acmd, stdout=sp.PIPE, stderr=sp.PIPE).communicate()
sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE).communicate()
return True
except:
return False
else:
return bool(shutil.which(scmd))
return bool(shutil.which(cmd))
HAVE_FFMPEG = have_ff("ffmpeg")
@@ -36,16 +32,13 @@ HAVE_FFPROBE = have_ff("ffprobe")
class MParser(object):
def __init__(self, cmdline: str) -> None:
def __init__(self, cmdline):
self.tag, args = cmdline.split("=", 1)
self.tags = self.tag.split(",")
self.timeout = 60
self.timeout = 30
self.force = False
self.kill = "t" # tree; all children recursively
self.capture = 3 # outputs to consume
self.audio = "y"
self.pri = 0 # priority; higher = later
self.ext = []
while True:
@@ -54,7 +47,7 @@ class MParser(object):
if WINDOWS:
bp = uncyg(bp)
if bos.path.exists(bp):
if os.path.exists(bp):
self.bin = bp
return
except:
@@ -67,14 +60,6 @@ class MParser(object):
self.audio = arg[1:] # [r]equire [n]ot [d]ontcare
continue
if arg.startswith("k"):
self.kill = arg[1:] # [t]ree [m]ain [n]one
continue
if arg.startswith("c"):
self.capture = int(arg[1:]) # 0=none 1=stdout 2=stderr 3=both
continue
if arg == "f":
self.force = True
continue
@@ -87,16 +72,10 @@ class MParser(object):
self.ext.append(arg[1:])
continue
if arg.startswith("p"):
self.pri = int(arg[1:] or "1")
continue
raise Exception()
def ffprobe(
abspath: str, timeout: int = 60
) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
def ffprobe(abspath):
cmd = [
b"ffprobe",
b"-hide_banner",
@@ -105,20 +84,21 @@ def ffprobe(
b"--",
fsenc(abspath),
]
rc, so, se = runcmd(cmd, timeout=timeout)
retchk(rc, cmd, se)
return parse_ffprobe(so)
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
r = p.communicate()
txt = r[0].decode("utf-8", "replace")
return parse_ffprobe(txt)
def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[Any]]]:
def parse_ffprobe(txt):
"""ffprobe -show_format -show_streams"""
streams = []
fmt = {}
g = {}
g = None
for ln in [x.rstrip("\r") for x in txt.split("\n")]:
try:
sk, sv = ln.split("=", 1)
g[sk] = sv
k, v = ln.split("=", 1)
g[k] = v
continue
except:
pass
@@ -132,8 +112,8 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
fmt = g
streams = [fmt] + streams
ret: dict[str, Any] = {} # processed
md: dict[str, list[Any]] = {} # raw tags
ret = {} # processed
md = {} # raw tags
is_audio = fmt.get("format_name") in ["mp3", "ogg", "flac", "wav"]
if fmt.get("filename", "").split(".")[-1].lower() in ["m4a", "aac"]:
@@ -181,55 +161,52 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
]
if typ == "format":
kvm = [["duration", ".dur"], ["bit_rate", ".q"], ["format_name", "fmt"]]
kvm = [["duration", ".dur"], ["bit_rate", ".q"]]
for sk, rk in kvm:
v1 = strm.get(sk)
if v1 is None:
v = strm.get(sk)
if v is None:
continue
if rk.startswith("."):
try:
zf = float(v1)
v = float(v)
v2 = ret.get(rk)
if v2 is None or zf > v2:
ret[rk] = zf
if v2 is None or v > v2:
ret[rk] = v
except:
# sqlite doesnt care but the code below does
if v1 not in ["N/A"]:
ret[rk] = v1
if v not in ["N/A"]:
ret[rk] = v
else:
ret[rk] = v1
ret[rk] = v
if ret.get("vc") == "ansi": # shellscript
return {}, {}
for strm in streams:
for sk, sv in strm.items():
if not sk.startswith("TAG:"):
for k, v in strm.items():
if not k.startswith("TAG:"):
continue
sk = sk[4:].strip()
sv = sv.strip()
if sk and sv and sk not in md:
md[sk] = [sv]
k = k[4:].strip()
v = v.strip()
if k and v and k not in md:
md[k] = [v]
for sk in [".q", ".vq", ".aq"]:
if sk in ret:
ret[sk] /= 1000 # bit_rate=320000
for k in [".q", ".vq", ".aq"]:
if k in ret:
ret[k] /= 1000 # bit_rate=320000
for sk in [".q", ".vq", ".aq", ".resw", ".resh"]:
if sk in ret:
ret[sk] = int(ret[sk])
for k in [".q", ".vq", ".aq", ".resw", ".resh"]:
if k in ret:
ret[k] = int(ret[k])
if ".fps" in ret:
fps = ret[".fps"]
if "/" in fps:
fa, fb = fps.split("/")
try:
fps = int(fa) * 1.0 / int(fb)
except:
fps = 9001
fps = int(fa) * 1.0 / int(fb)
if fps < 1000 and fmt.get("format_name") not in ["image2", "png_pipe"]:
ret[".fps"] = round(fps, 3)
@@ -242,52 +219,48 @@ def parse_ffprobe(txt: str) -> tuple[dict[str, tuple[int, Any]], dict[str, list[
if ".q" in ret:
del ret[".q"]
if "fmt" in ret:
ret["fmt"] = ret["fmt"].split(",")[0]
if ".resw" in ret and ".resh" in ret:
ret["res"] = "{}x{}".format(ret[".resw"], ret[".resh"])
zd = {k: (0, v) for k, v in ret.items()}
ret = {k: [0, v] for k, v in ret.items()}
return zd, md
return ret, md
class MTag(object):
def __init__(self, log_func: "RootLogger", args: argparse.Namespace) -> None:
def __init__(self, log_func, args):
self.log_func = log_func
self.args = args
self.usable = True
self.prefer_mt = not args.no_mtag_ff
self.backend = "ffprobe" if args.no_mutagen else "mutagen"
self.can_ffprobe = HAVE_FFPROBE and not args.no_mtag_ff
self.prefer_mt = False
mappings = args.mtm
or_ffprobe = " or FFprobe"
self.backend = "ffprobe" if args.no_mutagen else "mutagen"
or_ffprobe = " or ffprobe"
if self.backend == "mutagen":
self.get = self.get_mutagen
try:
from mutagen import version # noqa: F401
import mutagen
except:
self.log("could not load Mutagen, trying FFprobe instead", c=3)
self.log("could not load mutagen, trying ffprobe instead", c=3)
self.backend = "ffprobe"
if self.backend == "ffprobe":
self.usable = self.can_ffprobe
self.get = self.get_ffprobe
self.prefer_mt = True
# about 20x slower
self.usable = HAVE_FFPROBE
if not HAVE_FFPROBE:
pass
elif args.no_mtag_ff:
msg = "found FFprobe but it was disabled by --no-mtag-ff"
self.log(msg, c=3)
if self.usable and WINDOWS and sys.version_info < (3, 8):
self.usable = False
or_ffprobe = " or python >= 3.8"
msg = "found ffprobe but your python is too old; need 3.8 or newer"
self.log(msg, c=1)
if not self.usable:
msg = "need Mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
pybin = os.path.basename(sys.executable)
self.log(msg.format(or_ffprobe, " " * 37, pybin), c=1)
msg = "need mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
self.log(
msg.format(or_ffprobe, " " * 37, os.path.basename(sys.executable)), c=1
)
return
# https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
@@ -359,49 +332,41 @@ class MTag(object):
}
# self.get = self.compare
def log(self, msg: str, c: Union[int, str] = 0) -> None:
def log(self, msg, c=0):
self.log_func("mtag", msg, c)
def normalize_tags(
self, parser_output: dict[str, tuple[int, Any]], md: dict[str, list[Any]]
) -> dict[str, Union[str, float]]:
for sk, tv in dict(md).items():
if not tv:
def normalize_tags(self, ret, md):
for k, v in dict(md).items():
if not v:
continue
sk = sk.lower().split("::")[0].strip()
key_mapping = self.rmap.get(sk)
if not key_mapping:
k = k.lower().split("::")[0].strip()
mk = self.rmap.get(k)
if not mk:
continue
priority, alias = key_mapping
if alias not in parser_output or parser_output[alias][0] > priority:
parser_output[alias] = (priority, tv[0])
pref, mk = mk
if mk not in ret or ret[mk][0] > pref:
ret[mk] = [pref, v[0]]
# take first value (lowest priority / most preferred)
ret: dict[str, Union[str, float]] = {
sk: unicode(tv[1]).strip() for sk, tv in parser_output.items()
}
# take first value
ret = {k: unicode(v[1]).strip() for k, v in ret.items()}
# track 3/7 => track 3
for sk, zv in ret.items():
if sk[0] == ".":
sv = str(zv).split("/")[0].strip().lstrip("0")
ret[sk] = sv or 0
for k, v in ret.items():
if k[0] == ".":
v = v.split("/")[0].strip().lstrip("0")
ret[k] = v or 0
# normalize key notation to rkeobo
okey = ret.get("key")
if okey:
key = str(okey).replace(" ", "").replace("maj", "").replace("min", "m")
key = okey.replace(" ", "").replace("maj", "").replace("min", "m")
ret["key"] = REKOBO_LKEY.get(key.lower(), okey)
if self.args.mtag_vv:
zl = " ".join("\033[36m{} \033[33m{}".format(k, v) for k, v in ret.items())
self.log("norm: {}\033[0m".format(zl), "90")
return ret
def compare(self, abspath: str) -> dict[str, Union[str, float]]:
def compare(self, abspath):
if abspath.endswith(".au"):
return {}
@@ -425,7 +390,7 @@ class MTag(object):
v2 = r2.get(k)
if v1 == v2:
print(" ", k, v1)
elif v1 != "0000": # FFprobe date=0
elif v1 != "0000": # ffprobe date=0
diffs.append(k)
print(" 1", k, v1)
print(" 2", k, v2)
@@ -439,135 +404,65 @@ class MTag(object):
return r1
def get_mutagen(self, abspath: str) -> dict[str, Union[str, float]]:
ret: dict[str, tuple[int, Any]] = {}
def get_mutagen(self, abspath):
import mutagen
if not bos.path.isfile(abspath):
try:
md = mutagen.File(fsenc(abspath), easy=True)
x = md.info.length
except Exception as ex:
return {}
from mutagen import File
ret = {}
try:
md = File(fsenc(abspath), easy=True)
assert md
if self.args.mtag_vv:
for zd in (md.info.__dict__, dict(md.tags)):
zl = ["\033[36m{} \033[33m{}".format(k, v) for k, v in zd.items()]
self.log("mutagen: {}\033[0m".format(" ".join(zl)), "90")
if not md.info.length and not md.info.codec:
raise Exception()
except:
return self.get_ffprobe(abspath) if self.can_ffprobe else {}
dur = int(md.info.length)
try:
q = int(md.info.bitrate / 1024)
except:
q = int((os.path.getsize(fsenc(abspath)) / dur) / 128)
sz = bos.path.getsize(abspath)
try:
ret[".q"] = (0, int((sz / md.info.length) / 128))
ret[".dur"] = [0, dur]
ret[".q"] = [0, q]
except:
pass
for attr, k, norm in [
["codec", "ac", unicode],
["channels", "chs", int],
["sample_rate", ".hz", int],
["bitrate", ".aq", int],
["length", ".dur", int],
]:
try:
v = getattr(md.info, attr)
except:
if k != "ac":
continue
try:
v = str(md.info).split(".")[1]
if v.startswith("ogg"):
v = v[3:]
except:
continue
if not v:
continue
if k == ".aq":
v /= 1000
if k == "ac" and v.startswith("mp4a.40."):
v = "aac"
ret[k] = (0, norm(v))
return self.normalize_tags(ret, md)
def get_ffprobe(self, abspath: str) -> dict[str, Union[str, float]]:
if not bos.path.isfile(abspath):
return {}
ret, md = ffprobe(abspath, self.args.mtag_to)
if self.args.mtag_vv:
for zd in (ret, dict(md)):
zl = ["\033[36m{} \033[33m{}".format(k, v) for k, v in zd.items()]
self.log("ffprobe: {}\033[0m".format(" ".join(zl)), "90")
def get_ffprobe(self, abspath):
ret, md = ffprobe(abspath)
return self.normalize_tags(ret, md)
def get_bin(
self, parsers: dict[str, MParser], abspath: str, oth_tags: dict[str, Any]
) -> dict[str, Any]:
if not bos.path.isfile(abspath):
return {}
def get_bin(self, parsers, abspath):
pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
pypath = [str(pypath)] + [str(x) for x in sys.path if x]
pypath = str(os.pathsep.join(pypath))
env = os.environ.copy()
try:
pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
zsl = [str(pypath)] + [str(x) for x in sys.path if x]
pypath = str(os.pathsep.join(zsl))
env["PYTHONPATH"] = pypath
except:
if not E.ox:
raise
env["PYTHONPATH"] = pypath
ret: dict[str, Any] = {}
for tagname, parser in sorted(parsers.items(), key=lambda x: (x[1].pri, x[0])):
ret = {}
for tagname, mp in parsers.items():
try:
cmd = [parser.bin, abspath]
if parser.bin.endswith(".py"):
cmd = [sys.executable] + cmd
args = {
"env": env,
"timeout": parser.timeout,
"kill": parser.kill,
"capture": parser.capture,
}
if parser.pri:
zd = oth_tags.copy()
zd.update(ret)
args["sin"] = json.dumps(zd).encode("utf-8", "replace")
cmd = [sys.executable, mp.bin, abspath]
args = {"env": env, "timeout": mp.timeout}
if WINDOWS:
args["creationflags"] = 0x4000
else:
cmd = ["nice"] + cmd
bcmd = [fsenc(x) for x in cmd]
rc, v, err = runcmd(bcmd, **args) # type: ignore
retchk(rc, bcmd, err, self.log, 5, self.args.mtag_v)
v = v.strip()
cmd = [fsenc(x) for x in cmd]
v = sp.check_output(cmd, **args).strip()
if not v:
continue
if "," not in tagname:
ret[tagname] = v
ret[tagname] = v.decode("utf-8")
else:
zj = json.loads(v)
v = json.loads(v)
for tag in tagname.split(","):
if tag and tag in zj:
ret[tag] = zj[tag]
if tag and tag in v:
ret[tag] = v[tag]
except:
if self.args.mtag_v:
t = "mtag error: tagname {}, parser {}, file {} => {}"
self.log(t.format(tagname, parser.bin, abspath, min_ex()))
pass
return ret

View File

@@ -1,370 +0,0 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import socket
import time
import ipaddress
from ipaddress import (
IPv4Address,
IPv4Network,
IPv6Address,
IPv6Network,
ip_address,
ip_network,
)
from .__init__ import MACOS, TYPE_CHECKING
from .util import Netdev, find_prefix, min_ex, spack
if TYPE_CHECKING:
from .svchub import SvcHub
if True: # pylint: disable=using-constant-test
from typing import Optional, Union
if not hasattr(socket, "IPPROTO_IPV6"):
setattr(socket, "IPPROTO_IPV6", 41)
class NoIPs(Exception):
pass
class MC_Sck(object):
"""there is one socket for each server ip"""
def __init__(
self,
sck: socket.socket,
nd: Netdev,
grp: str,
ip: str,
net: Union[IPv4Network, IPv6Network],
):
self.sck = sck
self.idx = nd.idx
self.name = nd.name
self.grp = grp
self.mreq = b""
self.ip = ip
self.net = net
self.ips = {ip: net}
self.v6 = ":" in ip
self.have4 = ":" not in ip
self.have6 = ":" in ip
class MCast(object):
def __init__(
self,
hub: "SvcHub",
Srv: type[MC_Sck],
on: list[str],
off: list[str],
mc_grp_4: str,
mc_grp_6: str,
port: int,
vinit: bool,
) -> None:
"""disable ipv%d by setting mc_grp_%d empty"""
self.hub = hub
self.Srv = Srv
self.args = hub.args
self.asrv = hub.asrv
self.log_func = hub.log
self.on = on
self.off = off
self.grp4 = mc_grp_4
self.grp6 = mc_grp_6
self.port = port
self.vinit = vinit
self.srv: dict[socket.socket, MC_Sck] = {} # listening sockets
self.sips: set[str] = set() # all listening ips (including failed attempts)
self.ll_ok: set[str] = set() # fallback linklocal IPv4 and IPv6 addresses
self.b2srv: dict[bytes, MC_Sck] = {} # binary-ip -> server socket
self.b4: list[bytes] = [] # sorted list of binary-ips
self.b6: list[bytes] = [] # sorted list of binary-ips
self.cscache: dict[str, Optional[MC_Sck]] = {} # client ip -> server cache
self.running = True
def log(self, msg: str, c: Union[int, str] = 0) -> None:
self.log_func("multicast", msg, c)
def create_servers(self) -> list[str]:
bound: list[str] = []
netdevs = self.hub.tcpsrv.netdevs
ips = [x[0] for x in self.hub.tcpsrv.bound]
if "::" in ips:
ips = [x for x in ips if x != "::"] + list(
[x.split("/")[0] for x in netdevs if ":" in x]
)
ips.append("0.0.0.0")
if "0.0.0.0" in ips:
ips = [x for x in ips if x != "0.0.0.0"] + list(
[x.split("/")[0] for x in netdevs if ":" not in x]
)
ips = [x for x in ips if x not in ("::1", "127.0.0.1")]
ips = find_prefix(ips, netdevs)
on = self.on[:]
off = self.off[:]
for lst in (on, off):
for av in list(lst):
try:
arg_net = ip_network(av, False)
except:
arg_net = None
for sk, sv in netdevs.items():
if arg_net:
net_ip = ip_address(sk.split("/")[0])
if net_ip in arg_net and sk not in lst:
lst.append(sk)
if (av == str(sv.idx) or av == sv.name) and sk not in lst:
lst.append(sk)
if on:
ips = [x for x in ips if x in on]
elif off:
ips = [x for x in ips if x not in off]
if not self.grp4:
ips = [x for x in ips if ":" in x]
if not self.grp6:
ips = [x for x in ips if ":" not in x]
ips = list(set(ips))
all_selected = ips[:]
# discard non-linklocal ipv6
ips = [x for x in ips if ":" not in x or x.startswith("fe80")]
if not ips:
raise NoIPs()
for ip in ips:
v6 = ":" in ip
netdev = netdevs[ip]
if not netdev.idx:
t = "using INADDR_ANY for ip [{}], netdev [{}]"
if not self.srv and ip not in ["::", "0.0.0.0"]:
self.log(t.format(ip, netdev), 3)
ipv = socket.AF_INET6 if v6 else socket.AF_INET
sck = socket.socket(ipv, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
sck.settimeout(None)
sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except:
pass
# most ipv6 clients expect multicast on linklocal ip only;
# add a/aaaa records for the other nic IPs
other_ips: set[str] = set()
if v6:
for nd in netdevs.values():
if nd.idx == netdev.idx and nd.ip in all_selected and ":" in nd.ip:
other_ips.add(nd.ip)
net = ipaddress.ip_network(ip, False)
ip = ip.split("/")[0]
srv = self.Srv(sck, netdev, self.grp6 if ":" in ip else self.grp4, ip, net)
for oth_ip in other_ips:
srv.ips[oth_ip.split("/")[0]] = ipaddress.ip_network(oth_ip, False)
# gvfs breaks if a linklocal ip appears in a dns reply
ll = {
k: v
for k, v in srv.ips.items()
if k.startswith("169.254") or k.startswith("fe80")
}
rt = {k: v for k, v in srv.ips.items() if k not in ll}
if self.args.ll or not rt:
self.ll_ok.update(list(ll))
if not self.args.ll:
srv.ips = rt or ll
if not srv.ips:
self.log("no IPs on {}; skipping [{}]".format(netdev, ip), 3)
continue
try:
self.setup_socket(srv)
self.srv[sck] = srv
bound.append(ip)
except:
t = "announce failed on {} [{}]:\n{}"
self.log(t.format(netdev, ip, min_ex()), 3)
if self.args.zm_msub:
for s1 in self.srv.values():
for s2 in self.srv.values():
if s1.idx != s2.idx:
continue
if s1.ip not in s2.ips:
s2.ips[s1.ip] = s1.net
if self.args.zm_mnic:
for s1 in self.srv.values():
for s2 in self.srv.values():
for ip1, net1 in list(s1.ips.items()):
for ip2, net2 in list(s2.ips.items()):
if net1 == net2 and ip1 != ip2:
s1.ips[ip2] = net2
self.sips = set([x.split("/")[0] for x in all_selected])
for srv in self.srv.values():
assert srv.ip in self.sips
return bound
def setup_socket(self, srv: MC_Sck) -> None:
sck = srv.sck
if srv.v6:
if self.vinit:
zsl = list(srv.ips.keys())
self.log("v6({}) idx({}) {}".format(srv.ip, srv.idx, zsl), 6)
for ip in srv.ips:
bip = socket.inet_pton(socket.AF_INET6, ip)
self.b2srv[bip] = srv
self.b6.append(bip)
grp = self.grp6 if srv.idx else ""
try:
if MACOS:
raise Exception()
sck.bind((grp, self.port, 0, srv.idx))
except:
sck.bind(("", self.port, 0, srv.idx))
bgrp = socket.inet_pton(socket.AF_INET6, self.grp6)
dev = spack(b"@I", srv.idx)
srv.mreq = bgrp + dev
if srv.idx != socket.INADDR_ANY:
sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, dev)
try:
sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS, 255)
sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_LOOP, 1)
except:
# macos
t = "failed to set IPv6 TTL/LOOP; announcements may not survive multiple switches/routers"
self.log(t, 3)
else:
if self.vinit:
self.log("v4({}) idx({})".format(srv.ip, srv.idx), 6)
bip = socket.inet_aton(srv.ip)
self.b2srv[bip] = srv
self.b4.append(bip)
grp = self.grp4 if srv.idx else ""
try:
if MACOS:
raise Exception()
sck.bind((grp, self.port))
except:
sck.bind(("", self.port))
bgrp = socket.inet_aton(self.grp4)
dev = (
spack(b"=I", socket.INADDR_ANY)
if srv.idx == socket.INADDR_ANY
else socket.inet_aton(srv.ip)
)
srv.mreq = bgrp + dev
if srv.idx != socket.INADDR_ANY:
sck.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, dev)
try:
sck.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 255)
sck.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, 1)
except:
# probably can't happen but dontcare if it does
t = "failed to set IPv4 TTL/LOOP; announcements may not survive multiple switches/routers"
self.log(t, 3)
self.hop(srv)
self.b4.sort(reverse=True)
self.b6.sort(reverse=True)
def hop(self, srv: MC_Sck) -> None:
"""rejoin to keepalive on routers/switches without igmp-snooping"""
sck = srv.sck
req = srv.mreq
if ":" in srv.ip:
try:
sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_LEAVE_GROUP, req)
# linux does leaves/joins twice with 0.2~1.05s spacing
time.sleep(1.2)
except:
pass
sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, req)
else:
try:
sck.setsockopt(socket.IPPROTO_IP, socket.IP_DROP_MEMBERSHIP, req)
time.sleep(1.2)
except:
pass
# t = "joining {} from ip {} idx {} with mreq {}"
# self.log(t.format(srv.grp, srv.ip, srv.idx, repr(srv.mreq)), 6)
sck.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, req)
def map_client(self, cip: str) -> Optional[MC_Sck]:
try:
return self.cscache[cip]
except:
pass
ret: Optional[MC_Sck] = None
v6 = ":" in cip
ci = IPv6Address(cip) if v6 else IPv4Address(cip)
for x in self.b6 if v6 else self.b4:
srv = self.b2srv[x]
if any([x for x in srv.ips.values() if ci in x]):
ret = srv
break
if not ret and cip in ("127.0.0.1", "::1"):
# just give it something
ret = list(self.srv.values())[0]
if not ret and cip.startswith("169.254"):
# idk how to map LL IPv4 msgs to nics;
# just pick one and hope for the best
lls = (
x
for x in self.srv.values()
if next((y for y in x.ips if y in self.ll_ok), None)
)
ret = next(lls, None)
if ret:
t = "new client on {} ({}): {}"
self.log(t.format(ret.name, ret.net, cip), 6)
else:
t = "could not map client {} to known subnet; maybe forwarded from another network?"
self.log(t.format(cip), 3)
if len(self.cscache) > 9000:
self.cscache = {}
self.cscache[cip] = ret
return ret

View File

@@ -1,321 +0,0 @@
# coding: utf-8
import inspect
import logging
import os
import random
import stat
import sys
import time
from types import SimpleNamespace
from .__init__ import ANYWIN, TYPE_CHECKING
from .authsrv import LEELOO_DALLAS, VFS
from .bos import bos
from .util import Daemon, min_ex
if True: # pylint: disable=using-constant-test
from typing import Any
if TYPE_CHECKING:
from .svchub import SvcHub
lg = logging.getLogger("smb")
debug, info, warning, error = (lg.debug, lg.info, lg.warning, lg.error)
class SMB(object):
def __init__(self, hub: "SvcHub") -> None:
self.hub = hub
self.args = hub.args
self.asrv = hub.asrv
self.log = hub.log
self.files: dict[int, tuple[float, str]] = {}
lg.setLevel(logging.DEBUG if self.args.smbvvv else logging.INFO)
for x in ["impacket", "impacket.smbserver"]:
lgr = logging.getLogger(x)
lgr.setLevel(logging.DEBUG if self.args.smbvv else logging.INFO)
try:
from impacket import smbserver
from impacket.ntlm import compute_lmhash, compute_nthash
except ImportError:
m = "\033[36m\n{}\033[31m\n\nERROR: need 'impacket'; please run this command:\033[33m\n {} -m pip install --user impacket\n\033[0m"
print(m.format(min_ex(), sys.executable))
sys.exit(1)
# patch vfs into smbserver.os
fos = SimpleNamespace()
for k in os.__dict__:
try:
setattr(fos, k, getattr(os, k))
except:
pass
fos.close = self._close
fos.listdir = self._listdir
fos.mkdir = self._mkdir
fos.open = self._open
fos.remove = self._unlink
fos.rename = self._rename
fos.stat = self._stat
fos.unlink = self._unlink
fos.utime = self._utime
smbserver.os = fos
# ...and smbserver.os.path
fop = SimpleNamespace()
for k in os.path.__dict__:
try:
setattr(fop, k, getattr(os.path, k))
except:
pass
fop.exists = self._p_exists
fop.getsize = self._p_getsize
fop.isdir = self._p_isdir
smbserver.os.path = fop
if not self.args.smb_nwa_2:
fop.join = self._p_join
# other patches
smbserver.isInFileJail = self._is_in_file_jail
self._disarm()
ip = next((x for x in self.args.i if ":" not in x), None)
if not ip:
self.log("smb", "IPv6 not supported for SMB; listening on 0.0.0.0", 3)
ip = "0.0.0.0"
port = int(self.args.smb_port)
srv = smbserver.SimpleSMBServer(listenAddress=ip, listenPort=port)
ro = "no" if self.args.smbw else "yes" # (does nothing)
srv.addShare("A", "/", readOnly=ro)
srv.setSMB2Support(not self.args.smb1)
for name, pwd in self.asrv.acct.items():
for u, p in ((name, pwd), (pwd, "k")):
lmhash = compute_lmhash(p)
nthash = compute_nthash(p)
srv.addCredential(u, 0, lmhash, nthash)
chi = [random.randint(0, 255) for x in range(8)]
cha = "".join(["{:02x}".format(x) for x in chi])
srv.setSMBChallenge(cha)
self.srv = srv
self.stop = srv.stop
self.log("smb", "listening @ {}:{}".format(ip, port))
def start(self) -> None:
Daemon(self.srv.start)
def _v2a(self, caller: str, vpath: str, *a: Any) -> tuple[VFS, str]:
vpath = vpath.replace("\\", "/").lstrip("/")
# cf = inspect.currentframe().f_back
# c1 = cf.f_back.f_code.co_name
# c2 = cf.f_code.co_name
debug('%s("%s", %s)\033[K\033[0m', caller, vpath, str(a))
# TODO find a way to grab `identity` in smbComSessionSetupAndX and smb2SessionSetup
vfs, rem = self.asrv.vfs.get(vpath, LEELOO_DALLAS, True, True)
return vfs, vfs.canonical(rem)
def _listdir(self, vpath: str, *a: Any, **ka: Any) -> list[str]:
vpath = vpath.replace("\\", "/").lstrip("/")
# caller = inspect.currentframe().f_back.f_code.co_name
debug('listdir("%s", %s)\033[K\033[0m', vpath, str(a))
vfs, rem = self.asrv.vfs.get(vpath, LEELOO_DALLAS, False, False)
_, vfs_ls, vfs_virt = vfs.ls(
rem, LEELOO_DALLAS, not self.args.no_scandir, [[False, False]]
)
dirs = [x[0] for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
fils = [x[0] for x in vfs_ls if x[0] not in dirs]
ls = list(vfs_virt.keys()) + dirs + fils
if self.args.smb_nwa_1:
return ls
# clients crash somewhere around 65760 byte
ret = []
sz = 112 * 2 # ['.', '..']
for n, fn in enumerate(ls):
if sz >= 64000:
t = "listing only %d of %d files (%d byte); see impacket#1433"
warning(t, n, len(ls), sz)
break
nsz = len(fn.encode("utf-16", "replace"))
nsz = ((nsz + 7) // 8) * 8
sz += 104 + nsz
ret.append(fn)
return ret
def _open(
self, vpath: str, flags: int, *a: Any, chmod: int = 0o777, **ka: Any
) -> Any:
f_ro = os.O_RDONLY
if ANYWIN:
f_ro |= os.O_BINARY
wr = flags != f_ro
if wr and not self.args.smbw:
yeet("blocked write (no --smbw): " + vpath)
vfs, ap = self._v2a("open", vpath, *a)
if wr and not vfs.axs.uwrite:
yeet("blocked write (no-write-acc): " + vpath)
ret = bos.open(ap, flags, *a, mode=chmod, **ka)
if wr:
now = time.time()
nf = len(self.files)
if nf > 9000:
oldest = min([x[0] for x in self.files.values()])
cutoff = oldest + (now - oldest) / 2
self.files = {k: v for k, v in self.files.items() if v[0] > cutoff}
info("was tracking %d files, now %d", nf, len(self.files))
vpath = vpath.replace("\\", "/").lstrip("/")
self.files[ret] = (now, vpath)
return ret
def _close(self, fd: int) -> None:
os.close(fd)
if fd not in self.files:
return
_, vp = self.files.pop(fd)
vp, fn = os.path.split(vp)
vfs, rem = self.hub.asrv.vfs.get(vp, LEELOO_DALLAS, False, True)
vfs, rem = vfs.get_dbv(rem)
self.hub.up2k.hash_file(
vfs.realpath,
vfs.flags,
rem,
fn,
"1.7.6.2",
time.time(),
)
def _rename(self, vp1: str, vp2: str) -> None:
if not self.args.smbw:
yeet("blocked rename (no --smbw): " + vp1)
vp1 = vp1.lstrip("/")
vp2 = vp2.lstrip("/")
vfs2, ap2 = self._v2a("rename", vp2, vp1)
if not vfs2.axs.uwrite:
yeet("blocked rename (no-write-acc): " + vp2)
vfs1, _ = self.asrv.vfs.get(vp1, LEELOO_DALLAS, True, True)
if not vfs1.axs.umove:
yeet("blocked rename (no-move-acc): " + vp1)
self.hub.up2k.handle_mv(LEELOO_DALLAS, vp1, vp2)
try:
bos.makedirs(ap2)
except:
pass
def _mkdir(self, vpath: str) -> None:
if not self.args.smbw:
yeet("blocked mkdir (no --smbw): " + vpath)
vfs, ap = self._v2a("mkdir", vpath)
if not vfs.axs.uwrite:
yeet("blocked mkdir (no-write-acc): " + vpath)
return bos.mkdir(ap)
def _stat(self, vpath: str, *a: Any, **ka: Any) -> os.stat_result:
return bos.stat(self._v2a("stat", vpath, *a)[1], *a, **ka)
def _unlink(self, vpath: str) -> None:
if not self.args.smbw:
yeet("blocked delete (no --smbw): " + vpath)
# return bos.unlink(self._v2a("stat", vpath, *a)[1])
vfs, ap = self._v2a("delete", vpath)
if not vfs.axs.udel:
yeet("blocked delete (no-del-acc): " + vpath)
vpath = vpath.replace("\\", "/").lstrip("/")
self.hub.up2k.handle_rm(LEELOO_DALLAS, "1.7.6.2", [vpath], [])
def _utime(self, vpath: str, times: tuple[float, float]) -> None:
if not self.args.smbw:
yeet("blocked utime (no --smbw): " + vpath)
vfs, ap = self._v2a("utime", vpath)
if not vfs.axs.uwrite:
yeet("blocked utime (no-write-acc): " + vpath)
return bos.utime(ap, times)
def _p_exists(self, vpath: str) -> bool:
try:
bos.stat(self._v2a("p.exists", vpath)[1])
return True
except:
return False
def _p_getsize(self, vpath: str) -> int:
st = bos.stat(self._v2a("p.getsize", vpath)[1])
return st.st_size
def _p_isdir(self, vpath: str) -> bool:
try:
st = bos.stat(self._v2a("p.isdir", vpath)[1])
return stat.S_ISDIR(st.st_mode)
except:
return False
def _p_join(self, *a) -> str:
# impacket.smbserver reads globs from queryDirectoryRequest['Buffer']
# where somehow `fds.*` becomes `fds"*` so lets fix that
ret = os.path.join(*a)
return ret.replace('"', ".") # type: ignore
def _hook(self, *a: Any, **ka: Any) -> None:
src = inspect.currentframe().f_back.f_code.co_name
error("\033[31m%s:hook(%s)\033[0m", src, a)
raise Exception("nope")
def _disarm(self) -> None:
from impacket import smbserver
smbserver.os.chmod = self._hook
smbserver.os.chown = self._hook
smbserver.os.ftruncate = self._hook
smbserver.os.lchown = self._hook
smbserver.os.link = self._hook
smbserver.os.lstat = self._hook
smbserver.os.replace = self._hook
smbserver.os.scandir = self._hook
smbserver.os.symlink = self._hook
smbserver.os.truncate = self._hook
smbserver.os.walk = self._hook
smbserver.os.path.abspath = self._hook
smbserver.os.path.expanduser = self._hook
smbserver.os.path.getatime = self._hook
smbserver.os.path.getctime = self._hook
smbserver.os.path.getmtime = self._hook
smbserver.os.path.isabs = self._hook
smbserver.os.path.isfile = self._hook
smbserver.os.path.islink = self._hook
smbserver.os.path.realpath = self._hook
def _is_in_file_jail(self, *a: Any) -> bool:
# handled by vfs
return True
def yeet(msg: str) -> None:
info(msg)
raise Exception(msg)

View File

@@ -1,210 +0,0 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import re
import select
import socket
from email.utils import formatdate
from .__init__ import TYPE_CHECKING
from .multicast import MC_Sck, MCast
from .util import CachedSet, html_escape, min_ex
if TYPE_CHECKING:
from .broker_util import BrokerCli
from .httpcli import HttpCli
from .svchub import SvcHub
if True: # pylint: disable=using-constant-test
from typing import Optional, Union
GRP = "239.255.255.250"
class SSDP_Sck(MC_Sck):
def __init__(self, *a):
super(SSDP_Sck, self).__init__(*a)
self.hport = 0
class SSDPr(object):
"""generates http responses for httpcli"""
def __init__(self, broker: "BrokerCli") -> None:
self.broker = broker
self.args = broker.args
def reply(self, hc: "HttpCli") -> bool:
if hc.vpath.endswith("device.xml"):
return self.tx_device(hc)
hc.reply(b"unknown request", 400)
return False
def tx_device(self, hc: "HttpCli") -> bool:
zs = """
<?xml version="1.0"?>
<root xmlns="urn:schemas-upnp-org:device-1-0">
<specVersion>
<major>1</major>
<minor>0</minor>
</specVersion>
<URLBase>{}</URLBase>
<device>
<presentationURL>{}</presentationURL>
<deviceType>urn:schemas-upnp-org:device:Basic:1</deviceType>
<friendlyName>{}</friendlyName>
<modelDescription>file server</modelDescription>
<manufacturer>ed</manufacturer>
<manufacturerURL>https://ocv.me/</manufacturerURL>
<modelName>copyparty</modelName>
<modelURL>https://github.com/9001/copyparty/</modelURL>
<UDN>{}</UDN>
<serviceList>
<service>
<serviceType>urn:schemas-upnp-org:device:Basic:1</serviceType>
<serviceId>urn:schemas-upnp-org:device:Basic</serviceId>
<controlURL>/.cpr/ssdp/services.xml</controlURL>
<eventSubURL>/.cpr/ssdp/services.xml</eventSubURL>
<SCPDURL>/.cpr/ssdp/services.xml</SCPDURL>
</service>
</serviceList>
</device>
</root>"""
c = html_escape
sip, sport = hc.s.getsockname()[:2]
sip = sip.replace("::ffff:", "")
proto = "https" if self.args.https_only else "http"
ubase = "{}://{}:{}".format(proto, sip, sport)
zsl = self.args.zsl
url = zsl if "://" in zsl else ubase + "/" + zsl.lstrip("/")
name = "{} @ {}".format(self.args.doctitle, self.args.name)
zs = zs.strip().format(c(ubase), c(url), c(name), c(self.args.zsid))
hc.reply(zs.encode("utf-8", "replace"))
return False # close connectino
class SSDPd(MCast):
"""communicates with ssdp clients over multicast"""
def __init__(self, hub: "SvcHub", ngen: int) -> None:
al = hub.args
vinit = al.zsv and not al.zmv
super(SSDPd, self).__init__(
hub, SSDP_Sck, al.zs_on, al.zs_off, GRP, "", 1900, vinit
)
self.srv: dict[socket.socket, SSDP_Sck] = {}
self.logsrc = "SSDP-{}".format(ngen)
self.ngen = ngen
self.rxc = CachedSet(0.7)
self.txc = CachedSet(5) # win10: every 3 sec
self.ptn_st = re.compile(b"\nst: *upnp:rootdevice", re.I)
def log(self, msg: str, c: Union[int, str] = 0) -> None:
self.log_func(self.logsrc, msg, c)
def run(self) -> None:
try:
bound = self.create_servers()
except:
t = "no server IP matches the ssdp config\n{}"
self.log(t.format(min_ex()), 1)
bound = []
if not bound:
self.log("failed to announce copyparty services on the network", 3)
return
# find http port for this listening ip
for srv in self.srv.values():
tcps = self.hub.tcpsrv.bound
hp = next((x[1] for x in tcps if x[0] in ("0.0.0.0", srv.ip)), 0)
hp = hp or next((x[1] for x in tcps if x[0] == "::"), 0)
if not hp:
hp = tcps[0][1]
self.log("assuming port {} for {}".format(hp, srv.ip), 3)
srv.hport = hp
self.log("listening")
while self.running:
rdy = select.select(self.srv, [], [], self.args.z_chk or 180)
rx: list[socket.socket] = rdy[0] # type: ignore
self.rxc.cln()
buf = b""
addr = ("0", 0)
for sck in rx:
try:
buf, addr = sck.recvfrom(4096)
self.eat(buf, addr)
except:
if not self.running:
break
t = "{} {} \033[33m|{}| {}\n{}".format(
self.srv[sck].name, addr, len(buf), repr(buf)[2:-1], min_ex()
)
self.log(t, 6)
self.log("stopped", 2)
def stop(self) -> None:
self.running = False
for srv in self.srv.values():
try:
srv.sck.close()
except:
pass
self.srv = {}
def eat(self, buf: bytes, addr: tuple[str, int]) -> None:
cip = addr[0]
if cip.startswith("169.254") and not self.ll_ok:
return
if buf in self.rxc.c:
return
srv: Optional[SSDP_Sck] = self.map_client(cip) # type: ignore
if not srv:
return
self.rxc.add(buf)
if not buf.startswith(b"M-SEARCH * HTTP/1."):
return
if not self.ptn_st.search(buf):
return
if self.args.zsv:
t = "{} [{}] \033[36m{} \033[0m|{}|"
self.log(t.format(srv.name, srv.ip, cip, len(buf)), "90")
zs = """
HTTP/1.1 200 OK
CACHE-CONTROL: max-age=1800
DATE: {0}
EXT:
LOCATION: http://{1}:{2}/.cpr/ssdp/device.xml
OPT: "http://schemas.upnp.org/upnp/1/0/"; ns=01
01-NLS: {3}
SERVER: UPnP/1.0
ST: upnp:rootdevice
USN: {3}::upnp:rootdevice
BOOTID.UPNP.ORG: 0
CONFIGID.UPNP.ORG: 1
"""
v4 = srv.ip.replace("::ffff:", "")
zs = zs.format(formatdate(usegmt=True), v4, srv.hport, self.args.zsid)
zb = zs[1:].replace("\n", "\r\n").encode("utf-8", "replace")
srv.sck.sendto(zb, addr[:2])
if cip not in self.txc.c:
self.log("{} [{}] --> {}".format(srv.name, srv.ip, cip), "6")
self.txc.add(cip)
self.txc.cln()

View File

@@ -1,30 +1,23 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import stat
import os
import tarfile
import threading
from queue import Queue
from .bos import bos
from .sutil import StreamArc, errdesc
from .util import Daemon, fsenc, min_ex
if True: # pylint: disable=using-constant-test
from typing import Any, Generator, Optional
from .util import NamedLogger
from .sutil import errdesc
from .util import Queue, fsenc
class QFile(object): # inherit io.StringIO for painful typing
class QFile(object):
"""file-like object which buffers writes into a queue"""
def __init__(self) -> None:
self.q: Queue[Optional[bytes]] = Queue(64)
self.bq: list[bytes] = []
def __init__(self):
self.q = Queue(64)
self.bq = []
self.nq = 0
def write(self, buf: Optional[bytes]) -> None:
def write(self, buf):
if buf is None or self.nq >= 240 * 1024:
self.q.put(b"".join(self.bq))
self.bq = []
@@ -37,52 +30,43 @@ class QFile(object): # inherit io.StringIO for painful typing
self.nq += len(buf)
class StreamTar(StreamArc):
class StreamTar(object):
"""construct in-memory tar file from the given path"""
def __init__(
self,
log: "NamedLogger",
fgen: Generator[dict[str, Any], None, None],
**kwargs: Any
):
super(StreamTar, self).__init__(log, fgen)
def __init__(self, fgen, **kwargs):
self.ci = 0
self.co = 0
self.qfile = QFile()
self.errf: dict[str, Any] = {}
self.fgen = fgen
self.errf = None
# python 3.8 changed to PAX_FORMAT as default,
# waste of space and don't care about the new features
fmt = tarfile.GNU_FORMAT
self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt) # type: ignore
self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt)
Daemon(self._gen, "star-gen")
w = threading.Thread(target=self._gen, name="star-gen")
w.daemon = True
w.start()
def gen(self) -> Generator[Optional[bytes], None, None]:
try:
while True:
buf = self.qfile.q.get()
if not buf:
break
def gen(self):
while True:
buf = self.qfile.q.get()
if not buf:
break
self.co += len(buf)
yield buf
self.co += len(buf)
yield buf
yield None
finally:
if self.errf:
bos.unlink(self.errf["ap"])
yield None
if self.errf:
os.unlink(self.errf["ap"])
def ser(self, f: dict[str, Any]) -> None:
def ser(self, f):
name = f["vp"]
src = f["ap"]
fsi = f["st"]
if stat.S_ISDIR(fsi.st_mode):
return
inf = tarfile.TarInfo(name=name)
inf.mode = fsi.st_mode
inf.size = fsi.st_size
@@ -91,25 +75,23 @@ class StreamTar(StreamArc):
inf.gid = 0
self.ci += inf.size
with open(fsenc(src), "rb", 512 * 1024) as fo:
self.tar.addfile(inf, fo)
with open(fsenc(src), "rb", 512 * 1024) as f:
self.tar.addfile(inf, f)
def _gen(self) -> None:
def _gen(self):
errors = []
for f in self.fgen:
if "err" in f:
errors.append((f["vp"], f["err"]))
errors.append([f["vp"], f["err"]])
continue
try:
self.ser(f)
except:
ex = min_ex(5, True).replace("\n", "\n-- ")
errors.append((f["vp"], ex))
except Exception as ex:
errors.append([f["vp"], repr(ex)])
if errors:
self.errf, txt = errdesc(errors)
self.log("\n".join(([repr(self.errf)] + txt[1:])))
self.errf = errdesc(errors)
self.ser(self.errf)
self.tar.close()

View File

@@ -1,5 +0,0 @@
`dnslib` but heavily simplified/feature-stripped
L: MIT
Copyright (c) 2010 - 2017 Paul Chakravarti
https://github.com/paulc/dnslib/

View File

@@ -1,11 +0,0 @@
# coding: utf-8
"""
L: MIT
Copyright (c) 2010 - 2017 Paul Chakravarti
https://github.com/paulc/dnslib/tree/0.9.23
"""
from .dns import *
version = "0.9.23"

View File

@@ -1,41 +0,0 @@
# coding: utf-8
import types
class BimapError(Exception):
pass
class Bimap(object):
def __init__(self, name, forward, error=AttributeError):
self.name = name
self.error = error
self.forward = forward.copy()
self.reverse = dict([(v, k) for (k, v) in list(forward.items())])
def get(self, k, default=None):
try:
return self.forward[k]
except KeyError:
return default or str(k)
def __getitem__(self, k):
try:
return self.forward[k]
except KeyError:
if isinstance(self.error, types.FunctionType):
return self.error(self.name, k, True)
else:
raise self.error("%s: Invalid forward lookup: [%s]" % (self.name, k))
def __getattr__(self, k):
try:
if k == "__wrapped__":
raise AttributeError()
return self.reverse[k]
except KeyError:
if isinstance(self.error, types.FunctionType):
return self.error(self.name, k, False)
else:
raise self.error("%s: Invalid reverse lookup: [%s]" % (self.name, k))

View File

@@ -1,15 +0,0 @@
# coding: utf-8
from __future__ import print_function
def get_bits(data, offset, bits=1):
mask = ((1 << bits) - 1) << offset
return (data & mask) >> offset
def set_bits(data, value, offset, bits=1):
mask = ((1 << bits) - 1) << offset
clear = 0xFFFF ^ mask
data = (data & clear) | ((value << offset) & mask)
return data

View File

@@ -1,56 +0,0 @@
# coding: utf-8
import binascii
import struct
class BufferError(Exception):
pass
class Buffer(object):
def __init__(self, data=b""):
self.data = bytearray(data)
self.offset = 0
def remaining(self):
return len(self.data) - self.offset
def get(self, length):
if length > self.remaining():
raise BufferError(
"Not enough bytes [offset=%d,remaining=%d,requested=%d]"
% (self.offset, self.remaining(), length)
)
start = self.offset
end = self.offset + length
self.offset += length
return bytes(self.data[start:end])
def hex(self):
return binascii.hexlify(self.data)
def pack(self, fmt, *args):
self.offset += struct.calcsize(fmt)
self.data += struct.pack(fmt, *args)
def append(self, s):
self.offset += len(s)
self.data += s
def update(self, ptr, fmt, *args):
s = struct.pack(fmt, *args)
self.data[ptr : ptr + len(s)] = s
def unpack(self, fmt):
try:
data = self.get(struct.calcsize(fmt))
return struct.unpack(fmt, data)
except struct.error:
raise BufferError(
"Error unpacking struct '%s' <%s>"
% (fmt, binascii.hexlify(data).decode())
)
def __len__(self):
return len(self.data)

View File

@@ -1,775 +0,0 @@
# coding: utf-8
from __future__ import print_function
import binascii
from itertools import chain
from .bimap import Bimap, BimapError
from .bit import get_bits, set_bits
from .buffer import BufferError
from .label import DNSBuffer, DNSLabel
from .ranges import IP4, IP6, H, I, check_bytes
class DNSError(Exception):
pass
def unknown_qtype(name, key, forward):
if forward:
try:
return "TYPE%d" % (key,)
except:
raise DNSError("%s: Invalid forward lookup: [%s]" % (name, key))
else:
if key.startswith("TYPE"):
try:
return int(key[4:])
except:
pass
raise DNSError("%s: Invalid reverse lookup: [%s]" % (name, key))
QTYPE = Bimap(
"QTYPE",
{1: "A", 12: "PTR", 16: "TXT", 28: "AAAA", 33: "SRV", 47: "NSEC", 255: "ANY"},
unknown_qtype,
)
CLASS = Bimap("CLASS", {1: "IN", 254: "None", 255: "*", 0x8001: "F_IN"}, DNSError)
QR = Bimap("QR", {0: "QUERY", 1: "RESPONSE"}, DNSError)
RCODE = Bimap(
"RCODE",
{
0: "NOERROR",
1: "FORMERR",
2: "SERVFAIL",
3: "NXDOMAIN",
4: "NOTIMP",
5: "REFUSED",
6: "YXDOMAIN",
7: "YXRRSET",
8: "NXRRSET",
9: "NOTAUTH",
10: "NOTZONE",
},
DNSError,
)
OPCODE = Bimap(
"OPCODE", {0: "QUERY", 1: "IQUERY", 2: "STATUS", 4: "NOTIFY", 5: "UPDATE"}, DNSError
)
def label(label, origin=None):
if label.endswith("."):
return DNSLabel(label)
else:
return (origin if isinstance(origin, DNSLabel) else DNSLabel(origin)).add(label)
class DNSRecord(object):
@classmethod
def parse(cls, packet) -> "DNSRecord":
buffer = DNSBuffer(packet)
try:
header = DNSHeader.parse(buffer)
questions = []
rr = []
auth = []
ar = []
for i in range(header.q):
questions.append(DNSQuestion.parse(buffer))
for i in range(header.a):
rr.append(RR.parse(buffer))
for i in range(header.auth):
auth.append(RR.parse(buffer))
for i in range(header.ar):
ar.append(RR.parse(buffer))
return cls(header, questions, rr, auth=auth, ar=ar)
except (BufferError, BimapError) as e:
raise DNSError(
"Error unpacking DNSRecord [offset=%d]: %s" % (buffer.offset, e)
)
@classmethod
def question(cls, qname, qtype="A", qclass="IN"):
return DNSRecord(
q=DNSQuestion(qname, getattr(QTYPE, qtype), getattr(CLASS, qclass))
)
def __init__(
self, header=None, questions=None, rr=None, q=None, a=None, auth=None, ar=None
) -> None:
self.header = header or DNSHeader()
self.questions: list[DNSQuestion] = questions or []
self.rr: list[RR] = rr or []
self.auth: list[RR] = auth or []
self.ar: list[RR] = ar or []
if q:
self.questions.append(q)
if a:
self.rr.append(a)
self.set_header_qa()
def reply(self, ra=1, aa=1):
return DNSRecord(
DNSHeader(id=self.header.id, bitmap=self.header.bitmap, qr=1, ra=ra, aa=aa),
q=self.q,
)
def add_question(self, *q) -> None:
self.questions.extend(q)
self.set_header_qa()
def add_answer(self, *rr) -> None:
self.rr.extend(rr)
self.set_header_qa()
def add_auth(self, *auth) -> None:
self.auth.extend(auth)
self.set_header_qa()
def add_ar(self, *ar) -> None:
self.ar.extend(ar)
self.set_header_qa()
def set_header_qa(self) -> None:
self.header.q = len(self.questions)
self.header.a = len(self.rr)
self.header.auth = len(self.auth)
self.header.ar = len(self.ar)
def get_q(self):
return self.questions[0] if self.questions else DNSQuestion()
q = property(get_q)
def get_a(self):
return self.rr[0] if self.rr else RR()
a = property(get_a)
def pack(self) -> bytes:
self.set_header_qa()
buffer = DNSBuffer()
self.header.pack(buffer)
for q in self.questions:
q.pack(buffer)
for rr in self.rr:
rr.pack(buffer)
for auth in self.auth:
auth.pack(buffer)
for ar in self.ar:
ar.pack(buffer)
return buffer.data
def truncate(self):
return DNSRecord(DNSHeader(id=self.header.id, bitmap=self.header.bitmap, tc=1))
def format(self, prefix="", sort=False):
s = sorted if sort else lambda x: x
sections = [repr(self.header)]
sections.extend(s([repr(q) for q in self.questions]))
sections.extend(s([repr(rr) for rr in self.rr]))
sections.extend(s([repr(rr) for rr in self.auth]))
sections.extend(s([repr(rr) for rr in self.ar]))
return prefix + ("\n" + prefix).join(sections)
short = format
def __repr__(self):
return self.format()
__str__ = __repr__
class DNSHeader(object):
id = H("id")
bitmap = H("bitmap")
q = H("q")
a = H("a")
auth = H("auth")
ar = H("ar")
@classmethod
def parse(cls, buffer):
try:
(id, bitmap, q, a, auth, ar) = buffer.unpack("!HHHHHH")
return cls(id, bitmap, q, a, auth, ar)
except (BufferError, BimapError) as e:
raise DNSError(
"Error unpacking DNSHeader [offset=%d]: %s" % (buffer.offset, e)
)
def __init__(self, id=None, bitmap=None, q=0, a=0, auth=0, ar=0, **args) -> None:
self.id = id if id else 0
if bitmap is None:
self.bitmap = 0
else:
self.bitmap = bitmap
self.q = q
self.a = a
self.auth = auth
self.ar = ar
for k, v in args.items():
if k.lower() == "qr":
self.qr = v
elif k.lower() == "opcode":
self.opcode = v
elif k.lower() == "aa":
self.aa = v
elif k.lower() == "tc":
self.tc = v
elif k.lower() == "rd":
self.rd = v
elif k.lower() == "ra":
self.ra = v
elif k.lower() == "z":
self.z = v
elif k.lower() == "ad":
self.ad = v
elif k.lower() == "cd":
self.cd = v
elif k.lower() == "rcode":
self.rcode = v
def get_qr(self):
return get_bits(self.bitmap, 15)
def set_qr(self, val):
self.bitmap = set_bits(self.bitmap, val, 15)
qr = property(get_qr, set_qr)
def get_opcode(self):
return get_bits(self.bitmap, 11, 4)
def set_opcode(self, val):
self.bitmap = set_bits(self.bitmap, val, 11, 4)
opcode = property(get_opcode, set_opcode)
def get_aa(self):
return get_bits(self.bitmap, 10)
def set_aa(self, val):
self.bitmap = set_bits(self.bitmap, val, 10)
aa = property(get_aa, set_aa)
def get_tc(self):
return get_bits(self.bitmap, 9)
def set_tc(self, val):
self.bitmap = set_bits(self.bitmap, val, 9)
tc = property(get_tc, set_tc)
def get_rd(self):
return get_bits(self.bitmap, 8)
def set_rd(self, val):
self.bitmap = set_bits(self.bitmap, val, 8)
rd = property(get_rd, set_rd)
def get_ra(self):
return get_bits(self.bitmap, 7)
def set_ra(self, val):
self.bitmap = set_bits(self.bitmap, val, 7)
ra = property(get_ra, set_ra)
def get_z(self):
return get_bits(self.bitmap, 6)
def set_z(self, val):
self.bitmap = set_bits(self.bitmap, val, 6)
z = property(get_z, set_z)
def get_ad(self):
return get_bits(self.bitmap, 5)
def set_ad(self, val):
self.bitmap = set_bits(self.bitmap, val, 5)
ad = property(get_ad, set_ad)
def get_cd(self):
return get_bits(self.bitmap, 4)
def set_cd(self, val):
self.bitmap = set_bits(self.bitmap, val, 4)
cd = property(get_cd, set_cd)
def get_rcode(self):
return get_bits(self.bitmap, 0, 4)
def set_rcode(self, val):
self.bitmap = set_bits(self.bitmap, val, 0, 4)
rcode = property(get_rcode, set_rcode)
def pack(self, buffer):
buffer.pack("!HHHHHH", self.id, self.bitmap, self.q, self.a, self.auth, self.ar)
def __repr__(self):
f = [
self.aa and "AA",
self.tc and "TC",
self.rd and "RD",
self.ra and "RA",
self.z and "Z",
self.ad and "AD",
self.cd and "CD",
]
if OPCODE.get(self.opcode) == "UPDATE":
f1 = "zo"
f2 = "pr"
f3 = "up"
f4 = "ad"
else:
f1 = "q"
f2 = "a"
f3 = "ns"
f4 = "ar"
return (
"<DNS Header: id=0x%x type=%s opcode=%s flags=%s "
"rcode='%s' %s=%d %s=%d %s=%d %s=%d>"
% (
self.id,
QR.get(self.qr),
OPCODE.get(self.opcode),
",".join(filter(None, f)),
RCODE.get(self.rcode),
f1,
self.q,
f2,
self.a,
f3,
self.auth,
f4,
self.ar,
)
)
__str__ = __repr__
class DNSQuestion(object):
@classmethod
def parse(cls, buffer):
try:
qname = buffer.decode_name()
qtype, qclass = buffer.unpack("!HH")
return cls(qname, qtype, qclass)
except (BufferError, BimapError) as e:
raise DNSError(
"Error unpacking DNSQuestion [offset=%d]: %s" % (buffer.offset, e)
)
def __init__(self, qname=None, qtype=1, qclass=1) -> None:
self.qname = qname
self.qtype = qtype
self.qclass = qclass
def set_qname(self, qname):
if isinstance(qname, DNSLabel):
self._qname = qname
else:
self._qname = DNSLabel(qname)
def get_qname(self):
return self._qname
qname = property(get_qname, set_qname)
def pack(self, buffer):
buffer.encode_name(self.qname)
buffer.pack("!HH", self.qtype, self.qclass)
def __repr__(self):
return "<DNS Question: '%s' qtype=%s qclass=%s>" % (
self.qname,
QTYPE.get(self.qtype),
CLASS.get(self.qclass),
)
__str__ = __repr__
class RR(object):
rtype = H("rtype")
rclass = H("rclass")
ttl = I("ttl")
rdlength = H("rdlength")
@classmethod
def parse(cls, buffer):
try:
rname = buffer.decode_name()
rtype, rclass, ttl, rdlength = buffer.unpack("!HHIH")
if rdlength:
rdata = RDMAP.get(QTYPE.get(rtype), RD).parse(buffer, rdlength)
else:
rdata = ""
return cls(rname, rtype, rclass, ttl, rdata)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking RR [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, rname=None, rtype=1, rclass=1, ttl=0, rdata=None) -> None:
self.rname = rname
self.rtype = rtype
self.rclass = rclass
self.ttl = ttl
self.rdata = rdata
def set_rname(self, rname):
if isinstance(rname, DNSLabel):
self._rname = rname
else:
self._rname = DNSLabel(rname)
def get_rname(self):
return self._rname
rname = property(get_rname, set_rname)
def pack(self, buffer):
buffer.encode_name(self.rname)
buffer.pack("!HHI", self.rtype, self.rclass, self.ttl)
rdlength_ptr = buffer.offset
buffer.pack("!H", 0)
start = buffer.offset
self.rdata.pack(buffer)
end = buffer.offset
buffer.update(rdlength_ptr, "!H", end - start)
def __repr__(self):
return "<DNS RR: '%s' rtype=%s rclass=%s ttl=%d rdata='%s'>" % (
self.rname,
QTYPE.get(self.rtype),
CLASS.get(self.rclass),
self.ttl,
self.rdata,
)
__str__ = __repr__
class RD(object):
@classmethod
def parse(cls, buffer, length):
try:
data = buffer.get(length)
return cls(data)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking RD [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, data=b"") -> None:
check_bytes("data", data)
self.data = bytes(data)
def pack(self, buffer):
buffer.append(self.data)
def __repr__(self):
if len(self.data) > 0:
return "\\# %d %s" % (
len(self.data),
binascii.hexlify(self.data).decode().upper(),
)
else:
return "\\# 0"
attrs = ("data",)
def _force_bytes(x):
if isinstance(x, bytes):
return x
else:
return x.encode()
class TXT(RD):
@classmethod
def parse(cls, buffer, length):
try:
data = list()
start_bo = buffer.offset
now_length = 0
while buffer.offset < start_bo + length:
(txtlength,) = buffer.unpack("!B")
if now_length + txtlength < length:
now_length += txtlength
data.append(buffer.get(txtlength))
else:
raise DNSError(
"Invalid TXT record: len(%d) > RD len(%d)" % (txtlength, length)
)
return cls(data)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking TXT [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, data) -> None:
if type(data) in (tuple, list):
self.data = [_force_bytes(x) for x in data]
else:
self.data = [_force_bytes(data)]
if any([len(x) > 255 for x in self.data]):
raise DNSError("TXT record too long: %s" % self.data)
def pack(self, buffer):
for ditem in self.data:
if len(ditem) > 255:
raise DNSError("TXT record too long: %s" % ditem)
buffer.pack("!B", len(ditem))
buffer.append(ditem)
def __repr__(self):
return ",".join([repr(x) for x in self.data])
class A(RD):
data = IP4("data")
@classmethod
def parse(cls, buffer, length):
try:
data = buffer.unpack("!BBBB")
return cls(data)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking A [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, data) -> None:
if type(data) in (tuple, list):
self.data = tuple(data)
else:
self.data = tuple(map(int, data.rstrip(".").split(".")))
def pack(self, buffer):
buffer.pack("!BBBB", *self.data)
def __repr__(self):
return "%d.%d.%d.%d" % self.data
def _parse_ipv6(a):
l, _, r = a.partition("::")
l_groups = list(chain(*[divmod(int(x, 16), 256) for x in l.split(":") if x]))
r_groups = list(chain(*[divmod(int(x, 16), 256) for x in r.split(":") if x]))
zeros = [0] * (16 - len(l_groups) - len(r_groups))
return tuple(l_groups + zeros + r_groups)
def _format_ipv6(a):
left = []
right = []
current = "left"
for i in range(0, 16, 2):
group = (a[i] << 8) + a[i + 1]
if current == "left":
if group == 0 and i < 14:
if (a[i + 2] << 8) + a[i + 3] == 0:
current = "right"
else:
left.append("0")
else:
left.append("%x" % group)
else:
if group == 0 and len(right) == 0:
pass
else:
right.append("%x" % group)
if len(left) < 8:
return ":".join(left) + "::" + ":".join(right)
else:
return ":".join(left)
class AAAA(RD):
data = IP6("data")
@classmethod
def parse(cls, buffer, length):
try:
data = buffer.unpack("!16B")
return cls(data)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking AAAA [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, data) -> None:
if type(data) in (tuple, list):
self.data = tuple(data)
else:
self.data = _parse_ipv6(data)
def pack(self, buffer):
buffer.pack("!16B", *self.data)
def __repr__(self):
return _format_ipv6(self.data)
class CNAME(RD):
@classmethod
def parse(cls, buffer, length):
try:
label = buffer.decode_name()
return cls(label)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking CNAME [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, label=None) -> None:
self.label = label
def set_label(self, label):
if isinstance(label, DNSLabel):
self._label = label
else:
self._label = DNSLabel(label)
def get_label(self):
return self._label
label = property(get_label, set_label)
def pack(self, buffer):
buffer.encode_name(self.label)
def __repr__(self):
return "%s" % (self.label)
attrs = ("label",)
class PTR(CNAME):
pass
class SRV(RD):
priority = H("priority")
weight = H("weight")
port = H("port")
@classmethod
def parse(cls, buffer, length):
try:
priority, weight, port = buffer.unpack("!HHH")
target = buffer.decode_name()
return cls(priority, weight, port, target)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking SRV [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, priority=0, weight=0, port=0, target=None) -> None:
self.priority = priority
self.weight = weight
self.port = port
self.target = target
def set_target(self, target):
if isinstance(target, DNSLabel):
self._target = target
else:
self._target = DNSLabel(target)
def get_target(self):
return self._target
target = property(get_target, set_target)
def pack(self, buffer):
buffer.pack("!HHH", self.priority, self.weight, self.port)
buffer.encode_name(self.target)
def __repr__(self):
return "%d %d %d %s" % (self.priority, self.weight, self.port, self.target)
attrs = ("priority", "weight", "port", "target")
def decode_type_bitmap(type_bitmap):
rrlist = []
buf = DNSBuffer(type_bitmap)
while buf.remaining():
winnum, winlen = buf.unpack("BB")
bitmap = bytearray(buf.get(winlen))
for (pos, value) in enumerate(bitmap):
for i in range(8):
if (value << i) & 0x80:
bitpos = (256 * winnum) + (8 * pos) + i
rrlist.append(QTYPE[bitpos])
return rrlist
def encode_type_bitmap(rrlist):
rrlist = sorted([getattr(QTYPE, rr) for rr in rrlist])
buf = DNSBuffer()
curWindow = rrlist[0] // 256
bitmap = bytearray(32)
n = len(rrlist) - 1
for i, rr in enumerate(rrlist):
v = rr - curWindow * 256
bitmap[v // 8] |= 1 << (7 - v % 8)
if i == n or rrlist[i + 1] >= (curWindow + 1) * 256:
while bitmap[-1] == 0:
bitmap = bitmap[:-1]
buf.pack("BB", curWindow, len(bitmap))
buf.append(bitmap)
if i != n:
curWindow = rrlist[i + 1] // 256
bitmap = bytearray(32)
return buf.data
class NSEC(RD):
@classmethod
def parse(cls, buffer, length):
try:
end = buffer.offset + length
name = buffer.decode_name()
rrlist = decode_type_bitmap(buffer.get(end - buffer.offset))
return cls(name, rrlist)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking NSEC [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, label, rrlist) -> None:
self.label = label
self.rrlist = rrlist
def set_label(self, label):
if isinstance(label, DNSLabel):
self._label = label
else:
self._label = DNSLabel(label)
def get_label(self):
return self._label
label = property(get_label, set_label)
def pack(self, buffer):
buffer.encode_name(self.label)
buffer.append(encode_type_bitmap(self.rrlist))
def __repr__(self):
return "%s %s" % (self.label, " ".join(self.rrlist))
attrs = ("label", "rrlist")
RDMAP = {"A": A, "AAAA": AAAA, "TXT": TXT, "PTR": PTR, "SRV": SRV, "NSEC": NSEC}

View File

@@ -1,154 +0,0 @@
# coding: utf-8
from __future__ import print_function
import re
from .bit import get_bits, set_bits
from .buffer import Buffer, BufferError
LDH = set(range(33, 127))
ESCAPE = re.compile(r"\\([0-9][0-9][0-9])")
class DNSLabelError(Exception):
pass
class DNSLabel(object):
def __init__(self, label):
if type(label) == DNSLabel:
self.label = label.label
elif type(label) in (list, tuple):
self.label = tuple(label)
else:
if not label or label in (b".", "."):
self.label = ()
elif type(label) is not bytes:
if type("") != type(b""):
label = ESCAPE.sub(lambda m: chr(int(m[1])), label)
self.label = tuple(label.encode("idna").rstrip(b".").split(b"."))
else:
if type("") == type(b""):
label = ESCAPE.sub(lambda m: chr(int(m.groups()[0])), label)
self.label = tuple(label.rstrip(b".").split(b"."))
def add(self, name):
new = DNSLabel(name)
if self.label:
new.label += self.label
return new
def idna(self):
return ".".join([s.decode("idna") for s in self.label]) + "."
def _decode(self, s):
if set(s).issubset(LDH):
return s.decode()
else:
return "".join([(chr(c) if (c in LDH) else "\\%03d" % c) for c in s])
def __str__(self):
return ".".join([self._decode(bytearray(s)) for s in self.label]) + "."
def __repr__(self):
return "<DNSLabel: '%s'>" % str(self)
def __hash__(self):
return hash(tuple(map(lambda x: x.lower(), self.label)))
def __ne__(self, other):
return not self == other
def __eq__(self, other):
if type(other) != DNSLabel:
return self.__eq__(DNSLabel(other))
else:
return [l.lower() for l in self.label] == [l.lower() for l in other.label]
def __len__(self):
return len(b".".join(self.label))
class DNSBuffer(Buffer):
def __init__(self, data=b""):
super(DNSBuffer, self).__init__(data)
self.names = {}
def decode_name(self, last=-1):
label = []
done = False
while not done:
(length,) = self.unpack("!B")
if get_bits(length, 6, 2) == 3:
self.offset -= 1
pointer = get_bits(self.unpack("!H")[0], 0, 14)
save = self.offset
if last == save:
raise BufferError(
"Recursive pointer in DNSLabel [offset=%d,pointer=%d,length=%d]"
% (self.offset, pointer, len(self.data))
)
if pointer < self.offset:
self.offset = pointer
else:
raise BufferError(
"Invalid pointer in DNSLabel [offset=%d,pointer=%d,length=%d]"
% (self.offset, pointer, len(self.data))
)
label.extend(self.decode_name(save).label)
self.offset = save
done = True
else:
if length > 0:
l = self.get(length)
try:
l.decode()
except UnicodeDecodeError:
raise BufferError("Invalid label <%s>" % l)
label.append(l)
else:
done = True
return DNSLabel(label)
def encode_name(self, name):
if not isinstance(name, DNSLabel):
name = DNSLabel(name)
if len(name) > 253:
raise DNSLabelError("Domain label too long: %r" % name)
name = list(name.label)
while name:
if tuple(name) in self.names:
pointer = self.names[tuple(name)]
pointer = set_bits(pointer, 3, 14, 2)
self.pack("!H", pointer)
return
else:
self.names[tuple(name)] = self.offset
element = name.pop(0)
if len(element) > 63:
raise DNSLabelError("Label component too long: %r" % element)
self.pack("!B", len(element))
self.append(element)
self.append(b"\x00")
def encode_name_nocompress(self, name):
if not isinstance(name, DNSLabel):
name = DNSLabel(name)
if len(name) > 253:
raise DNSLabelError("Domain label too long: %r" % name)
name = list(name.label)
while name:
element = name.pop(0)
if len(element) > 63:
raise DNSLabelError("Label component too long: %r" % element)
self.pack("!B", len(element))
self.append(element)
self.append(b"\x00")

View File

@@ -1,105 +0,0 @@
# coding: utf-8
from __future__ import print_function
import collections
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
class Lexer(object):
escape_chars = "\\"
escape = {"n": "\n", "t": "\t", "r": "\r"}
def __init__(self, f, debug=False):
if hasattr(f, "read"):
self.f = f
elif type(f) == str:
self.f = StringIO(f)
elif type(f) == bytes:
self.f = StringIO(f.decode())
else:
raise ValueError("Invalid input")
self.debug = debug
self.q = collections.deque()
self.state = self.lexStart
self.escaped = False
self.eof = False
def __iter__(self):
return self.parse()
def next_token(self):
if self.debug:
print("STATE", self.state)
(tok, self.state) = self.state()
return tok
def parse(self):
while self.state is not None and not self.eof:
tok = self.next_token()
if tok:
yield tok
def read(self, n=1):
s = ""
while self.q and n > 0:
s += self.q.popleft()
n -= 1
s += self.f.read(n)
if s == "":
self.eof = True
if self.debug:
print("Read: >%s<" % repr(s))
return s
def peek(self, n=1):
s = ""
i = 0
while len(self.q) > i and n > 0:
s += self.q[i]
i += 1
n -= 1
r = self.f.read(n)
if n > 0 and r == "":
self.eof = True
self.q.extend(r)
if self.debug:
print("Peek : >%s<" % repr(s + r))
return s + r
def pushback(self, s):
p = collections.deque(s)
p.extend(self.q)
self.q = p
def readescaped(self):
c = self.read(1)
if c in self.escape_chars:
self.escaped = True
n = self.peek(3)
if n.isdigit():
n = self.read(3)
if self.debug:
print("Escape: >%s<" % n)
return chr(int(n, 8))
elif n[0] in "x":
x = self.read(3)
if self.debug:
print("Escape: >%s<" % x)
return chr(int(x[1:], 16))
else:
c = self.read(1)
if self.debug:
print("Escape: >%s<" % c)
return self.escape.get(c, c)
else:
self.escaped = False
return c
def lexStart(self):
return (None, None)

View File

@@ -1,81 +0,0 @@
# coding: utf-8
import sys
if sys.version_info < (3,):
int_types = (
int,
long,
)
byte_types = (str, bytearray)
else:
int_types = (int,)
byte_types = (bytes, bytearray)
def check_instance(name, val, types):
if not isinstance(val, types):
raise ValueError(
"Attribute '%s' must be instance of %s [%s]" % (name, types, type(val))
)
def check_bytes(name, val):
return check_instance(name, val, byte_types)
def range_property(attr, min, max):
def getter(obj):
return getattr(obj, "_%s" % attr)
def setter(obj, val):
if isinstance(val, int_types) and min <= val <= max:
setattr(obj, "_%s" % attr, val)
else:
raise ValueError(
"Attribute '%s' must be between %d-%d [%s]" % (attr, min, max, val)
)
return property(getter, setter)
def B(attr):
return range_property(attr, 0, 255)
def H(attr):
return range_property(attr, 0, 65535)
def I(attr):
return range_property(attr, 0, 4294967295)
def ntuple_range(attr, n, min, max):
f = lambda x: isinstance(x, int_types) and min <= x <= max
def getter(obj):
return getattr(obj, "_%s" % attr)
def setter(obj, val):
if len(val) != n:
raise ValueError(
"Attribute '%s' must be tuple with %d elements [%s]" % (attr, n, val)
)
if all(map(f, val)):
setattr(obj, "_%s" % attr, val)
else:
raise ValueError(
"Attribute '%s' elements must be between %d-%d [%s]"
% (attr, min, max, val)
)
return property(getter, setter)
def IP4(attr):
return ntuple_range(attr, 4, 0, 255)
def IP6(attr):
return ntuple_range(attr, 16, 0, 255)

View File

@@ -1,5 +0,0 @@
`ifaddr` with py2.7 support enabled by make-sfx.sh which strips py3 hints using strip_hints and removes the `^if True:` blocks
L: BSD-2-Clause
Copyright (c) 2014 Stefan C. Mueller
https://github.com/pydron/ifaddr/

View File

@@ -1,21 +0,0 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
"""
L: BSD-2-Clause
Copyright (c) 2014 Stefan C. Mueller
https://github.com/pydron/ifaddr/tree/0.2.0
"""
import os
from ._shared import IP, Adapter
if os.name == "nt":
from ._win32 import get_adapters
elif os.name == "posix":
from ._posix import get_adapters
else:
raise RuntimeError("Unsupported Operating System: %s" % os.name)
__all__ = ["Adapter", "IP", "get_adapters"]

Some files were not shown because too many files have changed in this diff Show More