diff --git a/bin/up2k.py b/bin/up2k.py index 8c0c32b5..8cf904c4 100755 --- a/bin/up2k.py +++ b/bin/up2k.py @@ -25,9 +25,10 @@ import hashlib import argparse import platform import threading -import requests import datetime +import requests + # from copyparty/__init__.py PY2 = sys.version_info[0] == 2 @@ -150,13 +151,11 @@ if not VT100: def termsize(): - import os - env = os.environ def ioctl_GWINSZ(fd): try: - import fcntl, termios, struct, os + import fcntl, termios, struct cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234")) except: @@ -360,7 +359,7 @@ def get_hashlist(file, pcb): def handshake(req_ses, url, file, pw, search): - # type: (requests.Session, str, File, any, bool) -> List[str] + # type: (requests.Session, str, File, any, bool) -> list[str] """ performs a handshake with the server; reply is: if search, a list of search results @@ -491,11 +490,34 @@ class Ctl(object): self.filegen = walkdirs([], ar.files) if ar.safe: - self.safe() + self._safe() else: - self.fancy() + self.hash_f = 0 + self.hash_c = 0 + self.hash_b = 0 + self.up_f = 0 + self.up_c = 0 + self.up_b = 0 + self.up_br = 0 + self.hasher_busy = 1 + self.handshaker_busy = 0 + self.uploader_busy = 0 - def safe(self): + self.t0 = time.time() + self.t0_up = None + self.spd = None + + self.mutex = threading.Lock() + self.q_handshake = Queue() # type: Queue[File] + self.q_recheck = Queue() # type: Queue[File] # partial upload exists [...] + self.q_upload = Queue() # type: Queue[tuple[File, str]] + + self.st_hash = [None, "(idle, starting...)"] # type: tuple[File, int] + self.st_up = [None, "(idle, starting...)"] # type: tuple[File, int] + + self._fancy() + + def _safe(self): """minimal basic slow boring fallback codepath""" search = self.ar.s for nf, (top, rel, inf) in enumerate(self.filegen): @@ -529,29 +551,7 @@ class Ctl(object): print(" ok!") - def fancy(self): - self.hash_f = 0 - self.hash_c = 0 - self.hash_b = 0 - self.up_f = 0 - self.up_c = 0 - self.up_b = 0 - self.up_br = 0 - self.hasher_busy = 1 - self.handshaker_busy = 0 - self.uploader_busy = 0 - - self.t0 = time.time() - self.t0_up = None - self.spd = None - - self.mutex = threading.Lock() - self.q_handshake = Queue() # type: Queue[File] - self.q_recheck = Queue() # type: Queue[File] # partial upload exists [...] - self.q_upload = Queue() # type: Queue[tuple[File, str]] - - self.st_hash = [None, "(idle, starting...)"] # type: tuple[File, int] - self.st_up = [None, "(idle, starting...)"] # type: tuple[File, int] + def _fancy(self): if VT100: atexit.register(self.cleanup_vt100) ss.scroll_region(3) @@ -619,7 +619,7 @@ class Ctl(object): tail = "\033[K\033[u" if VT100 else "\r" m = "{0} eta @ {1}/s, {2}, {3}# left".format(eta, spd, sleft, nleft) - eprint(txt + "\033]0;{0}\033\\\r{1}{2}".format(m, m, tail)) + eprint(txt + "\033]0;{0}\033\\\r{0}{1}".format(m, tail)) def cleanup_vt100(self): ss.scroll_region(None) diff --git a/copyparty/__main__.py b/copyparty/__main__.py index 38b13f2e..65ec9f18 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -45,13 +45,13 @@ class RiceFormatter(argparse.HelpFormatter): if not VT100: fmt = " (default: %(default)s)" - help = action.help + ret = action.help if "%(default)" not in action.help: if action.default is not argparse.SUPPRESS: defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE] if action.option_strings or action.nargs in defaulting_nargs: - help += fmt - return help + ret += fmt + return ret def _fill_text(self, text, width, indent): """same as RawDescriptionHelpFormatter(HelpFormatter)""" @@ -157,7 +157,7 @@ def configure_ssl_ver(al): for k in ["ssl_flags_en", "ssl_flags_de"]: num = getattr(al, k) - lprint("{}: {:8x} ({})".format(k, num, num)) + lprint("{0}: {1:8x} ({1})".format(k, num)) # think i need that beer now @@ -294,7 +294,7 @@ def run_argparse(argv, formatter): -v takes src:dst:\033[33mperm\033[0m1:\033[33mperm\033[0m2:\033[33mperm\033[0mN:\033[32mvolflag\033[0m1:\033[32mvolflag\033[0m2:\033[32mvolflag\033[0mN:... * "\033[33mperm\033[0m" is "permissions,username1,username2,..." * "\033[32mvolflag\033[0m" is config flags to set on this volume - + list of permissions: "r" (read): list folder contents, download files "w" (write): upload files; need "r" to see the uploads @@ -313,7 +313,7 @@ def run_argparse(argv, formatter): * w (write-only) for everyone * rw (read+write) for ed * reject duplicate files \033[0m - + if no accounts or volumes are configured, current folder will be read/write for everyone @@ -336,18 +336,18 @@ def run_argparse(argv, formatter): \033[36mnosub\033[35m forces all uploads into the top folder of the vfs \033[36mgz\033[35m allows server-side gzip of uploads with ?gz (also c,xz) \033[36mpk\033[35m forces server-side compression, optional arg: xz,9 - + \033[0mupload rules: \033[36mmaxn=250,600\033[35m max 250 uploads over 15min \033[36mmaxb=1g,300\033[35m max 1 GiB over 5min (suffixes: b, k, m, g) \033[36msz=1k-3m\033[35m allow filesizes between 1 KiB and 3MiB - + \033[0mupload rotation: (moves all uploads into the specified folder structure) \033[36mrotn=100,3\033[35m 3 levels of subfolders with 100 entries in each \033[36mrotf=%Y-%m/%d-%H\033[35m date-formatted organizing \033[36mlifetime=3600\033[35m uploads are deleted after 1 hour - + \033[0mdatabase, general: \033[36me2d\033[35m sets -e2d (all -e2* args can be set using ce2* volflags) \033[36md2ts\033[35m disables metadata collection for existing files @@ -358,24 +358,24 @@ def run_argparse(argv, formatter): \033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso \033[36mhist=/tmp/cdb\033[35m puts thumbnails and indexes at that location \033[36mscan=60\033[35m scan for new files every 60sec, same as --re-maxage - + \033[0mdatabase, audio tags: "mte", "mth", "mtp", "mtm" all work the same as -mte, -mth, ... \033[36mmtp=.bpm=f,audio-bpm.py\033[35m uses the "audio-bpm.py" program to generate ".bpm" tags from uploads (f = overwrite tags) \033[36mmtp=ahash,vhash=media-hash.py\033[35m collects two tags at once - + \033[0mthumbnails: \033[36mdthumb\033[35m disables all thumbnails \033[36mdvthumb\033[35m disables video thumbnails \033[36mdathumb\033[35m disables audio thumbnails (spectrograms) \033[36mdithumb\033[35m disables image thumbnails - + \033[0mclient and ux: \033[36mhtml_head=TXT\033[35m includes TXT in the \033[36mrobots\033[35m allows indexing by search engines (default) \033[36mnorobots\033[35m kindly asks search engines to leave - + \033[0mothers: \033[36mfk=8\033[35m generates per-file accesskeys, which will then be required at the "g" permission @@ -547,7 +547,7 @@ def run_argparse(argv, formatter): ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' volflag") ap2.add_argument("--srch-time", metavar="SEC", type=int, default=30, help="search deadline -- terminate searches running for more than SEC seconds") ap2.add_argument("--srch-hits", metavar="N", type=int, default=7999, help="max search results to allow clients to fetch; 125 results will be shown initially") - + ap2 = ap.add_argument_group('metadata db options') ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing; makes it possible to search for artist/title/codec/resolution/...") ap2.add_argument("-e2ts", action="store_true", help="scan existing files on startup; sets -e2t") diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index aff7e28f..4ca406ee 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -1003,7 +1003,7 @@ class AuthSrv(object): if a == b: local = False - for mtp in local_only_mtp.keys(): + for mtp in local_only_mtp: if mtp not in local_mte: m = 'volume "/{}" defines metadata tag "{}", but doesnt use it in "-mte" (or with "cmte" in its volume-flags)' self.log(m.format(vol.vpath, mtp), 1) @@ -1090,7 +1090,7 @@ class AuthSrv(object): raise Exception("user not found: " + u) if vols == "*": - vols = ["/" + x for x in self.vfs.all_vols.keys()] + vols = ["/" + x for x in self.vfs.all_vols] else: vols = [vols] diff --git a/copyparty/broker_mpw.py b/copyparty/broker_mpw.py index 88e957ef..c4a1054c 100644 --- a/copyparty/broker_mpw.py +++ b/copyparty/broker_mpw.py @@ -8,7 +8,7 @@ import threading from .broker_util import ExceptionalQueue from .httpsrv import HttpSrv from .util import FAKE_MP -from copyparty.authsrv import AuthSrv +from .authsrv import AuthSrv class MpWorker(object): diff --git a/copyparty/broker_util.py b/copyparty/broker_util.py index 527ecc7b..896cba2c 100644 --- a/copyparty/broker_util.py +++ b/copyparty/broker_util.py @@ -11,7 +11,6 @@ class ExceptionalQueue(Queue, object): def get(self, block=True, timeout=None): rv = super(ExceptionalQueue, self).get(block, timeout) - # TODO: how expensive is this? if isinstance(rv, list): if rv[0] == "exception": if rv[1] == "pebkac": diff --git a/copyparty/ftpd.py b/copyparty/ftpd.py index 50043175..8914e8d1 100644 --- a/copyparty/ftpd.py +++ b/copyparty/ftpd.py @@ -6,8 +6,15 @@ import sys import stat import time import logging +import argparse import threading +from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed +from pyftpdlib.filesystems import AbstractedFS, FilesystemError +from pyftpdlib.handlers import FTPHandler +from pyftpdlib.servers import FTPServer +from pyftpdlib.log import config_logging + from .__init__ import E, PY2 from .util import Pebkac, fsenc, exclude_dotfiles from .bos import bos @@ -20,12 +27,6 @@ except ImportError: sys.path.append(p) from pyftpdlib.ioloop import IOLoop -from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed -from pyftpdlib.filesystems import AbstractedFS, FilesystemError -from pyftpdlib.handlers import FTPHandler -from pyftpdlib.servers import FTPServer -from pyftpdlib.log import config_logging - try: from typing import TYPE_CHECKING @@ -154,7 +155,7 @@ class FtpFs(AbstractedFS): vfs_ls.sort() return vfs_ls - except Exception as ex: + except: if vpath: # display write-only folders as empty return [] @@ -266,6 +267,9 @@ class FtpHandler(FTPHandler): else: super(FtpHandler, self).__init__(conn, server, ioloop) + self.hub = None # type: SvcHub + self.args = None # type: argparse.Namespace + # abspath->vpath mapping to resolve log_transfer paths self.vfs_map = {} @@ -278,7 +282,7 @@ class FtpHandler(FTPHandler): # print("ftp_STOR: {} {} OK".format(vp, mode)) return ret - def log_transfer(self, cmd, filename, receive, completed, elapsed, bytes): + def log_transfer(self, cmd, filename, receive, completed, elapsed, nbytes): ap = filename.decode("utf-8", "replace") vp = self.vfs_map.pop(ap, None) # print("xfer_end: {} => {}".format(ap, vp)) @@ -298,7 +302,7 @@ class FtpHandler(FTPHandler): ) return FTPHandler.log_transfer( - self, cmd, filename, receive, completed, elapsed, bytes + self, cmd, filename, receive, completed, elapsed, nbytes ) diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index f662b917..5368bf62 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -532,7 +532,6 @@ class HttpCli(object): return self.handle_post_multipart() if "text/plain" in ctype or "application/xml" in ctype: - # TODO this will be intredasting return self.handle_post_json() if "application/octet-stream" in ctype: diff --git a/copyparty/mtag.py b/copyparty/mtag.py index ce723f3b..454fd43f 100644 --- a/copyparty/mtag.py +++ b/copyparty/mtag.py @@ -8,7 +8,7 @@ import shutil import subprocess as sp from .__init__ import PY2, WINDOWS, unicode -from .util import fsenc, fsdec, uncyg, runcmd, retchk, REKOBO_LKEY +from .util import fsenc, uncyg, runcmd, retchk, REKOBO_LKEY from .bos import bos @@ -91,7 +91,7 @@ def parse_ffprobe(txt): """ffprobe -show_format -show_streams""" streams = [] fmt = {} - g = None + g = {} for ln in [x.rstrip("\r") for x in txt.split("\n")]: try: k, v = ln.split("=", 1) @@ -421,7 +421,7 @@ class MTag(object): md = mutagen.File(fsenc(abspath), easy=True) if not md.info.length and not md.info.codec: raise Exception() - except Exception as ex: + except: return self.get_ffprobe(abspath) if self.can_ffprobe else {} sz = bos.path.getsize(abspath) diff --git a/copyparty/star.py b/copyparty/star.py index b560f9d5..804ad724 100644 --- a/copyparty/star.py +++ b/copyparty/star.py @@ -88,7 +88,7 @@ class StreamTar(object): try: self.ser(f) - except Exception: + except: ex = min_ex(5, True).replace("\n", "\n-- ") errors.append([f["vp"], ex]) diff --git a/copyparty/sutil.py b/copyparty/sutil.py index 8d27757d..22de0066 100644 --- a/copyparty/sutil.py +++ b/copyparty/sutil.py @@ -1,7 +1,6 @@ # coding: utf-8 from __future__ import print_function, unicode_literals -import time import tempfile from datetime import datetime diff --git a/copyparty/szip.py b/copyparty/szip.py index c5c107db..177ff3ea 100644 --- a/copyparty/szip.py +++ b/copyparty/szip.py @@ -257,7 +257,7 @@ class StreamZip(object): try: for x in self.ser(f): yield x - except Exception: + except: ex = min_ex(5, True).replace("\n", "\n-- ") errors.append([f["vp"], ex]) diff --git a/copyparty/tcpsrv.py b/copyparty/tcpsrv.py index 25516d08..29d23759 100644 --- a/copyparty/tcpsrv.py +++ b/copyparty/tcpsrv.py @@ -297,7 +297,6 @@ class TcpSrv(object): ]: try: s.connect((ip, 1)) - # raise OSError(13, "a") default_route = s.getsockname()[0] break except (OSError, socket.error) as ex: @@ -318,23 +317,23 @@ class TcpSrv(object): return eps - def _set_wintitle(self, vars): - vars["all"] = vars.get("all", {"Local-Only": 1}) - vars["pub"] = vars.get("pub", vars["all"]) + def _set_wintitle(self, vs): + vs["all"] = vs.get("all", {"Local-Only": 1}) + vs["pub"] = vs.get("pub", vs["all"]) - vars2 = {} - for k, eps in vars.items(): - vars2[k] = { + vs2 = {} + for k, eps in vs.items(): + vs2[k] = { ep: 1 for ep in eps.keys() if ":" not in ep or ep.split(":")[0] not in eps } title = "" - vars = vars2 + vs = vs2 for p in self.args.wintitle.split(" "): if p.startswith("$"): - p = " and ".join(sorted(vars.get(p[1:], {"(None)": 1}).keys())) + p = " and ".join(sorted(vs.get(p[1:], {"(None)": 1}).keys())) title += "{} ".format(p)