v0.11.42

* extract codec and format info
* add FFprobe as fallback when mutagen fails
* add option to blacklist FFprobe for tags

README.md

@@ -70,7 +70,7 @@ running the sfx without arguments (for example doubleclicking it on Windows) wil
 
 some recommended options:
 * `-e2dsa` enables general file indexing, see [search configuration](#search-configuration)
-* `-e2ts` enables audio metadata indexing (needs either FFprobe or mutagen), see [optional dependencies](#optional-dependencies)
+* `-e2ts` enables audio metadata indexing (needs either FFprobe or Mutagen), see [optional dependencies](#optional-dependencies)
 * `-v /mnt/music:/music:r:afoo -a foo:bar` shares `/mnt/music` as `/music`, `r`eadable by anyone, with user `foo` as `a`dmin (read/write), password `bar`
   * the syntax is `-v src:dst:perm:perm:...` so local-path, url-path, and one or more permissions to set
   * replace `:r:afoo` with `:rfoo` to only make the folder readable by `foo` and nobody else
@@ -155,7 +155,7 @@ small collection of user feedback
 
 # bugs
 
-* Windows: python 3.7 and older cannot read tags with ffprobe, so use mutagen or upgrade
+* Windows: python 3.7 and older cannot read tags with FFprobe, so use Mutagen or upgrade
 * Windows: python 2.7 cannot index non-ascii filenames with `-e2d`
 * Windows: python 2.7 cannot handle filenames with mojibake
 * `--th-ff-jpg` may fix video thumbnails on some FFmpeg versions
@@ -176,6 +176,9 @@ small collection of user feedback
 * Windows: msys2-python 3.8.6 occasionally throws `RuntimeError: release unlocked lock` when leaving a scoped mutex in up2k
   * this is an msys2 bug, the regular windows edition of python is fine
 
+* VirtualBox: sqlite throws `Disk I/O Error` when running in a VM and the up2k database is in a vboxsf
+  * use `--hist` or the `hist` volflag (`-v [...]:chist=/tmp/foo`) to place the db inside the vm instead
+
 
 # the browser
 
@@ -406,12 +409,12 @@ tags that start with a `.` such as `.bpm` and `.dur`(ation) indicate numeric val
 
 see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copyparty/blob/master/copyparty/mtag.py) for the default mappings (should cover mp3,opus,flac,m4a,wav,aif,)
 
-`--no-mutagen` disables mutagen and uses ffprobe instead, which...
-* is about 20x slower than mutagen
-* catches a few tags that mutagen doesn't
+`--no-mutagen` disables Mutagen and uses FFprobe instead, which...
+* is about 20x slower than Mutagen
+* catches a few tags that Mutagen doesn't
   * melodic key, video resolution, framerate, pixfmt
 * avoids pulling any GPL code into copyparty
-* more importantly runs ffprobe on incoming files which is bad if your ffmpeg has a cve
+* more importantly runs FFprobe on incoming files which is bad if your FFmpeg has a cve
 
 
 ## file parser plugins
@@ -545,7 +548,7 @@ below are some tweaks roughly ordered by usefulness:
 
 enable music tags:
 * either `mutagen` (fast, pure-python, skips a few tags, makes copyparty GPL? idk)
-* or `FFprobe` (20x slower, more accurate, possibly dangerous depending on your distro and users)
+* or `ffprobe` (20x slower, more accurate, possibly dangerous depending on your distro and users)
 
 enable thumbnails of images:
 * `Pillow` (requires py2.7 or py3.5+)

contrib/systemd/copyparty.service

@@ -10,8 +10,8 @@
 #
 # with `Type=notify`, copyparty will signal systemd when it is ready to
 #   accept connections; correctly delaying units depending on copyparty.
-#   But note that journalctl does not show messages in the correct order
-#   (or with correct timestamps even), so you get confusing stuff like
+#   But note that journalctl will get the timestamps wrong due to
+#   python disabling line-buffering, so messages are out-of-order:
 #   https://user-images.githubusercontent.com/241032/126040249-cb535cc7-c599-4931-a796-a5d9af691bad.png
 
 [Unit]
@@ -19,6 +19,7 @@ Description=copyparty file server
 
 [Service]
 Type=notify
+SyslogIdentifier=copyparty
 ExecStart=/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::a
 ExecStartPre=/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
 

copyparty/__main__.py

@@ -319,8 +319,9 @@ def run_argparse(argv, formatter):
     ap2.add_argument("-e2tsr", action="store_true", help="rescan all metadata, sets -e2ts")
     ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume state")
     ap2.add_argument("--no-hash", action="store_true", help="disable hashing during e2ds folder scans")
-    ap2.add_argument("--no-mutagen", action="store_true", help="use ffprobe for tags instead")
+    ap2.add_argument("--no-mutagen", action="store_true", help="use FFprobe for tags instead")
     ap2.add_argument("--no-mtag-mt", action="store_true", help="disable tag-read parallelism")
+    ap2.add_argument("--no-mtag-ff", action="store_true", help="never use FFprobe as tag reader")
     ap2.add_argument("-mtm", metavar="M=t,t,t", type=u, action="append", help="add/replace metadata mapping")
     ap2.add_argument("-mte", metavar="M,M,M", type=u, help="tags to index/display (comma-sep.)",
         default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,ac,vc,res,.fps")

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (0, 11, 41)
+VERSION = (0, 11, 42)
 CODENAME = "the grid"
-BUILD_DT = (2021, 7, 17)
+BUILD_DT = (2021, 7, 18)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/mtag.py

@@ -227,37 +227,47 @@ def parse_ffprobe(txt):
 class MTag(object):
     def __init__(self, log_func, args):
         self.log_func = log_func
+        self.args = args
         self.usable = True
-        self.prefer_mt = False
-        mappings = args.mtm
+        self.prefer_mt = not args.no_mtag_ff
         self.backend = "ffprobe" if args.no_mutagen else "mutagen"
-        or_ffprobe = " or ffprobe"
+        self.can_ffprobe = (
+            HAVE_FFPROBE
+            and not args.no_mtag_ff
+            and (not WINDOWS or sys.version_info >= (3, 8))
+        )
+        mappings = args.mtm
+        or_ffprobe = " or FFprobe"
 
         if self.backend == "mutagen":
             self.get = self.get_mutagen
             try:
                 import mutagen
             except:
-                self.log("could not load mutagen, trying ffprobe instead", c=3)
+                self.log("could not load Mutagen, trying FFprobe instead", c=3)
                 self.backend = "ffprobe"
 
         if self.backend == "ffprobe":
+            self.usable = self.can_ffprobe
             self.get = self.get_ffprobe
             self.prefer_mt = True
-            # about 20x slower
-            self.usable = HAVE_FFPROBE
 
-            if self.usable and WINDOWS and sys.version_info < (3, 8):
-                self.usable = False
+            if not HAVE_FFPROBE:
+                pass
+
+            elif args.no_mtag_ff:
+                msg = "found FFprobe but it was disabled by --no-mtag-ff"
+                self.log(msg, c=3)
+
+            elif WINDOWS and sys.version_info < (3, 8):
                 or_ffprobe = " or python >= 3.8"
-                msg = "found ffprobe but your python is too old; need 3.8 or newer"
+                msg = "found FFprobe but your python is too old; need 3.8 or newer"
                 self.log(msg, c=1)
 
         if not self.usable:
-            msg = "need mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
-            self.log(
-                msg.format(or_ffprobe, " " * 37, os.path.basename(sys.executable)), c=1
-            )
+            msg = "need Mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
+            pybin = os.path.basename(sys.executable)
+            self.log(msg.format(or_ffprobe, " " * 37, pybin), c=1)
             return
 
         # https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
@@ -387,7 +397,7 @@ class MTag(object):
             v2 = r2.get(k)
             if v1 == v2:
                 print("  ", k, v1)
-            elif v1 != "0000":  # ffprobe date=0
+            elif v1 != "0000":  # FFprobe date=0
                 diffs.append(k)
                 print(" 1", k, v1)
                 print(" 2", k, v2)
@@ -408,20 +418,33 @@ class MTag(object):
             md = mutagen.File(fsenc(abspath), easy=True)
             x = md.info.length
         except Exception as ex:
-            return {}
+            return self.get_ffprobe(abspath) if self.can_ffprobe else {}
 
-        ret = {}
-        try:
-            dur = int(md.info.length)
+        sz = os.path.getsize(fsenc(abspath))
+        ret = {".q": [0, int((sz / md.info.length) / 128)]}
+
+        for attr, k, norm in [
+            ["codec", "ac", unicode],
+            ["channels", "chs", int],
+            ["sample_rate", ".hz", int],
+            ["bitrate", ".aq", int],
+            ["length", ".dur", int],
+        ]:
             try:
-                q = int(md.info.bitrate / 1024)
+                v = getattr(md.info, attr)
             except:
-                q = int((os.path.getsize(fsenc(abspath)) / dur) / 128)
+                continue
 
-            ret[".dur"] = [0, dur]
-            ret[".q"] = [0, q]
-        except:
-            pass
+            if not v:
+                continue
+
+            if k == ".aq":
+                v /= 1000
+
+            if k == "ac" and v.startswith("mp4a.40."):
+                v = "aac"
+
+            ret[k] = [0, norm(v)]
 
         return self.normalize_tags(ret, md)
 

copyparty/th_srv.py

@@ -121,10 +121,10 @@ class ThumbSrv(object):
         if not self.args.no_vthumb and (not HAVE_FFMPEG or not HAVE_FFPROBE):
             missing = []
             if not HAVE_FFMPEG:
-                missing.append("ffmpeg")
+                missing.append("FFmpeg")
 
             if not HAVE_FFPROBE:
-                missing.append("ffprobe")
+                missing.append("FFprobe")
 
             msg = "cannot create video thumbnails because some of the required programs are not available: "
             msg += ", ".join(missing)

copyparty/u2idx.py

@@ -7,6 +7,7 @@ import time
 import threading
 from datetime import datetime
 
+from .__init__ import unicode
 from .util import s3dec, Pebkac, min_ex
 from .up2k import up2k_wark_from_hashlist
 
@@ -90,6 +91,8 @@ class U2idx(object):
         mt_ctr = 0
         mt_keycmp = "substr(up.w,1,16)"
         mt_keycmp2 = None
+        ptn_lc = re.compile(r" (mt[0-9]+\.v) ([=<!>]+) \? $")
+        ptn_lcv = re.compile(r"[a-zA-Z]")
 
         while True:
             uq = uq.strip()
@@ -182,6 +185,21 @@ class U2idx(object):
             va.append(v)
             is_key = True
 
+            # lowercase tag searches
+            m = ptn_lc.search(q)
+            if not m or not ptn_lcv.search(unicode(v)):
+                continue
+
+            va.pop()
+            va.append(v.lower())
+            q = q[: m.start()]
+
+            field, oper = m.groups()
+            if oper in ["=", "=="]:
+                q += " {} like ? ".format(field)
+            else:
+                q += " lower({}) {} ? ".format(field, oper)
+
         try:
             return self.run_query(vols, joins + "where " + q, va)
         except Exception as ex:

copyparty/up2k.py

@@ -304,7 +304,7 @@ class Up2k(object):
         self.log(msg.format(len(vols), time.time() - t0))
 
         if needed_mutagen:
-            msg = "could not read tags because no backends are available (mutagen or ffprobe)"
+            msg = "could not read tags because no backends are available (Mutagen or FFprobe)"
             self.log(msg, c=1)
 
         thr = None
@@ -596,7 +596,7 @@ class Up2k(object):
             c2 = conn.cursor()
             c3 = conn.cursor()
             n_left = cur.execute("select count(w) from up").fetchone()[0]
-            for w, rd, fn in cur.execute("select w, rd, fn from up"):
+            for w, rd, fn in cur.execute("select w, rd, fn from up order by rd, fn"):
                 n_left -= 1
                 q = "select w from mt where w = ?"
                 if c2.execute(q, (w[:16],)).fetchone():
@@ -1512,7 +1512,7 @@ def up2k_chunksize(filesize):
 
 
 def up2k_wark_from_hashlist(salt, filesize, hashes):
-    """ server-reproducible file identifier, independent of name or location """
+    """server-reproducible file identifier, independent of name or location"""
     ident = [salt, str(filesize)]
     ident.extend(hashes)
     ident = "\n".join(ident)

copyparty/web/browser.js

@@ -177,8 +177,10 @@ function opclick(e) {
 	goto(dest);
 
 	var input = QS('.opview.act input:not([type="hidden"])')
-	if (input && !is_touch)
+	if (input && !is_touch) {
+		tt.skip = true;
 		input.focus();
+	}
 }
 
 
@@ -1711,6 +1713,25 @@ var thegrid = (function () {
 })();
 
 
+function tree_scrollto() {
+	var act = QS('#treeul a.hl');
+	if (!act)
+		return;
+
+	var ctr = ebi('tree'),
+		ul = act.offsetParent,
+		em = parseFloat(getComputedStyle(act).fontSize),
+		top = act.offsetTop + ul.offsetTop,
+		min = top - 11 * em,
+		max = top - (ctr.offsetHeight - 10 * em);
+
+	if (ctr.scrollTop > min)
+		ctr.scrollTop = Math.floor(min);
+	else if (ctr.scrollTop < max)
+		ctr.scrollTop = Math.floor(max);
+}
+
+
 function tree_neigh(n) {
 	var links = QSA('#treeul li>a+a');
 	if (!links.length) {
@@ -1736,6 +1757,7 @@ function tree_neigh(n) {
 	if (act >= links.length)
 		act = 0;
 
+	treectl.dir_cb = tree_scrollto;
 	links[act].click();
 }
 
@@ -1940,7 +1962,7 @@ document.onkeydown = function (e) {
 			for (var b = 1; b < sconf[a].length; b++) {
 				var k = sconf[a][b][0],
 					chk = 'srch_' + k + 'c',
-					tvs = ebi('srch_' + k + 'v').value.split(/ /g);
+					tvs = ebi('srch_' + k + 'v').value.split(/ +/g);
 
 				if (!ebi(chk).checked)
 					continue;
@@ -1953,7 +1975,7 @@ document.onkeydown = function (e) {
 					q += ' and ';
 
 					if (k == 'adv') {
-						q += tv.replace(/ /g, " and ").replace(/([=!><]=?)/, " $1 ");
+						q += tv.replace(/ +/g, " and ").replace(/([=!><]=?)/, " $1 ");
 						continue;
 					}
 
@@ -2576,7 +2598,7 @@ function find_file_col(txt) {
 	for (var a = 0; a < tds.length; a++) {
 		var spans = tds[a].getElementsByTagName('span');
 		if (spans.length && spans[0].textContent == txt) {
-			min = tds[a].getAttribute('class').indexOf('min') !== -1;
+			min = (tds[a].getAttribute('class') || '').indexOf('min') !== -1;
 			i = a;
 			break;
 		}
@@ -2706,6 +2728,10 @@ var filecols = (function () {
 			for (var b = 0, bb = tds.length; b < bb; b++)
 				tds[b].setAttribute('class', cls);
 		}
+		if (window['tt']) {
+			tt.att(ebi('hcols'));
+			tt.att(QS('#files thead'));
+		}
 	};
 	set_style();
 
@@ -3090,7 +3116,7 @@ function reload_browser(not_mp) {
 
 	var oo = QSA('#files>tbody>tr>td:nth-child(3)');
 	for (var a = 0, aa = oo.length; a < aa; a++) {
-		var sz = oo[a].textContent.replace(/ /g, ""),
+		var sz = oo[a].textContent.replace(/ +/g, ""),
 			hsz = sz.replace(/\B(?=(\d{3})+(?!\d))/g, " ");
 
 		oo[a].textContent = hsz;

copyparty/web/util.js

@@ -503,13 +503,19 @@ var tt = (function () {
     var r = {
         "tt": mknod("div"),
         "en": true,
-        "el": null
+        "el": null,
+        "skip": false
     };
 
     r.tt.setAttribute('id', 'tt');
     document.body.appendChild(r.tt);
 
     r.show = function () {
+        if (r.skip) {
+            r.skip = false;
+            return;
+        }
+
         var cfg = sread('tooltips');
         if (cfg !== null && cfg != '1')
             return;