Compare commits

...

347 Commits

Author SHA1 Message Date
ed
dc2e2cbd4b v1.6.5 2023-02-12 14:11:45 +00:00
ed
5c12dac30f most ffmpeg builds dont support compressed modules 2023-02-12 14:02:43 +00:00
ed
641929191e fix reading smb shares on windows 2023-02-12 13:59:34 +00:00
ed
617321631a docker: add annotations 2023-02-11 21:10:28 +00:00
ed
ddc0c899f8 update archpkg to 1.6.4 2023-02-11 21:01:45 +00:00
ed
cdec42c1ae v1.6.4 2023-02-11 18:02:05 +00:00
ed
c48f469e39 park all clients waiting for a transcode 2023-02-11 17:23:29 +00:00
ed
44909cc7b8 print ffmpeg download url on windows 2023-02-11 17:22:24 +00:00
ed
8f61e1568c transcode chiptunes to opus;
* new audio/MPT formats: apac bonk dfpwm ilbc it itgz itr itz mo3 mod mptm mt2 okt s3gz s3m s3r s3z xm xmgz xmr xmz xpk
* new image/PIL formats: blp dcx emf eps fits flc fli fpx im j2k j2p psd spi wmf
2023-02-11 11:17:37 +00:00
ed
b7be7a0fd8 mirror docker images to ghcr 2023-02-10 23:40:30 +00:00
ed
1526a4e084 add docker packaging 2023-02-10 23:02:01 +00:00
ed
dbdb9574b1 doc-browser: fix md scaling + download hotkey 2023-02-10 21:33:48 +00:00
ed
853ae6386c config load summary + safer windows defaults 2023-02-10 21:32:42 +00:00
ed
a4b56c74c7 support long filepaths on win7 + misc windows fixes 2023-02-10 18:37:37 +00:00
ed
d7f1951e44 fix --cgen for 'g' perms 2023-02-08 22:38:21 +00:00
ed
7e2ff9825e ensure -e2tsr takes effect by ignoring dhash 2023-02-08 22:33:02 +00:00
ed
9b423396ec better description for anonymous permissions 2023-02-07 20:12:45 +00:00
ed
781146b2fb describe all database volflags in --help-flags 2023-02-07 20:07:06 +00:00
ed
84937d1ce0 add v2 config syntax (#20) 2023-02-07 19:54:08 +00:00
ed
98cce66aa4 cgen: update set of multivalue keys 2023-02-06 07:26:23 +00:00
ed
043c2d4858 cgen: fix permissions listing 2023-02-06 07:23:35 +00:00
ed
99cc434779 add config explainer + generator (#20) 2023-02-05 22:09:17 +00:00
ed
5095d17e81 more interesting config example 2023-02-05 21:32:20 +00:00
ed
87d835ae37 dont allow multiple volumes at the same fs-path 2023-02-05 21:16:36 +00:00
ed
6939ca768b pkg/arch: add prisonparty 2023-02-05 00:07:04 +00:00
ed
e3957e8239 systemd: prisonparty improvements 2023-02-05 00:03:40 +00:00
ed
4ad6e45216 only load *.conf files when including a folder 2023-02-05 00:01:10 +00:00
ed
76e5eeea3f prisonparty: fix reload signal 2023-02-05 00:00:18 +00:00
ed
eb17f57761 pypi fixes 2023-02-04 17:35:20 +00:00
ed
b0db14d8b0 indicate forced-randomized filenames 2023-02-04 15:18:09 +00:00
ed
2b644fa81b don't alias randomized filenames 2023-02-04 13:41:43 +00:00
ed
190ccee820 add optional version number on controlpanel 2023-02-04 13:41:34 +00:00
JeremyStarTM
4e7dd32e78 Added "wow this is better than nextcloud" (#19)
* Added "wow this is better than nextcloud"
2023-02-04 13:00:16 +00:00
john smith
5817fb66ae goddamn tabs 2023-02-03 12:50:17 +01:00
john smith
9cb04eef93 misc PKGBUILD fixes 2023-02-03 12:50:17 +01:00
john smith
0019fe7f04 indent PKGBUILD with spaces instead of tabs 2023-02-03 12:50:17 +01:00
john smith
852c6f2de1 remove unnecessary dependencies from PKGBUILD 2023-02-03 12:50:17 +01:00
john smith
c4191de2e7 improve PKGBUILD based on stuff in https://github.com/9001/copyparty/issues/17 2023-02-03 12:50:17 +01:00
ed
4de61defc9 add a link exporter to the unpost ui too 2023-02-02 22:57:59 +00:00
ed
0aa88590d0 should generalize this somehow 2023-02-02 22:35:13 +00:00
ed
405f3ee5fe adjustable toast position 2023-02-02 22:28:31 +00:00
ed
bc339f774a button to show/copy links for all recent uploads 2023-02-02 22:27:53 +00:00
ed
e67b695b23 show filekeys in recent-uploads ui 2023-02-02 21:22:51 +00:00
ed
4a7633ab99 fix outdated docs mentioned in #17 sry 2023-02-02 20:12:32 +00:00
john smith
c58f2ef61f fix PKGBUILD more 2023-02-02 20:48:20 +01:00
john smith
3866e6a3f2 fix PKGBUILD indentation 2023-02-02 20:30:48 +01:00
john smith
381686fc66 add PKGBUILD 2023-02-02 20:30:48 +01:00
ed
a918c285bf up2k-ui: button to randomize upload filenames 2023-02-01 22:26:18 +00:00
ed
1e20eafbe0 volflag to randomize all upload filenames 2023-02-01 21:58:01 +00:00
ed
39399934ee v1.6.3 2023-01-31 21:03:43 +00:00
ed
b47635150a shove #files aside while prologue sandbox is loading 2023-01-31 21:02:58 +00:00
ed
78d2f69ed5 prisonparty: support opus transcoding on debian
libblas.so and liblapack.so are symlinks into /etc/alternatives
2023-01-31 20:50:59 +00:00
ed
7a98dc669e block alerts in sandbox by default + add translation 2023-01-31 19:16:28 +00:00
ed
2f15bb5085 include filesize in notification 2023-01-31 19:03:13 +00:00
ed
712a578e6c indicate when a readme/logue was hidden 2023-01-31 19:01:24 +00:00
ed
d8dfc4ccb2 support davfs2 LOCK (uploads) + misc windows support + logue filtering 2023-01-31 18:53:38 +00:00
ed
e413007eb0 hide dotfiles from search results by default 2023-01-31 18:13:33 +00:00
ed
6d1d3e48d8 sandbox height didnt account for scrollbars 2023-01-31 17:54:04 +00:00
ed
04966164ce more iframe-resize-concealing tricks 2023-01-31 17:43:21 +00:00
ed
8b62aa7cc7 unlink files before replacing them
to avoid hardlink-related surprises
2023-01-31 17:17:18 +00:00
ed
1088e8c6a5 optimize 2023-01-30 22:53:27 +00:00
ed
8c54c2226f cover up most of the layout jank 2023-01-30 22:52:16 +00:00
ed
f74ac1f18b fix sandbox lag by helping the iframe cache js 2023-01-30 22:36:05 +00:00
ed
25931e62fd and nofollow the basic-browser link too 2023-01-29 22:15:22 +00:00
ed
707a940399 add nofollow to zip links 2023-01-29 22:10:03 +00:00
ed
87ef50d384 doc 2023-01-29 21:23:48 +00:00
ed
dcadf2b11c v1.6.2 2023-01-29 18:42:21 +00:00
ed
37a690a4c3 fix cookie + rproxy oversights 2023-01-29 18:34:48 +00:00
ed
87ad23fb93 docs + chmod 2023-01-29 18:28:53 +00:00
ed
5f54d534e3 hook/notify: add android support 2023-01-29 15:14:22 +00:00
ed
aecae552a4 v1.6.1 2023-01-29 04:41:16 +00:00
ed
eaa6b3d0be mute some startup noise 2023-01-29 04:33:28 +00:00
ed
c2ace91e52 v1.6.0 2023-01-29 02:55:44 +00:00
ed
0bac87c36f make loss of hotkeys more obvious 2023-01-29 01:40:02 +00:00
ed
e650d05939 shovel across most of the env too 2023-01-29 01:19:53 +00:00
ed
85a96e4446 add custom text selection colors because chrome is broken on fedora 2023-01-29 01:03:10 +00:00
ed
2569005139 support sandboxed markdown plugins 2023-01-29 00:57:08 +00:00
ed
c50cb66aef sandboxed other-origin iframes dont cache css 2023-01-28 23:40:25 +00:00
ed
d4c5fca15b sandbox readme.md / prologue / epilogue 2023-01-28 21:24:40 +00:00
ed
75cea4f684 misc 2023-01-28 13:35:49 +00:00
ed
68c6794d33 rewrite other symlinks after the actual move;
fixes volumes where symlinking is disabled
2023-01-28 01:14:29 +00:00
ed
82f98dd54d delete/move is now POST 2023-01-28 01:02:50 +00:00
ed
741d781c18 add cors controls + improve preflight + pw header 2023-01-28 00:59:04 +00:00
ed
0be1e43451 mention mtp in the hooks readme 2023-01-28 00:07:50 +00:00
ed
5366bf22bb describe detected network changes 2023-01-27 23:56:54 +00:00
ed
bcd91b1809 add eventhook examples 2023-01-27 23:55:57 +00:00
ed
9bd5738e6f shorter fallback hostname 2023-01-27 22:19:25 +00:00
ed
bab4aa4c0a mkdir fix 2023-01-27 22:16:10 +00:00
ed
e965b9b9e2 mkdir missing volumes on startup 2023-01-27 21:52:28 +00:00
ed
31101427d3 support downloading blockdev contents 2023-01-27 21:09:57 +00:00
ed
a083dc36ba dont get confused by dangling symlinks at target 2023-01-27 20:27:00 +00:00
ed
9b7b9262aa promote dedup control to volflags 2023-01-25 21:46:15 +00:00
ed
660011fa6e md-editor: make hotkey ^e more global 2023-01-25 20:58:28 +00:00
ed
ead31b6823 add eventhook sanchecks 2023-01-25 20:51:02 +00:00
ed
4310580cd4 separate http/https logins (breaks ie4 / win3.11 login) 2023-01-24 21:23:57 +00:00
ed
b005acbfda enable text selection between breadcrumbs + update vs 2023-01-23 22:44:29 +00:00
ed
460709e6f3 upgrade wget downloader to use event hooks 2023-01-22 23:45:11 +00:00
ed
a8768d05a9 add comparison to similar software 2023-01-22 23:39:19 +00:00
ed
f8e3e87a52 add event hooks 2023-01-22 23:35:31 +00:00
ed
70f1642d0d allow tar/zip download of hidden folders 2023-01-21 20:56:44 +00:00
ed
3fc7561da4 macos 2023-01-21 10:36:31 +00:00
ed
9065226c3d oh great its in lts too 2023-01-21 10:19:04 +00:00
ed
b7e321fa47 cleanup 2023-01-19 22:26:49 +00:00
ed
664665b86b fix some location-rproxy bugs 2023-01-19 22:26:24 +00:00
ed
f4f362b7a4 add --freebind 2023-01-18 21:55:36 +00:00
ed
577d23f460 zeroconf: detect network change and reannounce 2023-01-18 21:27:27 +00:00
ed
504e168486 compensate avg.speed for single-chunk uploads 2023-01-18 19:53:19 +00:00
ed
f2f9640371 workaround firefox layout bug:
three-line toasts get a scrollbar even if it doesn't need one
and the width is not adjusted correctly when that happens
2023-01-18 19:45:04 +00:00
ed
ee46f832b1 u2cli: add option -ns for slow terminals 2023-01-17 23:29:51 +00:00
ed
b0e755d410 give curl colored (yet sortable) plaintext listings 2023-01-17 23:22:43 +00:00
ed
cfd24604d5 ux tweaks 2023-01-17 23:21:31 +00:00
ed
264894e595 add cursed usecases 2023-01-16 21:46:11 +00:00
ed
5bb9f56247 linux 6.1 fixed the 6.0 bugs; remove workarounds 2023-01-16 20:44:57 +00:00
ed
18942ed066 location-based rproxy fixes 2023-01-16 20:09:45 +00:00
ed
85321a6f31 stale tree is better than no tree 2023-01-15 20:54:03 +00:00
ed
baf641396d add optional powered-by footnode 2023-01-15 20:52:38 +00:00
ed
17c91e7014 override bogus mimetypes 2023-01-14 15:10:32 +00:00
ed
010770684d workaround another linux kernel bug 2023-01-14 08:16:15 +00:00
ed
b4c503657b ignore loss of stdout 2023-01-14 07:35:44 +00:00
ed
71bd306268 fix unpost filters with slashes 2023-01-13 17:56:32 +00:00
ed
dd7fab1352 u2cli: properly retry failed handshakes 2023-01-13 07:17:41 +00:00
ed
dacca18863 v1.5.6 2023-01-12 05:15:30 +00:00
ed
53d92cc0a6 faster upload of small files on high-latency nets 2023-01-12 02:53:22 +00:00
ed
434823f6f0 ui: allow changing num.threads in search-only 2023-01-11 16:14:02 +00:00
ed
2cb1f50370 fix dualstack on lo 2023-01-11 16:10:07 +00:00
ed
03f53f6392 gallery: fix js error on digit-keypress viewing pics 2023-01-11 16:08:15 +00:00
ed
a70ecd7af0 v1.5.5 2022-12-30 07:54:34 +00:00
ed
8b81e58205 mdns fixes 2022-12-30 07:47:53 +00:00
ed
4500c04edf v1.5.4 2022-12-29 04:44:15 +00:00
ed
6222ddd720 fix ssdp on dualstack 2022-12-22 16:50:46 +00:00
ed
8a7135cf41 support fat32 time precision, avoiding rescans
posted from warzaw airport otw to japan
2022-12-20 22:19:32 +01:00
ed
b4c7282956 password from file 2022-12-20 13:28:48 +00:00
ed
8491a40a04 Create SECURITY.md 2022-12-19 21:18:27 +00:00
ed
343d38b693 extend image-viewer with modern formats 2022-12-15 22:38:33 +00:00
ed
6cf53d7364 try next thumbnailer if one fails;
libvips assumes imagemagick was built with avif
2022-12-15 22:34:51 +00:00
ed
b070d44de7 libvips logging + raise codec errors 2022-12-15 22:22:04 +00:00
ed
79aa40fdea cosmetic fixes 2022-12-14 23:12:51 +00:00
ed
dcaff2785f v1.5.3 2022-12-13 19:56:34 +00:00
ed
497f5b4307 add hotkey to enable download mode 2022-12-13 19:50:20 +00:00
ed
be32ad0da6 add sfx tester 2022-12-13 19:05:10 +00:00
ed
8ee2bf810b stop battleplan from indirectly crashing the browser 2022-12-13 18:58:16 +00:00
ed
28232656a9 folder-sync optimizations 2022-12-13 18:56:40 +00:00
ed
fbc2424e8f v1.5.2 2022-12-12 22:59:31 +00:00
ed
94cd13e8b8 reorder help categories 2022-12-12 22:18:17 +00:00
ed
447ed5ab37 windows fixes 2022-12-12 21:59:50 +00:00
ed
af59808611 u2cli: always compare toplevel in syncs 2022-12-12 07:16:05 +01:00
ed
e3406a9f86 dont cls by default 2022-12-11 22:46:21 +00:00
ed
7fd1d6a4e8 rename --webroot to --rp-loc and fix related bugs 2022-12-11 21:09:50 +00:00
ed
0ab2a665de add example apache config + readme notes 2022-12-11 21:01:38 +00:00
ed
3895575bc2 add sliding window for upload eta 2022-12-11 19:46:39 +00:00
ed
138c2bbcbb o no 2022-12-11 18:30:29 +00:00
ed
bc7af1d1c8 u2cli: add basic folder sync 2022-12-11 17:41:10 +00:00
ed
19cd96e392 cleanup + optimizations 2022-12-11 14:16:51 +00:00
ed
db194ab519 support location-based rproxy 2022-12-10 23:43:31 +00:00
ed
02ad4bfab2 ensure consistency between db tables 2022-12-10 22:13:21 +00:00
ed
56b73dcc8a up2k: add option to replace existing file 2022-12-10 19:22:16 +00:00
ed
7704b9c8a2 sqlite durability profiles 2022-12-10 10:01:33 +00:00
ed
999b7ae919 safer to merge wal on startup instead 2022-12-09 19:58:13 +00:00
ed
252b5a88b1 use linklocal on NICs without routable IPs 2022-12-09 19:11:26 +00:00
ed
01e2681a07 davfs2 requires realm 2022-12-09 17:59:24 +00:00
ed
aa32f30202 zeroconf: dont cache until resolved 2022-12-08 18:05:45 +00:00
ed
195eb53995 merge wal on shutdown 2022-12-07 23:09:40 +00:00
ed
06fa78f54a windows: set .hist folder hidden 2022-12-07 22:56:30 +00:00
ed
7a57c9dbf1 translation 2022-12-07 22:47:33 +00:00
ed
bb657bfa85 more intuitive batch-unpost ordering 2022-12-07 22:30:48 +00:00
ed
87181726b0 sfx: fix multiprocessing on windows 2022-12-07 22:21:28 +00:00
ed
f1477a1c14 block other copyparties from sniping tcp ports 2022-12-07 21:50:52 +00:00
ed
4f94a9e38b exe: survive ascii locales 2022-12-07 21:35:53 +00:00
ed
fbed322d3b option to skip database syncs entirely 2022-12-07 21:35:04 +00:00
ed
9b0f519e4e switch to wal for ~2x faster uploads 2022-12-07 20:52:17 +00:00
ed
6cd6dadd06 optional linklocal ipv6 support (firefox/ie11 only) 2022-12-05 20:45:21 +00:00
ed
9a28afcb48 custom mediaplayer-toggle cursor 2022-12-05 19:46:48 +00:00
ed
45b701801d fix ssdp xml escaping + target url 2022-12-05 19:13:47 +00:00
ed
062246fb12 allow specifying zeroconf filters by subnet 2022-12-05 17:56:39 +00:00
ed
416ebfdd68 right, windows nic names have whitespace 2022-12-05 17:35:12 +00:00
ed
731eb92f33 fix exception opening the connect page on phones 2022-12-04 17:18:14 +00:00
ed
dbe2aec79c v1.5.1 2022-12-03 20:48:52 +00:00
ed
cd9cafe3a1 v1.5.0 2022-12-03 20:45:49 +00:00
ed
067cc23346 docs + cleanup 2022-12-03 18:58:56 +00:00
ed
c573a780e9 some failsafes 2022-12-03 16:37:14 +00:00
ed
8ef4a0aa71 fix testrunner + packaging 2022-12-03 15:07:47 +00:00
ed
89ba12065c ssdp: add ie8 compat 2022-12-03 13:59:46 +00:00
ed
99efc290df fix mdns on windows 2022-12-03 13:31:00 +00:00
ed
2fbdc0a85e misc fixes / cleanup 2022-12-02 23:42:46 +00:00
ed
4242422898 update deps: marked.js, codemirror 2022-12-02 21:39:04 +00:00
ed
008d9b1834 add textbox placeholders 2022-12-02 18:33:04 +00:00
ed
7c76d08958 drop one of the slowloris detectors 2022-12-02 17:53:23 +00:00
ed
89c9f45fd0 add option for cross-volume dedupe 2022-12-02 17:25:37 +00:00
ed
f107497a94 a bit better 2022-12-01 22:18:17 +00:00
ed
b5dcf30e53 w/a firefox sometimes loading stale documents
never been able to reproduce it intentionally but this should work
2022-12-01 21:52:40 +00:00
ed
0cef062084 misc cleanup 2022-12-01 21:44:31 +00:00
ed
5c30148be4 also scroll to playing track when resizing window 2022-11-29 22:16:14 +00:00
ed
3a800585bc u2cli: server is allowed to reject dupes 2022-11-29 22:09:32 +00:00
ed
29c212a60e macos bigsur breaks on symlinks in ftp listings 2022-11-28 22:10:05 +00:00
ed
2997baa7cb better recovery from i/o errors 2022-11-28 22:06:31 +00:00
ed
dc6bde594d fix make-sfx macos support 2022-11-28 21:38:50 +00:00
ed
e357aa546c add browserchrome color hint 2022-11-28 21:19:42 +00:00
ed
d3fe19c5aa misc fixes 2022-11-28 20:25:32 +00:00
ed
bd24bf9bae option to follow playing song 2022-11-28 20:24:47 +00:00
ed
ee141544aa option for compact mediaplayer 2022-11-28 20:10:10 +00:00
ed
db6f6e6a23 option to hide scrollbars 2022-11-28 19:47:14 +00:00
ed
c7d950dd5e ux tweaks + devdocs 2022-11-27 22:07:28 +00:00
ed
6a96c62fde ok windows is just gonna have to make do 2022-11-27 22:05:38 +00:00
ed
36dc8cd686 readme + misc 2022-11-27 01:30:18 +00:00
ed
7622601a77 forgot to actually enable the new landing page 2022-11-27 00:01:28 +00:00
ed
cfd41fcf41 zeroconf: add network filtering options 2022-11-26 22:37:12 +00:00
ed
f39e370e2a cosmetic 2022-11-26 22:27:09 +00:00
ed
c1315a3b39 webdav: misc fixes 2022-11-26 20:06:48 +00:00
ed
53b32f97e8 ftp: support touch+write, windows-login, verbosity 2022-11-26 20:03:17 +00:00
ed
6c962ec7d3 rename copyparty-fuse to partyfuse 2022-11-26 20:01:20 +00:00
ed
6bc1bc542f rename copyparty-fuse to partyfuse 2022-11-26 19:53:41 +00:00
ed
f0e78a6826 add landing page with mounting instructions 2022-11-26 19:47:27 +00:00
ed
e53531a9fb ssdp: get rid of ipv6 + fix http port selection 2022-11-23 22:44:17 +00:00
ed
5cd9d11329 add ssdp responder 2022-11-22 21:40:12 +00:00
ed
5a3e504ec4 uninvent a square wheel 2022-11-22 19:12:41 +00:00
ed
d6e09c3880 ux: dedicated column-hiding mode on phones 2022-11-21 20:44:58 +00:00
ed
04f44c3c7c add global option for rejecting dupe uploads 2022-11-21 10:58:15 +00:00
ed
ec587423e8 show/hide tagsearch ui based on folder flags 2022-11-20 23:30:01 +00:00
ed
f57b31146d improve parent-folder button on phones 2022-11-20 22:37:55 +00:00
ed
35175fd685 mdns: support primitive clients (android, rfc-6.7) 2022-11-20 20:31:11 +00:00
ed
d326ba9723 ftp: ban password-bruteforcing IPs 2022-11-20 11:06:07 +00:00
ed
ab655a56af add buttons for prev/next folder 2022-11-19 22:19:38 +00:00
ed
d1eb113ea8 add button+hotkey to download all selected files 2022-11-19 21:57:25 +00:00
ed
74effa9b8d audioplayer: time at mousecursor while scrubbing 2022-11-19 20:00:50 +00:00
ed
bba4b1c663 sfx: py3.12 support 2022-11-19 10:47:54 +00:00
ed
8709d4dba0 macos smb: avoid hang on shutdown 2022-11-17 21:17:54 +00:00
ed
4ad4657774 mdns: support running on macos 2022-11-17 20:18:24 +00:00
ed
5abe0c955c this spec is confusing 2022-11-17 09:08:58 +00:00
ed
0cedaf4fa9 isort 2022-11-15 22:41:35 +00:00
ed
0aa7d12704 add option to disable .hist/up2k.snap 2022-11-15 22:16:53 +00:00
ed
a234aa1f7e cleaner shutdown of smbd, mdns 2022-11-15 21:55:02 +00:00
ed
9f68287846 workaround impacket glob bug 2022-11-15 21:29:02 +00:00
ed
cd2513ec16 logging fixes 2022-11-15 21:28:27 +00:00
ed
91d132c2b4 add basic-ui hint for firefox 8 and older 2022-11-15 20:17:53 +00:00
ed
97ff0ebd06 xz-compress logs only if -lo ends with .xz 2022-11-15 20:16:41 +00:00
ed
8829f56d4c mdns ipv6 fixes; now works on ie11/safari, not linux:
* subscribe/announce on LL only
* add NSEC records if 4/6-only
2022-11-15 06:39:53 +00:00
ed
37c1cab726 dnslib tweaks for mdns / py3 2022-11-13 20:06:39 +00:00
ed
b3eb117e87 add mdns zeroconf announcer 2022-11-13 20:05:16 +00:00
ed
fc0a941508 support old linux consoles 2022-11-06 16:58:00 +00:00
ed
c72753c5da add native ipv6 support 2022-11-06 16:48:05 +00:00
ed
e442cb677a improve ftp/smb logging 2022-11-06 13:30:16 +00:00
ed
450121eac9 ftpd: kde tries to cwd into images 2022-11-05 13:24:00 +00:00
ed
b2ab8f971e add config-file preprocessor (%include) 2022-11-04 23:48:14 +00:00
ed
e9c6268568 add more sfx opt-outs 2022-11-04 20:50:52 +00:00
ed
2170ee8da4 improve scheduling 2022-11-04 20:28:05 +00:00
ed
357e7333cc cleanup 2022-11-04 20:27:16 +00:00
ed
8bb4f02601 add textlabel on volume slider 2022-11-04 20:04:39 +00:00
ed
4213efc7a6 optimize more 2022-11-04 19:33:48 +00:00
ed
67a744c3e8 audioplayer: optimize ui for week-long audio files 2022-11-03 23:20:58 +00:00
ed
98818e7d63 smb: workaround impacket response size limit 2022-11-03 23:17:24 +00:00
ed
8650ce1295 smb: too many clients get confused by blank password 2022-11-03 23:08:04 +00:00
ed
9638267b4c up2k-ui: survive hitting inaccessible subfolders 2022-11-02 22:02:46 +00:00
ed
304e053155 improve default-gateway / external-IP detection 2022-11-02 21:43:20 +00:00
ed
89d1f52235 cursory slowloris / buggy-webdav-client detector 2022-11-01 22:18:20 +00:00
ed
3312c6f5bd autoclose connection-flooding clients 2022-10-31 22:42:47 +00:00
ed
d4ba644d07 autodefault -nc based on OS limits 2022-10-31 19:37:37 +00:00
ed
b9a504fd3a x32/x64-agnostic exe builder 2022-10-30 18:35:27 +00:00
ed
cebac523dc fix url anchors into markdown docs 2022-10-30 18:03:40 +00:00
ed
c2f4090318 webdav: mute some macos spam 2022-10-30 17:45:28 +00:00
ed
d562956809 webdav: windows configurator util 2022-10-30 17:41:33 +00:00
ed
62499f9b71 webdav: more sensible overwrite logic 2022-10-30 17:13:06 +00:00
ed
89cf7608f9 webdav: help windows deal with read-only volumes 2022-10-30 17:11:43 +00:00
ed
dd26b8f183 webdav: bump chunksize from 2048 to 32760 byte 2022-10-30 16:53:15 +00:00
ed
79303dac6d webdav: default-disable recursive listing 2022-10-30 16:47:20 +00:00
ed
4203fc161b misc 2022-10-30 16:31:04 +00:00
ed
f8a31cc24f chrome can play some mkv files 2022-10-30 16:12:47 +00:00
ed
fc5bfe81a0 add hotkey '?' for hotkeys listing 2022-10-30 16:05:14 +00:00
ed
aae14de796 mouse3 docs in the navpane 2022-10-30 13:13:58 +00:00
ed
54e1c8d261 remove 697 GiB upload filesize limit 2022-10-30 12:51:20 +00:00
ed
a0cc4ca4b7 up2k-cli: enable mt if chrome 107 or later 2022-10-29 22:57:59 +00:00
ed
2701108c5b up2k-ui: suggest potato to avoid firefox-bug 1790500 2022-10-29 22:46:13 +00:00
ed
73bd2df2c6 more metadata-parser debug options 2022-10-29 21:59:59 +00:00
ed
0063021012 mtp-deps: add fedora support 2022-10-29 21:38:08 +00:00
ed
1c3e4750b3 better android howto 2022-10-29 20:46:22 +00:00
ed
edad3246e0 make pylance happier 2022-10-29 20:40:25 +00:00
ed
3411b0993f fix msg-to-log 2022-10-26 02:35:32 +02:00
ed
097b5609dc support grapheneos 2022-10-26 02:35:10 +02:00
ed
a42af7655e fix relative link 2022-10-26 02:32:24 +02:00
ed
69f78b86af cleanup 2022-10-25 01:23:41 +02:00
ed
5f60c509c6 smb: add better-than-nothing permission checks 2022-10-24 21:16:57 +02:00
ed
75e5e53276 readme refactor 2022-10-24 18:48:12 +02:00
ed
4b2b4ed52d smb: fix file rename 2022-10-24 16:08:02 +02:00
ed
fb21bfd6d6 update localmount / rclone docs 2022-10-24 15:48:34 +02:00
ed
f14369e038 webdav: mkdir semantics 2022-10-24 14:09:09 +02:00
ed
ff04b72f62 smb: add mkdir/copy/rename/delete 2022-10-24 14:08:32 +02:00
ed
4535a81617 smb: add up2k-indexing on write 2022-10-24 13:44:19 +02:00
ed
cce57b700b fix range-request on empty files 2022-10-24 03:26:32 +02:00
ed
5b6194d131 stop win10-webdav from flooding the server 2022-10-24 02:33:23 +02:00
ed
2701238cea reply raw markdown unless ?v 2022-10-24 02:10:07 +02:00
ed
835f8a20e6 default-enable webdav 2022-10-23 23:37:32 +02:00
ed
f3a501db30 add SMB/CIFS server 2022-10-23 23:08:00 +02:00
ed
4bcd30da6b cleaner daemon instancing 2022-10-23 12:05:44 +02:00
ed
947dbb6f8a webdav mimetypes based on file extensions (for gnome) 2022-10-22 02:08:19 +02:00
ed
1c2fedd2bf let webdav replace empty files when sufficiently safe 2022-10-22 01:31:18 +02:00
ed
32e826efbc catch and discard macos metadata files 2022-10-22 01:15:54 +02:00
ed
138b932c6a add webdav move/delete 2022-10-22 00:04:51 +02:00
ed
6da2f53aad avoid macos tmpfiles-cleaner 2022-10-21 18:49:25 +02:00
ed
20eeacaac3 add webdav write support + fix http 200/201 2022-10-21 18:47:48 +02:00
ed
81d896be9f webdav notes 2022-10-19 15:52:19 +02:00
ed
c003dfab03 unbold ansi grays 2022-10-19 15:30:17 +02:00
ed
20c6b82bec replace magic numbers with errno.* 2022-10-19 15:21:48 +02:00
ed
046b494b53 winpe support + windows webdav stuff 2022-10-19 00:06:48 +02:00
ed
f0e98d6e0d win7 webdav workarounds 2022-10-18 20:52:12 +02:00
ed
fe57321853 correct 401/403 usage for webdav 2022-10-18 20:29:06 +02:00
ed
8510804e57 initial webdav support 2022-10-18 19:36:52 +02:00
ed
acd32abac5 v1.4.6 2022-10-13 21:37:05 +02:00
ed
2b47c96cf2 move licenses into module proper 2022-10-13 21:14:42 +02:00
ed
1027378bda language + cleanup 2022-10-13 20:43:30 +02:00
ed
e979d30659 audioplayer: transcode wav to opus 2022-10-13 20:26:43 +02:00
ed
574db704cc packaging 2022-10-13 20:24:45 +02:00
ed
fdb969ea89 explain why extractall is safe to use 2022-10-11 17:44:38 +02:00
ed
08977854b3 a e s t h e t i c 2022-10-09 22:56:27 +02:00
ed
cecac64b68 v1.4.5 2022-10-09 11:19:40 +02:00
ed
7dabdade2a v1.4.4 2022-10-09 01:08:31 +02:00
ed
e788f098e2 dont fallback to icons for waveforms 2022-10-09 00:38:56 +02:00
ed
69406d4344 readme + better window title 2022-10-09 00:04:02 +02:00
ed
d16dd26c65 misc 2022-10-08 21:09:04 +02:00
ed
12219c1bea more fun with symlinks 2022-10-08 21:08:51 +02:00
ed
118bdcc26e 120x faster folder moves/renames 2022-10-08 19:11:03 +02:00
ed
78fa96f0f4 add unpost sanchk 2022-10-08 18:23:41 +02:00
ed
c7deb63a04 actually thats just an android-firefox bug 2022-10-08 17:52:29 +02:00
ed
4f811eb9e9 hmac cache limit + android ux:
onscroll doesnt trigger so files dont load in
2022-10-08 17:46:04 +02:00
ed
0b265bd673 naming is hard 2022-10-08 16:34:04 +02:00
ed
ee67fabbeb update readme 2022-10-08 14:25:13 +02:00
ed
b213de7e62 update readme + tests 2022-10-08 14:18:52 +02:00
ed
7c01505750 phone ux 2022-10-08 14:11:25 +02:00
ed
ae28dfd020 tweaks 2022-10-08 02:05:15 +02:00
ed
2a5a4e785f include filekeys in unpost list 2022-10-08 01:18:27 +02:00
ed
d8bddede6a new permission G returns filekey on write-only uploads 2022-10-08 01:17:41 +02:00
ed
b8a93e74bf fix default upload expiration + ux 2022-10-07 21:38:01 +02:00
ed
e60ec94d35 draw qr-code as ansi colors 2022-10-07 01:04:23 +02:00
ed
84af5fd0a3 scale qr-code to fit console size 2022-10-07 00:59:44 +02:00
ed
dbb3edec77 print qr-code on startup 2022-10-07 00:47:26 +02:00
ed
d284b46a3e rice 2022-10-06 23:40:06 +02:00
ed
9fcb4d222b reserve names to avoid ie11 pollution 2022-10-06 01:33:34 +02:00
ed
d0bb1ad141 v1.4.3 2022-09-26 22:37:54 +02:00
ed
b299aaed93 fix some cases of deleted files not being forgotten 2022-09-26 22:19:46 +02:00
ed
abb3224cc5 option to save a copy of corrupted uploads 2022-09-26 22:01:49 +02:00
ed
1c66d06702 cleanup versionchecks 2022-09-25 21:31:47 +02:00
ed
e00e80ae39 v1.4.2 2022-09-25 14:36:10 +02:00
ed
4f4f106c48 add ability to forget uploads by deleting the files 2022-09-25 14:24:01 +02:00
ed
a286cc9d55 fix printing big unicode messages 2022-09-25 14:04:35 +02:00
ed
53bb1c719b fix huge-filename trunc on ubuntu-20.04 zfs 2022-09-25 14:00:11 +02:00
ed
98d5aa17e2 notes on dealing with bitflips 2022-09-24 22:41:00 +02:00
ed
aaaa80e4b8 v1.4.1 2022-09-24 14:45:50 +02:00
ed
e70e926a40 support up2k uploads from old browsertabs 2022-09-24 14:35:51 +02:00
ed
e80c1f6d59 mention how ffmpeg was built 2022-09-24 00:05:47 +02:00
160 changed files with 13717 additions and 2525 deletions

4
.gitignore vendored
View File

@@ -22,8 +22,12 @@ copyparty.egg-info/
*.bak
# derived
copyparty/res/COPYING.txt
copyparty/web/deps/
srv/
scripts/docker/i/
contrib/package/arch/pkg/
contrib/package/arch/src/
# state/logs
up.*.txt

1
.vscode/launch.json vendored
View File

@@ -8,6 +8,7 @@
"module": "copyparty",
"console": "integratedTerminal",
"cwd": "${workspaceFolder}",
"justMyCode": false,
"args": [
//"-nw",
"-ed",

8
.vscode/launch.py vendored Normal file → Executable file
View File

@@ -1,3 +1,5 @@
#!/usr/bin/env python3
# takes arguments from launch.json
# is used by no_dbg in tasks.json
# launches 10x faster than mspython debugpy
@@ -9,15 +11,15 @@ import sys
print(sys.executable)
import json5
import shlex
import jstyleson
import subprocess as sp
with open(".vscode/launch.json", "r", encoding="utf-8") as f:
tj = f.read()
oj = jstyleson.loads(tj)
oj = json5.loads(tj)
argv = oj["configurations"][0]["args"]
try:
@@ -28,6 +30,8 @@ except:
argv = [os.path.expanduser(x) if x.startswith("~") else x for x in argv]
argv += sys.argv[1:]
if re.search(" -j ?[0-9]", " ".join(argv)):
argv = [sys.executable, "-m", "copyparty"] + argv
sp.check_call(argv)

View File

@@ -52,9 +52,11 @@
"--disable=missing-module-docstring",
"--disable=missing-class-docstring",
"--disable=missing-function-docstring",
"--disable=import-outside-toplevel",
"--disable=wrong-import-position",
"--disable=raise-missing-from",
"--disable=bare-except",
"--disable=broad-except",
"--disable=invalid-name",
"--disable=line-too-long",
"--disable=consider-using-f-string"
@@ -64,6 +66,7 @@
"editor.formatOnSave": true,
"[html]": {
"editor.formatOnSave": false,
"editor.autoIndent": "keep",
},
"[css]": {
"editor.formatOnSave": false,

640
README.md
View File

@@ -1,6 +1,6 @@
# ⇆🎉 copyparty
* http file sharing hub (py2/py3) [(on PyPI)](https://pypi.org/project/copyparty/)
* portable file sharing hub (py2/py3) [(on PyPI)](https://pypi.org/project/copyparty/)
* MIT-Licensed, 2019-05-26, ed @ irc.rizon.net
@@ -8,9 +8,9 @@
turn your phone or raspi into a portable file server with resumable uploads/downloads using *any* web browser
* server only needs `py2.7` or `py3.3+`, all dependencies optional
* server only needs Python (`2.7` or `3.3+`), all dependencies optional
* browse/upload with [IE4](#browser-support) / netscape4.0 on win3.11 (heh)
* *resumable* uploads need `firefox 34+` / `chrome 41+` / `safari 7+`
* protocols: [http](#the-browser) // [ftp](#ftp-server) // [webdav](#webdav-server) // [smb/cifs](#smb-server)
try the **[read-only demo server](https://a.ocv.me/pub/demo/)** 👀 running from a basement in finland
@@ -30,16 +30,17 @@ try the **[read-only demo server](https://a.ocv.me/pub/demo/)** 👀 running fro
* [quickstart](#quickstart) - download **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** and you're all set!
* [on servers](#on-servers) - you may also want these, especially on servers
* [on debian](#on-debian) - recommended additional steps on debian
* [notes](#notes) - general notes
* [status](#status) - feature summary
* [features](#features)
* [testimonials](#testimonials) - small collection of user feedback
* [motivations](#motivations) - project goals / philosophy
* [future plans](#future-plans) - some improvement ideas
* [notes](#notes) - general notes
* [bugs](#bugs)
* [general bugs](#general-bugs)
* [not my bugs](#not-my-bugs)
* [breaking changes](#breaking-changes) - upgrade notes
* [FAQ](#FAQ) - "frequently" asked questions
* [accounts and volumes](#accounts-and-volumes) - per-folder, per-user permissions
* [shadowing](#shadowing) - hiding specific subfolders
* [the browser](#the-browser) - accessing a copyparty server using a web-browser
* [tabs](#tabs) - the main tabs in the ui
* [hotkeys](#hotkeys) - the browser has the following hotkeys
@@ -56,7 +57,14 @@ try the **[read-only demo server](https://a.ocv.me/pub/demo/)** 👀 running fro
* [other tricks](#other-tricks)
* [searching](#searching) - search by size, date, path/name, mp3-tags, ...
* [server config](#server-config) - using arguments or config files, or a mix of both
* [ftp-server](#ftp-server) - an FTP server can be started using `--ftp 3921`
* [zeroconf](#zeroconf) - announce enabled services on the LAN ([pic](https://user-images.githubusercontent.com/241032/215344737-0eae8d98-9496-4256-9aa8-cd2f6971810d.png))
* [mdns](#mdns) - LAN domain-name and feature announcer
* [ssdp](#ssdp) - windows-explorer announcer
* [qr-code](#qr-code) - print a qr-code [(screenshot)](https://user-images.githubusercontent.com/241032/194728533-6f00849b-c6ac-43c6-9359-83e454d11e00.png) for quick access
* [ftp server](#ftp-server) - an FTP server can be started using `--ftp 3921`
* [webdav server](#webdav-server) - with read-write support
* [connecting to webdav from windows](#connecting-to-webdav-from-windows) - using the GUI
* [smb server](#smb-server) - unsafe, slow, not recommended for wan
* [file indexing](#file-indexing) - enables dedup and music search ++
* [exclude-patterns](#exclude-patterns) - to save some time
* [filesystem guards](#filesystem-guards) - avoid traversing into other filesystems
@@ -66,56 +74,52 @@ try the **[read-only demo server](https://a.ocv.me/pub/demo/)** 👀 running fro
* [other flags](#other-flags)
* [database location](#database-location) - in-volume (`.hist/up2k.db`, default) or somewhere else
* [metadata from audio files](#metadata-from-audio-files) - set `-e2t` to index tags on upload
* [file parser plugins](#file-parser-plugins) - provide custom parsers to index additional tags, also see [./bin/mtag/README.md](./bin/mtag/README.md)
* [upload events](#upload-events) - trigger a script/program on each upload
* [file parser plugins](#file-parser-plugins) - provide custom parsers to index additional tags
* [event hooks](#event-hooks) - trigger a program on uploads, renames etc ([examples](./bin/hooks/))
* [upload events](#upload-events) - the older, more powerful approach ([examples](./bin/mtag/))
* [hiding from google](#hiding-from-google) - tell search engines you dont wanna be indexed
* [themes](#themes)
* [complete examples](#complete-examples)
* [reverse-proxy](#reverse-proxy) - running copyparty next to other websites
* [browser support](#browser-support) - TLDR: yes
* [client examples](#client-examples) - interact with copyparty using non-browser clients
* [up2k](#up2k) - quick outline of the up2k protocol, see [uploading](#uploading) for the web-client
* [why chunk-hashes](#why-chunk-hashes) - a single sha512 would be better, right?
* [mount as drive](#mount-as-drive) - a remote copyparty server as a local filesystem
* [performance](#performance) - defaults are usually fine - expect `8 GiB/s` download, `1 GiB/s` upload
* [client-side](#client-side) - when uploading files
* [security](#security) - some notes on hardening
* [gotchas](#gotchas) - behavior that might be unexpected
* [cors](#cors) - cross-site request config
* [recovering from crashes](#recovering-from-crashes)
* [client crashes](#client-crashes)
* [frefox wsod](#frefox-wsod) - firefox 87 can crash during uploads
* [HTTP API](#HTTP-API)
* [read](#read)
* [write](#write)
* [admin](#admin)
* [general](#general)
* [HTTP API](#HTTP-API) - see [devnotes](#./docs/devnotes.md#http-api)
* [dependencies](#dependencies) - mandatory deps
* [optional dependencies](#optional-dependencies) - install these to enable bonus features
* [install recommended deps](#install-recommended-deps)
* [optional gpl stuff](#optional-gpl-stuff)
* [sfx](#sfx) - the self-contained "binary"
* [sfx repack](#sfx-repack) - reduce the size of an sfx by removing features
* [copyparty.exe](#copypartyexe) - download [copyparty.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty.exe) or [copyparty64.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty64.exe)
* [install on android](#install-on-android)
* [reporting bugs](#reporting-bugs) - ideas for context to include in bug reports
* [building](#building)
* [dev env setup](#dev-env-setup)
* [just the sfx](#just-the-sfx)
* [complete release](#complete-release)
* [todo](#todo) - roughly sorted by priority
* [discarded ideas](#discarded-ideas)
* [devnotes](#devnotes) - for build instructions etc, see [./docs/devnotes.md](./docs/devnotes.md)
## quickstart
download **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** and you're all set!
* or install through pypi (python3 only): `python3 -m pip install --user -U copyparty`
* or if you cannot install python, you can use [copyparty.exe](#copypartyexe) instead
* or if you prefer to [use docker](./scripts/docker/) 🐋 you can do that too
running the sfx without arguments (for example doubleclicking it on Windows) will give everyone read/write access to the current folder; you may want [accounts and volumes](#accounts-and-volumes)
some recommended options:
* `-e2dsa` enables general [file indexing](#file-indexing)
* `-e2ts` enables audio metadata indexing (needs either FFprobe or Mutagen), see [optional dependencies](#optional-dependencies)
* `-e2ts` enables audio metadata indexing (needs either FFprobe or Mutagen), see [optional dependencies](#optional-dependencies) to enable thumbnails and more
* `-v /mnt/music:/music:r:rw,foo -a foo:bar` shares `/mnt/music` as `/music`, `r`eadable by anyone, and read-write for user `foo`, password `bar`
* replace `:r:rw,foo` with `:r,foo` to only make the folder readable by `foo` and nobody else
* see [accounts and volumes](#accounts-and-volumes) for the syntax and other permissions (`r`ead, `w`rite, `m`ove, `d`elete, `g`et)
* `--ls '**,*,ln,p,r'` to crash on startup if any of the volumes contain a symlink which point outside the volume, as that could give users unintended access (see `--help-ls`)
* see [accounts and volumes](#accounts-and-volumes) for the syntax and other permissions (`r`ead, `w`rite, `m`ove, `d`elete, `g`et, up`G`et)
### on servers
@@ -124,8 +128,16 @@ you may also want these, especially on servers:
* [contrib/systemd/copyparty.service](contrib/systemd/copyparty.service) to run copyparty as a systemd service
* [contrib/systemd/prisonparty.service](contrib/systemd/prisonparty.service) to run it in a chroot (for extra security)
* [contrib/nginx/copyparty.conf](contrib/nginx/copyparty.conf) to reverse-proxy behind nginx (for better https)
* [contrib/nginx/copyparty.conf](contrib/nginx/copyparty.conf) to [reverse-proxy](#reverse-proxy) behind nginx (for better https)
and remember to open the ports you want; here's a complete example including every feature copyparty has to offer:
```
firewall-cmd --permanent --add-port={80,443,3921,3923,3945,3990}/tcp # --zone=libvirt
firewall-cmd --permanent --add-port=12000-12099/tcp --permanent # --zone=libvirt
firewall-cmd --permanent --add-port={1900,5353}/udp # --zone=libvirt
firewall-cmd --reload
```
(1900:ssdp, 3921:ftp, 3923:http/https, 3945:smb, 3990:ftps, 5353:mdns, 12000:passive-ftp)
### on debian
@@ -140,33 +152,22 @@ recommended additional steps on debian which enable audio metadata and thumbnai
(skipped `pyheif-pillow-opener` because apparently debian is too old to build it)
## notes
general notes:
* paper-printing is affected by dark/light-mode! use lightmode for color, darkmode for grayscale
* because no browsers currently implement the media-query to do this properly orz
browser-specific:
* iPhone/iPad: use Firefox to download files
* Android-Chrome: increase "parallel uploads" for higher speed (android bug)
* Android-Firefox: takes a while to select files (their fix for ☝️)
* Desktop-Firefox: ~~may use gigabytes of RAM if your files are massive~~ *seems to be OK now*
* Desktop-Firefox: may stop you from deleting files you've uploaded until you visit `about:memory` and click `Minimize memory usage`
## status
feature summary
## features
* backend stuff
*sanic multipart parser
* ☑ multiprocessing (actual multithreading)
*IPv6
*[multiprocessing](#performance) (actual multithreading)
* ☑ volumes (mountpoints)
* ☑ [accounts](#accounts-and-volumes)
* ☑ [ftp-server](#ftp-server)
* ☑ [ftp server](#ftp-server)
* ☑ [webdav server](#webdav-server)
* ☑ [smb/cifs server](#smb-server)
* ☑ [qr-code](#qr-code) for quick access
* ☑ [upnp / zeroconf / mdns / ssdp](#zeroconf)
* upload
* ☑ basic: plain multipart, ie6 support
* ☑ [up2k](#uploading): js, resumable, multithreaded
* unaffected by cloudflare's max-upload-size (100 MiB)
* ☑ stash: simple PUT filedropper
* ☑ [unpost](#unpost): undo/delete accidental uploads
* ☑ [self-destruct](#self-destruct) (specified server-side or client-side)
@@ -174,11 +175,11 @@ feature summary
* download
* ☑ single files in browser
* ☑ [folders as zip / tar files](#zip-downloads)
* ☑ [FUSE client](https://github.com/9001/copyparty/tree/hovudstraum/bin#copyparty-fusepy) (read-only)
* ☑ [FUSE client](https://github.com/9001/copyparty/tree/hovudstraum/bin#partyfusepy) (read-only)
* browser
* ☑ [navpane](#navpane) (directory tree sidebar)
* ☑ file manager (cut/paste, delete, [batch-rename](#batch-rename))
* ☑ audio player (with OS media controls and opus transcoding)
* ☑ audio player (with [OS media controls](https://user-images.githubusercontent.com/241032/215347492-b4250797-6c90-4e09-9a4c-721edf2fb15c.png) and opus transcoding)
* ☑ image gallery with webm player
* ☑ textfile browser with syntax hilighting
* ☑ [thumbnails](#thumbnails)
@@ -200,7 +201,7 @@ feature summary
small collection of user feedback
`good enough`, `surprisingly correct`, `certified good software`, `just works`, `why`
`good enough`, `surprisingly correct`, `certified good software`, `just works`, `why`, `wow this is better than nextcloud`
# motivations
@@ -209,8 +210,7 @@ project goals / philosophy
* inverse linux philosophy -- do all the things, and do an *okay* job
* quick drop-in service to get a lot of features in a pinch
* there are probably [better alternatives](https://github.com/awesome-selfhosted/awesome-selfhosted) if you have specific/long-term needs
* but the resumable multithreaded uploads are p slick ngl
* some of [the alternatives](./docs/versus.md) might be a better fit for you
* run anywhere, support everything
* as many web-browsers and python versions as possible
* every browser should at least be able to browse, download, upload files
@@ -223,20 +223,21 @@ project goals / philosophy
* no build steps; modify the js/python without needing node.js or anything like that
## future plans
## notes
some improvement ideas
general notes:
* paper-printing is affected by dark/light-mode! use lightmode for color, darkmode for grayscale
* because no browsers currently implement the media-query to do this properly orz
* the JS is a mess -- a preact rewrite would be nice
* preferably without build dependencies like webpack/babel/node.js, maybe a python thing to assemble js files into main.js
* good excuse to look at using virtual lists (browsers start to struggle when folders contain over 5000 files)
* the UX is a mess -- a proper design would be nice
* very organic (much like the python/js), everything was an afterthought
* true for both the layout and the visual flair
* something like the tron board-room ui (or most other hollywood ones, like ironman) would be :100:
* some of the python files are way too big
* `up2k.py` ended up doing all the file indexing / db management
* `httpcli.py` should be separated into modules in general
browser-specific:
* iPhone/iPad: use Firefox to download files
* Android-Chrome: increase "parallel uploads" for higher speed (android bug)
* Android-Firefox: takes a while to select files (their fix for ☝️)
* Desktop-Firefox: ~~may use gigabytes of RAM if your files are massive~~ *seems to be OK now*
* Desktop-Firefox: [may stop you from unplugging USB flashdrives](https://bugzilla.mozilla.org/show_bug.cgi?id=1792598) until you visit `about:memory` and click `Minimize memory usage`
server-os-specific:
* RHEL8 / Rocky8: you can run copyparty using `/usr/libexec/platform-python`
# bugs
@@ -251,23 +252,15 @@ some improvement ideas
* Windows: if the `up2k.db` (filesystem index) is on a samba-share or network disk, you'll get unpredictable behavior if the share is disconnected for a bit
* use `--hist` or the `hist` volflag (`-v [...]:c,hist=/tmp/foo`) to place the db on a local disk instead
* all volumes must exist / be available on startup; up2k (mtp especially) gets funky otherwise
* [the database can get stuck](https://github.com/9001/copyparty/issues/10)
* has only happened once but that is once too many
* luckily not dangerous for file integrity and doesn't really stop uploads or anything like that
* but would really appreciate some logs if anyone ever runs into it again
* probably more, pls let me know
## not my bugs
* [Chrome issue 1317069](https://bugs.chromium.org/p/chromium/issues/detail?id=1317069) -- if you try to upload a folder which contains symlinks by dragging it into the browser, the symlinked files will not get uploaded
* [Chrome issue 1354816](https://bugs.chromium.org/p/chromium/issues/detail?id=1354816) -- chrome may eat all RAM uploading over plaintext http with `mt` enabled
* [Chrome issue 1352210](https://bugs.chromium.org/p/chromium/issues/detail?id=1352210) -- plaintext http may be faster at filehashing than https (but also extremely CPU-intensive)
* more amusingly, [Chrome issue 1354800](https://bugs.chromium.org/p/chromium/issues/detail?id=1354800) -- chrome may eat all RAM uploading in general (altho you probably won't run into this one)
* [Chrome issue 1352210](https://bugs.chromium.org/p/chromium/issues/detail?id=1352210) -- plaintext http may be faster at filehashing than https (but also extremely CPU-intensive and likely to run into the above gc bugs)
* [Firefox issue 1790500](https://bugzilla.mozilla.org/show_bug.cgi?id=1790500) -- sometimes forgets to close filedescriptors during upload so the browser can crash after ~4000 files
* [Firefox issue 1790500](https://bugzilla.mozilla.org/show_bug.cgi?id=1790500) -- entire browser can crash after uploading ~4000 small files
* iPhones: the volume control doesn't work because [apple doesn't want it to](https://developer.apple.com/library/archive/documentation/AudioVideo/Conceptual/Using_HTML5_Audio_Video/Device-SpecificConsiderations/Device-SpecificConsiderations.html#//apple_ref/doc/uid/TP40009523-CH5-SW11)
* *future workaround:* enable the equalizer, make it all-zero, and set a negative boost to reduce the volume
@@ -286,6 +279,18 @@ some improvement ideas
* due to snap security policies -- see `snap connections firefox` for the allowlist, `removable-media` permits all of `/mnt` and `/media` apparently
# breaking changes
upgrade notes
* `1.6.0` (2023-01-29):
* http-api: delete/move is now `POST` instead of `GET`
* everything other than `GET` and `HEAD` must pass [cors validation](#cors)
* `1.5.0` (2022-12-03): [new chunksize formula](https://github.com/9001/copyparty/commit/54e1c8d261df) for files larger than 128 GiB
* **users:** upgrade to the latest [cli uploader](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) if you use that
* **devs:** update third-party up2k clients (if those even exist)
# FAQ
"frequently" asked questions
@@ -295,13 +300,14 @@ some improvement ideas
* you can also do this with linux filesystem permissions; `chmod 111 music` will make it possible to access files and folders inside the `music` folder but not list the immediate contents -- also works with other software, not just copyparty
* can I make copyparty download a file to my server if I give it a URL?
* not really, but there is a [terrible hack](https://github.com/9001/copyparty/blob/hovudstraum/bin/mtag/wget.py) which makes it possible
* yes, using [hooks](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/wget.py)
# accounts and volumes
per-folder, per-user permissions - if your setup is getting complex, consider making a [config file](./docs/example.conf) instead of using arguments
* much easier to manage, and you can modify the config at runtime with `systemctl reload copyparty` or more conveniently using the `[reload cfg]` button in the control-panel (if logged in as admin)
* changes to the `[global]` config section requires a restart to take effect
a quick summary can be seen using `--help-accounts`
@@ -318,6 +324,7 @@ permissions:
* `m` (move): move files/folders *from* this folder
* `d` (delete): delete files/folders
* `g` (get): only download files, cannot see folder contents or zip/tar
* `G` (upget): same as `g` except uploaders get to see their own filekeys (see `fk` in examples below)
examples:
* add accounts named u1, u2, u3 with passwords p1, p2, p3: `-a u1:p1 -a u2:p2 -a u3:p3`
@@ -328,14 +335,23 @@ examples:
* unauthorized users accessing the webroot can see that the `inc` folder exists, but cannot open it
* `u1` can open the `inc` folder, but cannot see the contents, only upload new files to it
* `u2` can browse it and move files *from* `/inc` into any folder where `u2` has write-access
* make folder `/mnt/ss` available at `/i`, read-write for u1, get-only for everyone else, and enable accesskeys: `-v /mnt/ss:i:rw,u1:g:c,fk=4`
* `c,fk=4` sets the `fk` volflag to 4, meaning each file gets a 4-character accesskey
* `u1` can upload files, browse the folder, and see the generated accesskeys
* other users cannot browse the folder, but can access the files if they have the full file URL with the accesskey
* make folder `/mnt/ss` available at `/i`, read-write for u1, get-only for everyone else, and enable filekeys: `-v /mnt/ss:i:rw,u1:g:c,fk=4`
* `c,fk=4` sets the `fk` (filekey) volflag to 4, meaning each file gets a 4-character accesskey
* `u1` can upload files, browse the folder, and see the generated filekeys
* other users cannot browse the folder, but can access the files if they have the full file URL with the filekey
* replacing the `g` permission with `wg` would let anonymous users upload files, but not see the required filekey to access it
* replacing the `g` permission with `wG` would let anonymous users upload files, receiving a working direct link in return
anyone trying to bruteforce a password gets banned according to `--ban-pw`; default is 24h ban for 9 failed attempts in 1 hour
## shadowing
hiding specific subfolders by mounting another volume on top of them
for example `-v /mnt::r -v /var/empty:web/certs:r` mounts the server folder `/mnt` as the webroot, but another volume is mounted at `/web/certs` -- so visitors can only see the contents of `/mnt` and `/mnt/web` (at URLs `/` and `/web`), but not `/mnt/web/certs` because URL `/web/certs` is mapped to `/var/empty`
# the browser
accessing a copyparty server using a web-browser
@@ -359,6 +375,7 @@ the main tabs in the ui
## hotkeys
the browser has the following hotkeys (always qwerty)
* `?` show hotkeys help
* `B` toggle breadcrumbs / [navpane](#navpane)
* `I/K` prev/next folder
* `M` parent folder (or unexpand current)
@@ -366,8 +383,10 @@ the browser has the following hotkeys (always qwerty)
* `G` toggle list / [grid view](#thumbnails) -- same as `田` bottom-right
* `T` toggle thumbnails / icons
* `ESC` close various things
* `ctrl-K` delete selected files/folders
* `ctrl-X` cut selected files/folders
* `ctrl-V` paste
* `Y` download selected files
* `F2` [rename](#batch-rename) selected file/folder
* when a file/folder is selected (in not-grid-view):
* `Up/Down` move cursor
@@ -670,12 +689,57 @@ for the above example to work, add the commandline argument `-e2ts` to also scan
# server config
using arguments or config files, or a mix of both:
* config files (`-c some.conf`) can set additional commandline arguments; see [./docs/example.conf](docs/example.conf)
* config files (`-c some.conf`) can set additional commandline arguments; see [./docs/example.conf](docs/example.conf) and [./docs/example2.conf](docs/example2.conf)
* `kill -s USR1` (same as `systemctl reload copyparty`) to reload accounts and volumes from config files without restarting
* or click the `[reload cfg]` button in the control-panel when logged in as admin
* changes to the `[global]` config section requires a restart to take effect
## ftp-server
## zeroconf
announce enabled services on the LAN ([pic](https://user-images.githubusercontent.com/241032/215344737-0eae8d98-9496-4256-9aa8-cd2f6971810d.png)) -- `-z` enables both [mdns](#mdns) and [ssdp](#ssdp)
* `--z-on` / `--z-off`' limits the feature to certain networks
### mdns
LAN domain-name and feature announcer
uses [multicast dns](https://en.wikipedia.org/wiki/Multicast_DNS) to give copyparty a domain which any machine on the LAN can use to access it
all enabled services ([webdav](#webdav-server), [ftp](#ftp-server), [smb](#smb-server)) will appear in mDNS-aware file managers (KDE, gnome, macOS, ...)
the domain will be http://partybox.local if the machine's hostname is `partybox` unless `--name` specifies soemthing else
### ssdp
windows-explorer announcer
uses [ssdp](https://en.wikipedia.org/wiki/Simple_Service_Discovery_Protocol) to make copyparty appear in the windows file explorer on all machines on the LAN
doubleclicking the icon opens the "connect" page which explains how to mount copyparty as a local filesystem
if copyparty does not appear in windows explorer, use `--zsv` to see why:
* maybe the discovery multicast was sent from an IP which does not intersect with the server subnets
## qr-code
print a qr-code [(screenshot)](https://user-images.githubusercontent.com/241032/194728533-6f00849b-c6ac-43c6-9359-83e454d11e00.png) for quick access, great between phones on android hotspots which keep changing the subnet
* `--qr` enables it
* `--qrs` does https instead of http
* `--qrl lootbox/?pw=hunter2` appends to the url, linking to the `lootbox` folder with password `hunter2`
* `--qrz 1` forces 1x zoom instead of autoscaling to fit the terminal size
* 1x may render incorrectly on some terminals/fonts, but 2x should always work
it uses the server hostname if [mdns](#mdns) is enbled, otherwise it'll use your external ip (default route) unless `--qri` specifies a specific ip-prefix or domain
## ftp server
an FTP server can be started using `--ftp 3921`, and/or `--ftps` for explicit TLS (ftpes)
@@ -685,6 +749,79 @@ an FTP server can be started using `--ftp 3921`, and/or `--ftps` for explicit T
* runs in active mode by default, you probably want `--ftp-pr 12000-13000`
* if you enable both `ftp` and `ftps`, the port-range will be divided in half
* some older software (filezilla on debian-stable) cannot passive-mode with TLS
* login with any username + your password, or put your password in the username field
## webdav server
with read-write support, supports winXP and later, macos, nautilus/gvfs
click the [connect](http://127.0.0.1:3923/?hc) button in the control-panel to see connection instructions for windows, linux, macos
general usage:
* login with any username + your password, or put your password in the username field (password field can be empty/whatever)
on macos, connect from finder:
* [Go] -> [Connect to Server...] -> http://192.168.123.1:3923/
### connecting to webdav from windows
using the GUI (winXP or later):
* rightclick [my computer] -> [map network drive] -> Folder: `http://192.168.123.1:3923/`
* on winXP only, click the `Sign up for online storage` hyperlink instead and put the URL there
* providing your password as the username is recommended; the password field can be anything or empty
known client bugs:
* win7+ doesn't actually send the password to the server when reauthenticating after a reboot unless you first try to login with an incorrect password and then switch to the correct password
* or just type your password into the username field instead to get around it entirely
* connecting to a folder which allows anonymous read will make writing impossible, as windows has decided it doesn't need to login
* workaround: connect twice; first to a folder which requires auth, then to the folder you actually want, and leave both of those mounted
* win7+ may open a new tcp connection for every file and sometimes forgets to close them, eventually needing a reboot
* maybe NIC-related (??), happens with win10-ltsc on e1000e but not virtio
* windows cannot access folders which contain filenames with invalid unicode or forbidden characters (`<>:"/\|?*`), or names ending with `.`
* winxp cannot show unicode characters outside of *some range*
* latin-1 is fine, hiragana is not (not even as shift-jis on japanese xp)
## smb server
unsafe, slow, not recommended for wan, enable with `--smb` for read-only or `--smbw` for read-write
click the [connect](http://127.0.0.1:3923/?hc) button in the control-panel to see connection instructions for windows, linux, macos
dependencies: `python3 -m pip install --user -U impacket==0.10.0`
* newer versions of impacket will hopefully work just fine but there is monkeypatching so maybe not
some **BIG WARNINGS** specific to SMB/CIFS, in decreasing importance:
* not entirely confident that read-only is read-only
* the smb backend is not fully integrated with vfs, meaning there could be security issues (path traversal). Please use `--smb-port` (see below) and [prisonparty](./bin/prisonparty.sh)
* account passwords work per-volume as expected, but account permissions are coalesced; all accounts have read-access to all volumes, and if a single account has write-access to some volume then all other accounts also do
* if no accounts have write-access to a specific volume, or if `--smbw` is not set, then writing to that volume from smb *should* be impossible
* will be fixed once [impacket v0.11.0](https://github.com/SecureAuthCorp/impacket/commit/d923c00f75d54b972bca573a211a82f09b55261a) is released
* [shadowing](#shadowing) probably works as expected but no guarantees
and some minor issues,
* clients only see the first ~400 files in big folders; [impacket#1433](https://github.com/SecureAuthCorp/impacket/issues/1433)
* hot-reload of server config (`/?reload=cfg`) does not include the `[global]` section (commandline args)
* listens on the first IPv4 `-i` interface only (default = :: = 0.0.0.0 = all)
* login doesn't work on winxp, but anonymous access is ok -- remove all accounts from copyparty config for that to work
* win10 onwards does not allow connecting anonymously / without accounts
* on windows, creating a new file through rightclick --> new --> textfile throws an error due to impacket limitations -- hit OK and F5 to get your file
* python3 only
* slow
known client bugs:
* on win7 only, `--smb1` is much faster than smb2 (default) because it keeps rescanning folders on smb2
* however smb1 is buggy and is not enabled by default on win10 onwards
* windows cannot access folders which contain filenames with invalid unicode or forbidden characters (`<>:"/\|?*`), or names ending with `.`
the smb protocol listens on TCP port 445, which is a privileged port on linux and macos, which would require running copyparty as root. However, this can be avoided by listening on another port using `--smb-port 3945` and then using NAT to forward the traffic from 445 to there;
* on linux: `iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 445 -j REDIRECT --to-port 3945`
authenticate with one of the following:
* username `$username`, password `$password`
* username `$password`, password `k`
## file indexing
@@ -703,6 +840,7 @@ through arguments:
* `-e2v` verfies file integrity at startup, comparing hashes from the db
* `-e2vu` patches the database with the new hashes from the filesystem
* `-e2vp` panics and kills copyparty instead
* `--xlink` enables deduplication across volumes
the same arguments can be set as volflags, in addition to `d2d`, `d2ds`, `d2t`, `d2ts`, `d2v` for disabling:
* `-v ~/music::r:c,e2dsa,e2tsr` does a full reindex of everything on startup
@@ -795,6 +933,8 @@ some examples,
## other flags
* `:c,magic` enables filetype detection for nameless uploads, same as `--magic`
* needs https://pypi.org/project/python-magic/ `python3 -m pip install --user -U python-magic`
* on windows grab this instead `python3 -m pip install --user -U python-magic-bin`
## database location
@@ -841,7 +981,7 @@ see the beautiful mess of a dictionary in [mtag.py](https://github.com/9001/copy
## file parser plugins
provide custom parsers to index additional tags, also see [./bin/mtag/README.md](./bin/mtag/README.md)
provide custom parsers to index additional tags, also see [./bin/mtag/README.md](./bin/mtag/README.md)
copyparty can invoke external programs to collect additional metadata for files using `mtp` (either as argument or volflag), there is a default timeout of 60sec, and only files which contain audio get analyzed by default (see ay/an/ad below)
@@ -863,9 +1003,18 @@ copyparty can invoke external programs to collect additional metadata for files
if something doesn't work, try `--mtag-v` for verbose error messages
## upload events
## event hooks
trigger a script/program on each upload like so:
trigger a program on uploads, renames etc ([examples](./bin/hooks/))
you can set hooks before and/or after an event happens, and currently you can hook uploads, moves/renames, and deletes
there's a bunch of flags and stuff, see `--help-hooks`
### upload events
the older, more powerful approach ([examples](./bin/mtag/)):
```
-v /mnt/inc:inc:w:c,mte=+x1:c,mtp=x1=ad,kn,/usr/bin/notify-send
@@ -875,11 +1024,12 @@ so filesystem location `/mnt/inc` shared at `/inc`, write-only for everyone, app
that'll run the command `notify-send` with the path to the uploaded file as the first and only argument (so on linux it'll show a notification on-screen)
note that it will only trigger on new unique files, not dupes
note that this is way more complicated than the new [event hooks](#event-hooks) but this approach has the following advantages:
* non-blocking and multithreaded; doesn't hold other uploads back
* you get access to tags from FFmpeg and other mtp parsers
* only trigger on new unique files, not dupes
and it will occupy the parsing threads, so fork anything expensive (or set `kn` to have copyparty fork it for you) -- otoh if you want to intentionally queue/singlethread you can combine it with `--mtag-mt 1`
if this becomes popular maybe there should be a less janky way to do it actually
note that it will occupy the parsing threads, so fork anything expensive (or set `kn` to have copyparty fork it for you) -- otoh if you want to intentionally queue/singlethread you can combine it with `--mtag-mt 1`
## hiding from google
@@ -931,6 +1081,21 @@ see the top of [./copyparty/web/browser.css](./copyparty/web/browser.css) where
`-lo log/cpp-%Y-%m%d-%H%M%S.txt.xz`
## reverse-proxy
running copyparty next to other websites hosted on an existing webserver such as nginx or apache
you can either:
* give copyparty its own domain or subdomain (recommended)
* or do location-based proxying, using `--rp-loc=/stuff` to tell copyparty where it is mounted -- has a slight performance cost and higher chance of bugs
* if copyparty says `incorrect --rp-loc or webserver config; expected vpath starting with [...]` it's likely because the webserver is stripping away the proxy location from the request URLs -- see the `ProxyPass` in the apache example below
example webserver configs:
* [nginx config](contrib/nginx/copyparty.conf) -- entire domain/subdomain
* [apache2 config](contrib/apache/copyparty.conf) -- location-based
# browser support
TLDR: yes
@@ -990,11 +1155,11 @@ interact with copyparty using non-browser clients
* curl/wget: upload some files (post=file, chunk=stdin)
* `post(){ curl -F act=bput -F f=@"$1" http://127.0.0.1:3923/?pw=wark;}`
`post movie.mkv`
* `post(){ curl -b cppwd=wark -H rand:8 -T "$1" http://127.0.0.1:3923/;}`
* `post(){ curl -H pw:wark -H rand:8 -T "$1" http://127.0.0.1:3923/;}`
`post movie.mkv`
* `post(){ wget --header='Cookie: cppwd=wark' --post-file="$1" -O- http://127.0.0.1:3923/?raw;}`
* `post(){ wget --header='pw: wark' --post-file="$1" -O- http://127.0.0.1:3923/?raw;}`
`post movie.mkv`
* `chunk(){ curl -b cppwd=wark -T- http://127.0.0.1:3923/;}`
* `chunk(){ curl -H pw:wark -T- http://127.0.0.1:3923/;}`
`chunk <movie.mkv`
* bash: when curl and wget is not available or too boring
@@ -1002,11 +1167,13 @@ interact with copyparty using non-browser clients
* `(printf 'PUT / HTTP/1.1\r\n\r\n'; cat movie.mkv) >/dev/tcp/127.0.0.1/3923`
* python: [up2k.py](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) is a command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
* file uploads, file-search, autoresume of aborted/broken uploads
* file uploads, file-search, folder sync, autoresume of aborted/broken uploads
* can be downloaded from copyparty: controlpanel -> connect -> [up2k.py](http://127.0.0.1:3923/.cpr/a/up2k.py)
* see [./bin/README.md#up2kpy](bin/README.md#up2kpy)
* FUSE: mount a copyparty server as a local filesystem
* cross-platform python client available in [./bin/](bin/)
* can be downloaded from copyparty: controlpanel -> connect -> [partyfuse.py](http://127.0.0.1:3923/.cpr/a/partyfuse.py)
* [rclone](https://rclone.org/) as client can give ~5x performance, see [./docs/rclone.md](docs/rclone.md)
* sharex (screenshot utility): see [./contrib/sharex.sxcu](contrib/#sharexsxcu)
@@ -1016,50 +1183,27 @@ copyparty returns a truncated sha512sum of your PUT/POST as base64; you can gene
b512(){ printf "$((sha512sum||shasum -a512)|sed -E 's/ .*//;s/(..)/\\x\1/g')"|base64|tr '+/' '-_'|head -c44;}
b512 <movie.mkv
you can provide passwords using cookie `cppwd=hunter2`, as a url-param `?pw=hunter2`, or with basic-authentication (either as the username or password)
you can provide passwords using header `PW: hunter2`, cookie `cppwd=hunter2`, url-param `?pw=hunter2`, or with basic-authentication (either as the username or password)
NOTE: curl will not send the original filename if you use `-T` combined with url-params! Also, make sure to always leave a trailing slash in URLs unless you want to override the filename
# up2k
## mount as drive
quick outline of the up2k protocol, see [uploading](#uploading) for the web-client
* the up2k client splits a file into an "optimal" number of chunks
* 1 MiB each, unless that becomes more than 256 chunks
* tries 1.5M, 2M, 3, 4, 6, ... until <= 256 chunks or size >= 32M
* client posts the list of hashes, filename, size, last-modified
* server creates the `wark`, an identifier for this upload
* `sha512( salt + filesize + chunk_hashes )`
* and a sparse file is created for the chunks to drop into
* client uploads each chunk
* header entries for the chunk-hash and wark
* server writes chunks into place based on the hash
* client does another handshake with the hashlist; server replies with OK or a list of chunks to reupload
a remote copyparty server as a local filesystem; go to the control-panel and click `connect` to see a list of commands to do that
up2k has saved a few uploads from becoming corrupted in-transfer already; caught an android phone on wifi redhanded in wireshark with a bitflip, however bup with https would *probably* have noticed as well (thanks to tls also functioning as an integrity check)
alternatively, some alternatives roughly sorted by speed (unreproducible benchmark), best first:
regarding the frequent server log message during uploads;
`6.0M 106M/s 2.77G 102.9M/s n948 thank 4/0/3/1 10042/7198 00:01:09`
* this chunk was `6 MiB`, uploaded at `106 MiB/s`
* on this http connection, `2.77 GiB` transferred, `102.9 MiB/s` average, `948` chunks handled
* client says `4` uploads OK, `0` failed, `3` busy, `1` queued, `10042 MiB` total size, `7198 MiB` and `00:01:09` left
* [rclone-http](./docs/rclone.md) (25s), read-only
* [rclone-ftp](./docs/rclone.md) (47s), read/WRITE
* [rclone-webdav](./docs/rclone.md) (51s), read/WRITE
* copyparty-1.5.0's webdav server is faster than rclone-1.60.0 (69s)
* [partyfuse.py](./bin/#partyfusepy) (71s), read-only
* davfs2 (103s), read/WRITE, *very fast* on small files
* [win10-webdav](#webdav-server) (138s), read/WRITE
* [win10-smb2](#smb-server) (387s), read/WRITE
## why chunk-hashes
a single sha512 would be better, right?
this is due to `crypto.subtle` [not yet](https://github.com/w3c/webcrypto/issues/73) providing a streaming api (or the option to seed the sha512 hasher with a starting hash)
as a result, the hashes are much less useful than they could have been (search the server by sha512, provide the sha512 in the response http headers, ...)
however it allows for hashing multiple chunks in parallel, greatly increasing upload speed from fast storage (NVMe, raid-0 and such)
* both the [browser uploader](#uploading) and the [commandline one](https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py) does this now, allowing for fast uploading even from plaintext http
hashwasm would solve the streaming issue but reduces hashing speed for sha512 (xxh128 does 6 GiB/s), and it would make old browsers and [iphones](https://bugs.webkit.org/show_bug.cgi?id=228552) unsupported
* blake2 might be a better choice since xxh is non-cryptographic, but that gets ~15 MiB/s on slower androids
most clients will fail to mount the root of a copyparty server unless there is a root volume (so you get the admin-panel instead of a browser when accessing it) -- in that case, mount a specific volume instead
# performance
@@ -1072,9 +1216,10 @@ below are some tweaks roughly ordered by usefulness:
* `--http-only` or `--https-only` (unless you want to support both protocols) will reduce the delay before a new connection is established
* `--hist` pointing to a fast location (ssd) will make directory listings and searches faster when `-e2d` or `-e2t` is set
* `--no-hash .` when indexing a network-disk if you don't care about the actual filehashes and only want the names/tags searchable
* `--no-htp --hash-mt=0 --mtag-mt=1 --th-mt=1` minimizes the number of threads; can help in some eccentric environments (like the vscode debugger)
* `-j` enables multiprocessing (actual multithreading) and can make copyparty perform better in cpu-intensive workloads, for example:
* huge amount of short-lived connections
* really heavy traffic (downloads/uploads)
* simultaneous downloads and uploads saturating a 20gbps connection
...however it adds an overhead to internal communication so it might be a net loss, see if it works 4 u
@@ -1099,6 +1244,11 @@ when uploading files,
some notes on hardening
* set `--rproxy 0` if your copyparty is directly facing the internet (not through a reverse-proxy)
* cors doesn't work right otherwise
safety profiles:
* option `-s` is a shortcut to set the following options:
* `--no-thumb` disables thumbnails and audio transcoding to stop copyparty from running `FFmpeg`/`Pillow`/`VIPS` on uploaded files, which is a [good idea](https://www.cvedetails.com/vulnerability-list.php?vendor_id=3611) if anonymous upload is enabled
* `--no-mtag-ff` uses `mutagen` to grab music tags instead of `FFmpeg`, which is safer and faster but less accurate
@@ -1106,23 +1256,26 @@ some notes on hardening
* `--no-robots` and `--force-js` makes life harder for crawlers, see [hiding from google](#hiding-from-google)
* option `-ss` is a shortcut for the above plus:
* `--no-logues` and `--no-readme` disables support for readme's and prologues / epilogues in directory listings, which otherwise lets people upload arbitrary `<script>` tags
* `--unpost 0`, `--no-del`, `--no-mv` disables all move/delete support
* `--hardlink` creates hardlinks instead of symlinks when deduplicating uploads, which is less maintenance
* however note if you edit one file it will also affect the other copies
* `--vague-403` returns a "404 not found" instead of "403 forbidden" which is a common enterprise meme
* `--vague-401` returns a "404 not found" instead of "401 unauthorized" which is a common enterprise meme
* `--ban-404=50,60,1440` ban client for 1440min (24h) if they hit 50 404's in 60min
* **NB:** will ban anyone who enables up2k turbo
* `--nih` removes the server hostname from directory listings
* option `-sss` is a shortcut for the above plus:
* `--no-dav` disables webdav support
* `--no-logues` and `--no-readme` disables support for readme's and prologues / epilogues in directory listings, which otherwise lets people upload arbitrary (but sandboxed) `<script>` tags
* `-lo cpp-%Y-%m%d-%H%M%S.txt.xz` enables logging to disk
* `-ls **,*,ln,p,r` does a scan on startup for any dangerous symlinks
other misc notes:
* you can disable directory listings by giving permission `g` instead of `r`, only accepting direct URLs to files
* combine this with volflag `c,fk` to generate per-file accesskeys; users which have full read-access will then see URLs with `?k=...` appended to the end, and `g` users must provide that URL including the correct key to avoid a 404
* combine this with volflag `c,fk` to generate filekeys (per-file accesskeys); users which have full read-access will then see URLs with `?k=...` appended to the end, and `g` users must provide that URL including the correct key to avoid a 404
* the default filekey entropy is fairly small so give `--fk-salt` around 30 characters if you want filekeys longer than 16 chars
* permissions `wG` lets users upload files and receive their own filekeys, still without being able to see other uploads
## gotchas
@@ -1130,6 +1283,22 @@ other misc notes:
behavior that might be unexpected
* users without read-access to a folder can still see the `.prologue.html` / `.epilogue.html` / `README.md` contents, for the purpose of showing a description on how to use the uploader for example
* users can submit `<script>`s which autorun for other visitors in a few ways;
* uploading a `README.md` -- avoid with `--no-readme`
* renaming `some.html` to `.epilogue.html` -- avoid with either `--no-logues` or `--no-dot-ren`
* the directory-listing embed is sandboxed (so any malicious scripts can't do any damage) but the markdown editor is not
## cors
cross-site request config
by default, except for `GET` and `HEAD` operations, all requests must either:
* not contain an `Origin` header at all
* or have an `Origin` matching the server domain
* or the header `PW` with your password as value
cors can be configured with `--acao` and `--acam`, or the protections entirely disabled with `--allow-csrf`
# recovering from crashes
@@ -1154,90 +1323,7 @@ however you can hit `F12` in the up2k tab and use the devtools to see how far yo
# HTTP API
* table-column `params` = URL parameters; `?foo=bar&qux=...`
* table-column `body` = POST payload
* method `jPOST` = json post
* method `mPOST` = multipart post
* method `uPOST` = url-encoded post
* `FILE` = conventional HTTP file upload entry (rfc1867 et al, filename in `Content-Disposition`)
authenticate using header `Cookie: cppwd=foo` or url param `&pw=foo`
## read
| method | params | result |
|--|--|--|
| GET | `?ls` | list files/folders at URL as JSON |
| GET | `?ls&dots` | list files/folders at URL as JSON, including dotfiles |
| GET | `?ls=t` | list files/folders at URL as plaintext |
| GET | `?ls=v` | list files/folders at URL, terminal-formatted |
| GET | `?b` | list files/folders at URL as simplified HTML |
| GET | `?tree=.` | list one level of subdirectories inside URL |
| GET | `?tree` | list one level of subdirectories for each level until URL |
| GET | `?tar` | download everything below URL as a tar file |
| GET | `?zip=utf-8` | download everything below URL as a zip file |
| GET | `?ups` | show recent uploads from your IP |
| GET | `?ups&filter=f` | ...where URL contains `f` |
| GET | `?mime=foo` | specify return mimetype `foo` |
| GET | `?raw` | get markdown file at URL as plaintext |
| GET | `?txt` | get file at URL as plaintext |
| GET | `?txt=iso-8859-1` | ...with specific charset |
| GET | `?th` | get image/video at URL as thumbnail |
| GET | `?th=opus` | convert audio file to 128kbps opus |
| GET | `?th=caf` | ...in the iOS-proprietary container |
| method | body | result |
|--|--|--|
| jPOST | `{"q":"foo"}` | do a server-wide search; see the `[🔎]` search tab `raw` field for syntax |
| method | params | body | result |
|--|--|--|--|
| jPOST | `?tar` | `["foo","bar"]` | download folders `foo` and `bar` inside URL as a tar file |
## write
| method | params | result |
|--|--|--|
| GET | `?move=/foo/bar` | move/rename the file/folder at URL to /foo/bar |
| method | params | body | result |
|--|--|--|--|
| PUT | | (binary data) | upload into file at URL |
| PUT | `?gz` | (binary data) | compress with gzip and write into file at URL |
| PUT | `?xz` | (binary data) | compress with xz and write into file at URL |
| mPOST | | `act=bput`, `f=FILE` | upload `FILE` into the folder at URL |
| mPOST | `?j` | `act=bput`, `f=FILE` | ...and reply with json |
| mPOST | | `act=mkdir`, `name=foo` | create directory `foo` at URL |
| GET | `?delete` | | delete URL recursively |
| jPOST | `?delete` | `["/foo","/bar"]` | delete `/foo` and `/bar` recursively |
| uPOST | | `msg=foo` | send message `foo` into server log |
| mPOST | | `act=tput`, `body=TEXT` | overwrite markdown document at URL |
upload modifiers:
| http-header | url-param | effect |
|--|--|--|
| `Accept: url` | `want=url` | return just the file URL |
| `Rand: 4` | `rand=4` | generate random filename with 4 characters |
| `Life: 30` | `life=30` | delete file after 30 seconds |
* `life` only has an effect if the volume has a lifetime, and the volume lifetime must be greater than the file's
* server behavior of `msg` can be reconfigured with `--urlform`
## admin
| method | params | result |
|--|--|--|
| GET | `?reload=cfg` | reload config files and rescan volumes |
| GET | `?scan` | initiate a rescan of the volume which provides URL |
| GET | `?stack` | show a stacktrace of all threads |
## general
| method | params | result |
|--|--|--|
| GET | `?pw=x` | logout |
see [devnotes](#./docs/devnotes.md#http-api)
# dependencies
@@ -1265,6 +1351,9 @@ enable [thumbnails](#thumbnails) of...
* **AVIF pictures:** `pyvips` or `ffmpeg` or `pillow-avif-plugin`
* **JPEG XL pictures:** `pyvips` or `ffmpeg`
enable [smb](#smb-server) support (**not** recommended):
* `impacket==0.10.0`
`pyvips` gives higher quality thumbnails than `Pillow` and is 320% faster, using 270% more ram: `sudo apt install libvips42 && python3 -m pip install --user -U pyvips`
@@ -1285,40 +1374,39 @@ these are standalone programs and will never be imported / evaluated by copypart
the self-contained "binary" [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) will unpack itself and run copyparty, assuming you have python installed of course
you can reduce the sfx size by repacking it; see [./docs/devnotes.md#sfx-repack](#./docs/devnotes.md#sfx-repack)
## sfx repack
reduce the size of an sfx by removing features
## copyparty.exe
if you don't need all the features, you can repack the sfx and save a bunch of space; all you need is an sfx and a copy of this repo (nothing else to download or build, except if you're on windows then you need msys2 or WSL)
* `393k` size of original sfx.py as of v1.1.3
* `310k` after `./scripts/make-sfx.sh re no-cm`
* `269k` after `./scripts/make-sfx.sh re no-cm no-hl`
download [copyparty.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty.exe) or [copyparty64.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty64.exe)
the features you can opt to drop are
* `cm`/easymde, the "fancy" markdown editor, saves ~82k
* `hl`, prism, the syntax hilighter, saves ~41k
* `fnt`, source-code-pro, the monospace font, saves ~9k
* `dd`, the custom mouse cursor for the media player tray tab, saves ~2k
![copyparty-exe-fs8](https://user-images.githubusercontent.com/241032/194707422-cb7f66c9-41a2-4cb9-8dbc-2ab866cd4338.png)
for the `re`pack to work, first run one of the sfx'es once to unpack it
can be convenient on old machines where installing python is problematic, however is **not recommended** and should be considered a last resort -- if possible, please use **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** instead
**note:** you can also just download and run [scripts/copyparty-repack.sh](scripts/copyparty-repack.sh) -- this will grab the latest copyparty release from github and do a few repacks; works on linux/macos (and windows with msys2 or WSL)
* [copyparty.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty.exe) is compatible with 32bit windows7, which means it uses an ancient copy of python (3.7.9) which cannot be upgraded and will definitely become a security hazard at some point
* [copyparty64.exe](https://github.com/9001/copyparty/releases/latest/download/copyparty64.exe) is identical except 64bit so it [works in WinPE](https://user-images.githubusercontent.com/241032/205454984-e6b550df-3c49-486d-9267-1614078dd0dd.png)
meanwhile [copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py) instead relies on your system python which gives better performance and will stay safe as long as you keep your python install up-to-date
then again, if you are already into downloading shady binaries from the internet, you may also want my [minimal builds](./scripts/pyinstaller#ffmpeg) of [ffmpeg](https://ocv.me/stuff/bin/ffmpeg.exe) and [ffprobe](https://ocv.me/stuff/bin/ffprobe.exe) which enables copyparty to extract multimedia-info, do audio-transcoding, and thumbnails/spectrograms/waveforms, however it's much better to instead grab a [recent official build](https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip) every once ina while if you can afford the size
# install on android
install [Termux](https://termux.com/) (see [ocv.me/termux](https://ocv.me/termux/)) and then copy-paste this into Termux (long-tap) all at once:
install [Termux](https://termux.com/) + its companion app `Termux:API` (see [ocv.me/termux](https://ocv.me/termux/)) and then copy-paste this into Termux (long-tap) all at once:
```sh
apt update && apt -y full-upgrade && apt update && termux-setup-storage && apt -y install python && python -m ensurepip && python -m pip install --user -U copyparty
yes | pkg upgrade && termux-setup-storage && yes | pkg install python termux-api && python -m ensurepip && python -m pip install --user -U copyparty && { grep -qE 'PATH=.*\.local/bin' ~/.bashrc 2>/dev/null || { echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc && . ~/.bashrc; }; }
echo $?
```
after the initial setup, you can launch copyparty at any time by running `copyparty` anywhere in Termux
after the initial setup, you can launch copyparty at any time by running `copyparty` anywhere in Termux -- and if you run it with `--qr` you'll get a [neat qr-code](#qr-code) pointing to your external ip
if you want thumbnails, `apt -y install ffmpeg`
if you want thumbnails (photos+videos) and you're okay with spending another 132 MiB of storage, `pkg install ffmpeg && python3 -m pip install --user -U pillow`
* or if you want to use vips instead, `apt -y install libvips && python -m pip install --user -U wheel && python -m pip install --user -U pyvips && (cd /data/data/com.termux/files/usr/lib/; ln -s libgobject-2.0.so{,.0}; ln -s libvips.so{,.42})`
* or if you want to use `vips` for photo-thumbs instead, `pkg install libvips && python -m pip install --user -U wheel && python -m pip install --user -U pyvips && (cd /data/data/com.termux/files/usr/lib/; ln -s libgobject-2.0.so{,.0}; ln -s libvips.so{,.42})`
# reporting bugs
@@ -1335,86 +1423,6 @@ journalctl -aS '48 hour ago' -u copyparty | grep -C10 FILENAME | tee bug.log
if there's a wall of base64 in the log (thread stacks) then please include that, especially if you run into something freezing up or getting stuck, for example `OperationalError('database is locked')` -- alternatively you can visit `/?stack` to see the stacks live, so http://127.0.0.1:3923/?stack for example
# building
# devnotes
## dev env setup
you need python 3.9 or newer due to type hints
the rest is mostly optional; if you need a working env for vscode or similar
```sh
python3 -m venv .venv
. .venv/bin/activate
pip install jinja2 strip_hints # MANDATORY
pip install mutagen # audio metadata
pip install pyftpdlib # ftp server
pip install Pillow pyheif-pillow-opener pillow-avif-plugin # thumbnails
pip install black==21.12b0 click==8.0.2 bandit pylint flake8 isort mypy # vscode tooling
```
## just the sfx
first grab the web-dependencies from a previous sfx (assuming you don't need to modify something in those):
```sh
rm -rf copyparty/web/deps
curl -L https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py >x.py
python3 x.py -h
rm x.py
mv /tmp/pe-copyparty/copyparty/web/deps/ copyparty/web/deps/
```
then build the sfx using any of the following examples:
```sh
./scripts/make-sfx.sh # regular edition
./scripts/make-sfx.sh gz no-cm # gzip-compressed + no fancy markdown editor
```
## complete release
also builds the sfx so skip the sfx section above
in the `scripts` folder:
* run `make -C deps-docker` to build all dependencies
* run `./rls.sh 1.2.3` which uploads to pypi + creates github release + sfx
# todo
roughly sorted by priority
* nothing! currently
## discarded ideas
* reduce up2k roundtrips
* start from a chunk index and just go
* terminate client on bad data
* not worth the effort, just throw enough conncetions at it
* single sha512 across all up2k chunks?
* crypto.subtle cannot into streaming, would have to use hashwasm, expensive
* separate sqlite table per tag
* performance fixed by skipping some indexes (`+mt.k`)
* audio fingerprinting
* only makes sense if there can be a wasm client and that doesn't exist yet (except for olaf which is agpl hence counts as not existing)
* `os.copy_file_range` for up2k cloning
* almost never hit this path anyways
* up2k partials ui
* feels like there isn't much point
* cache sha512 chunks on client
* too dangerous -- overtaken by turbo mode
* comment field
* nah
* look into android thumbnail cache file format
* absolutely not
* indexedDB for hashes, cfg enable/clear/sz, 2gb avail, ~9k for 1g, ~4k for 100m, 500k items before autoeviction
* blank hashlist when up-ok to skip handshake
* too many confusing side-effects
* hls framework for Someone Else to drop code into :^)
* probably not, too much stuff to consider -- seeking, start at offset, task stitching (probably np-hard), conditional passthru, rate-control (especially multi-consumer), session keepalive, cache mgmt...
for build instructions etc, see [./docs/devnotes.md](./docs/devnotes.md)

9
SECURITY.md Normal file
View File

@@ -0,0 +1,9 @@
# Security Policy
if you hit something extra juicy pls let me know on either of the following
* email -- `copyparty@ocv.ze` except `ze` should be `me`
* [mastodon dm](https://layer8.space/@tripflag) -- `@tripflag@layer8.space`
* [github private vulnerability report](https://github.com/9001/copyparty/security/advisories/new), wow that form is complicated
* [twitter dm](https://twitter.com/tripflag) (if im somehow not banned yet)
no bug bounties sorry! all i can offer is greetz in the release notes

View File

@@ -1,7 +1,8 @@
# [`up2k.py`](up2k.py)
* command-line up2k client [(webm)](https://ocv.me/stuff/u2cli.webm)
* file uploads, file-search, autoresume of aborted/broken uploads
* faster than browsers
* sync local folder to server
* generally faster than browsers
* if something breaks just restart it
@@ -11,7 +12,7 @@ produces a chronological list of all uploads by collecting info from up2k databa
* optional mapping from IP-addresses to nicknames
# [`copyparty-fuse.py`](copyparty-fuse.py)
# [`partyfuse.py`](partyfuse.py)
* mount a copyparty server as a local filesystem (read-only)
* **supports Windows!** -- expect `194 MiB/s` sequential read
* **supports Linux** -- expect `117 MiB/s` sequential read
@@ -30,19 +31,19 @@ also consider using [../docs/rclone.md](../docs/rclone.md) instead for 5x perfor
* install [winfsp](https://github.com/billziss-gh/winfsp/releases/latest) and [python 3](https://www.python.org/downloads/)
* [x] add python 3.x to PATH (it asks during install)
* `python -m pip install --user fusepy`
* `python ./copyparty-fuse.py n: http://192.168.1.69:3923/`
* `python ./partyfuse.py n: http://192.168.1.69:3923/`
10% faster in [msys2](https://www.msys2.org/), 700% faster if debug prints are enabled:
* `pacman -S mingw64/mingw-w64-x86_64-python{,-pip}`
* `/mingw64/bin/python3 -m pip install --user fusepy`
* `/mingw64/bin/python3 ./copyparty-fuse.py [...]`
* `/mingw64/bin/python3 ./partyfuse.py [...]`
you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releases/latest), let me know if you [figure out how](https://github.com/dokan-dev/dokany/wiki/FUSE)
(winfsp's sshfs leaks, doesn't look like winfsp itself does, should be fine)
# [`copyparty-fuse🅱️.py`](copyparty-fuseb.py)
# [`partyfuse2.py`](partyfuse2.py)
* mount a copyparty server as a local filesystem (read-only)
* does the same thing except more correct, `samba` approves
* **supports Linux** -- expect `18 MiB/s` (wait what)
@@ -50,7 +51,7 @@ you could replace winfsp with [dokan](https://github.com/dokan-dev/dokany/releas
# [`copyparty-fuse-streaming.py`](copyparty-fuse-streaming.py)
# [`partyfuse-streaming.py`](partyfuse-streaming.py)
* pretend this doesn't exist

19
bin/hooks/README.md Normal file
View File

@@ -0,0 +1,19 @@
standalone programs which are executed by copyparty when an event happens (upload, file rename, delete, ...)
these programs either take zero arguments, or a filepath (the affected file), or a json message with filepath + additional info
> **note:** in addition to event hooks (the stuff described here), copyparty has another api to run your programs/scripts while providing way more information such as audio tags / video codecs / etc and optionally daisychaining data between scripts in a processing pipeline; if that's what you want then see [mtp plugins](../mtag/) instead
# after upload
* [notify.py](notify.py) shows a desktop notification ([example](https://user-images.githubusercontent.com/241032/215335767-9c91ed24-d36e-4b6b-9766-fb95d12d163f.png))
* [discord-announce.py](discord-announce.py) announces new uploads on discord using webhooks ([example](https://user-images.githubusercontent.com/241032/215304439-1c1cb3c8-ec6f-4c17-9f27-81f969b1811a.png))
* [reject-mimetype.py](reject-mimetype.py) rejects uploads unless the mimetype is acceptable
# before upload
* [reject-extension.py](reject-extension.py) rejects uploads if they match a list of file extensions
# on message
* [wget.py](wget.py) lets you download files by POSTing URLs to copyparty

61
bin/hooks/discord-announce.py Executable file
View File

@@ -0,0 +1,61 @@
#!/usr/bin/env python3
import sys
import json
import requests
from copyparty.util import humansize, quotep
_ = r"""
announces a new upload on discord
example usage as global config:
--xau f,t5,j,bin/hooks/discord-announce.py
example usage as a volflag (per-volume config):
-v srv/inc:inc:c,xau=f,t5,j,bin/hooks/discord-announce.py
parameters explained,
f = fork; don't wait for it to finish
t5 = timeout if it's still running after 5 sec
j = provide upload information as json; not just the filename
replace "xau" with "xbu" to announce Before upload starts instead of After completion
# how to discord:
first create the webhook url; https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks
then use this to design your message: https://discohook.org/
"""
def main():
WEBHOOK = "https://discord.com/api/webhooks/1234/base64"
# read info from copyparty
inf = json.loads(sys.argv[1])
vpath = inf["vp"]
filename = vpath.split("/")[-1]
url = f"https://{inf['host']}/{quotep(vpath)}"
# compose the message to discord
j = {
"title": filename,
"url": url,
"description": url.rsplit("/", 1)[0],
"color": 0x449900,
"fields": [
{"name": "Size", "value": humansize(inf["sz"])},
{"name": "User", "value": inf["user"]},
{"name": "IP", "value": inf["ip"]},
],
}
for v in j["fields"]:
v["inline"] = True
r = requests.post(WEBHOOK, json={"embeds": [j]})
print(f"discord: {r}\n", end="")
if __name__ == "__main__":
main()

62
bin/hooks/notify.py Executable file
View File

@@ -0,0 +1,62 @@
#!/usr/bin/env python3
import os
import sys
import subprocess as sp
from plyer import notification
_ = r"""
show os notification on upload; works on windows, linux, macos, android
depdencies:
windows: python3 -m pip install --user -U plyer
linux: python3 -m pip install --user -U plyer
macos: python3 -m pip install --user -U plyer pyobjus
android: just termux and termux-api
example usages; either as global config (all volumes) or as volflag:
--xau f,bin/hooks/notify.py
-v srv/inc:inc:c,xau=f,bin/hooks/notify.py
^^^^^^^^^^^^^^^^^^^^^^^^^^^
parameters explained,
xau = execute after upload
f = fork so it doesn't block uploads
"""
try:
from copyparty.util import humansize
except:
def humansize(n):
return n
def main():
fp = sys.argv[1]
dp, fn = os.path.split(fp)
try:
sz = humansize(os.path.getsize(fp))
except:
sz = "?"
msg = "{} ({})\n📁 {}".format(fn, sz, dp)
title = "File received"
if "com.termux" in sys.executable:
sp.run(["termux-notification", "-t", title, "-c", msg])
return
icon = "emblem-documents-symbolic" if sys.platform == "linux" else ""
notification.notify(
title=title,
message=msg,
app_icon=icon,
timeout=10,
)
if __name__ == "__main__":
main()

30
bin/hooks/reject-extension.py Executable file
View File

@@ -0,0 +1,30 @@
#!/usr/bin/env python3
import sys
_ = r"""
reject file uploads by file extension
example usage as global config:
--xbu c,bin/hooks/reject-extension.py
example usage as a volflag (per-volume config):
-v srv/inc:inc:c,xbu=c,bin/hooks/reject-extension.py
parameters explained,
xbu = execute before upload
c = check result, reject upload if error
"""
def main():
bad = "exe scr com pif bat ps1 jar msi"
ext = sys.argv[1].split(".")[-1]
sys.exit(1 if ext in bad.split() else 0)
if __name__ == "__main__":
main()

39
bin/hooks/reject-mimetype.py Executable file
View File

@@ -0,0 +1,39 @@
#!/usr/bin/env python3
import sys
import magic
_ = r"""
reject file uploads by mimetype
dependencies (linux, macos):
python3 -m pip install --user -U python-magic
dependencies (windows):
python3 -m pip install --user -U python-magic-bin
example usage as global config:
--xau c,bin/hooks/reject-mimetype.py
example usage as a volflag (per-volume config):
-v srv/inc:inc:c,xau=c,bin/hooks/reject-mimetype.py
parameters explained,
xau = execute after upload
c = check result, reject upload if error
"""
def main():
ok = ["image/jpeg", "image/png"]
mt = magic.from_file(sys.argv[1], mime=True)
print(mt)
sys.exit(1 if mt not in ok else 0)
if __name__ == "__main__":
main()

54
bin/hooks/wget.py Executable file
View File

@@ -0,0 +1,54 @@
#!/usr/bin/env python3
import os
import sys
import json
import subprocess as sp
_ = r"""
use copyparty as a file downloader by POSTing URLs as
application/x-www-form-urlencoded (for example using the
message/pager function on the website)
example usage as global config:
--xm f,j,t3600,bin/hooks/wget.py
example usage as a volflag (per-volume config):
-v srv/inc:inc:c,xm=f,j,t3600,bin/hooks/wget.py
parameters explained,
f = fork so it doesn't block uploads
j = provide message information as json; not just the text
c3 = mute all output
t3600 = timeout and kill download after 1 hour
"""
def main():
inf = json.loads(sys.argv[1])
url = inf["txt"]
if "://" not in url:
url = "https://" + url
os.chdir(inf["ap"])
name = url.split("?")[0].split("/")[-1]
tfn = "-- DOWNLOADING " + name
print(f"{tfn}\n", end="")
open(tfn, "wb").close()
cmd = ["wget", "--trust-server-names", "-nv", "--", url]
try:
sp.check_call(cmd)
except:
t = "-- FAILED TO DONWLOAD " + name
print(f"{t}\n", end="")
open(t, "wb").close()
os.unlink(tfn)
if __name__ == "__main__":
main()

View File

@@ -1,5 +1,9 @@
standalone programs which take an audio file as argument
you may want to forget about all this fancy complicated stuff and just use [event hooks](../hooks/) instead (which doesn't need `-e2ts` or ffmpeg)
----
**NOTE:** these all require `-e2ts` to be functional, meaning you need to do at least one of these: `apt install ffmpeg` or `pip3 install mutagen`
some of these rely on libraries which are not MIT-compatible
@@ -17,6 +21,7 @@ these do not have any problematic dependencies at all:
* [cksum.py](./cksum.py) computes various checksums
* [exe.py](./exe.py) grabs metadata from .exe and .dll files (example for retrieving multiple tags with one parser)
* [wget.py](./wget.py) lets you download files by POSTing URLs to copyparty
* also available as an [event hook](../hooks/wget.py)
# dependencies

View File

@@ -61,7 +61,7 @@ def main():
os.chdir(cwd)
f1 = fsenc(fn)
f2 = os.path.join(b"noexif", f1)
f2 = fsenc(os.path.join(b"noexif", fn))
cmd = [
b"exiftool",
b"-exif:all=",

View File

@@ -6,6 +6,7 @@ set -e
#
# linux/alpine: requires gcc g++ make cmake patchelf {python3,ffmpeg,fftw,libsndfile}-dev py3-{wheel,pip} py3-numpy{,-dev}
# linux/debian: requires libav{codec,device,filter,format,resample,util}-dev {libfftw3,python3,libsndfile1}-dev python3-{numpy,pip} vamp-{plugin-sdk,examples} patchelf cmake
# linux/fedora: requires gcc gcc-c++ make cmake patchelf {python3,ffmpeg,fftw,libsndfile}-devel python3-numpy vamp-plugin-sdk qm-vamp-plugins
# win64: requires msys2-mingw64 environment
# macos: requires macports
#
@@ -56,6 +57,7 @@ hash -r
command -v python3 && pybin=python3 || pybin=python
}
$pybin -c 'import numpy' ||
$pybin -m pip install --user numpy
@@ -160,12 +162,12 @@ install_keyfinder() {
h="$HOME"
so="lib/libkeyfinder.so"
memes=()
memes=(-DBUILD_TESTING=OFF)
[ $win ] &&
so="bin/libkeyfinder.dll" &&
h="$(printf '%s\n' "$USERPROFILE" | tr '\\' '/')" &&
memes+=(-G "MinGW Makefiles" -DBUILD_TESTING=OFF)
memes+=(-G "MinGW Makefiles")
[ $mac ] &&
so="lib/libkeyfinder.dylib"
@@ -185,7 +187,7 @@ install_keyfinder() {
}
# rm -rf /Users/ed/Library/Python/3.9/lib/python/site-packages/*keyfinder*
CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include" \
CFLAGS="-I$h/pe/keyfinder/include -I/opt/local/include -I/usr/include/ffmpeg" \
LDFLAGS="-L$h/pe/keyfinder/lib -L$h/pe/keyfinder/lib64 -L/opt/local/lib" \
PKG_CONFIG_PATH=/c/msys64/mingw64/lib/pkgconfig \
$pybin -m pip install --user keyfinder

View File

@@ -1,6 +1,11 @@
#!/usr/bin/env python3
"""
DEPRECATED -- replaced by event hooks;
https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/wget.py
---
use copyparty as a file downloader by POSTing URLs as
application/x-www-form-urlencoded (for example using the
message/pager function on the website)

View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python3
from __future__ import print_function, unicode_literals
"""copyparty-fuse-streaming: remote copyparty as a local filesystem"""
"""partyfuse-streaming: remote copyparty as a local filesystem"""
__author__ = "ed <copyparty@ocv.me>"
__copyright__ = 2020
__license__ = "MIT"
@@ -12,7 +12,7 @@ __url__ = "https://github.com/9001/copyparty/"
mount a copyparty server (local or remote) as a filesystem
usage:
python copyparty-fuse-streaming.py http://192.168.1.69:3923/ ./music
python partyfuse-streaming.py http://192.168.1.69:3923/ ./music
dependencies:
python3 -m pip install --user fusepy
@@ -21,7 +21,7 @@ dependencies:
+ on Windows: https://github.com/billziss-gh/winfsp/releases/latest
this was a mistake:
fork of copyparty-fuse.py with a streaming cache rather than readahead,
fork of partyfuse.py with a streaming cache rather than readahead,
thought this was gonna be way faster (and it kind of is)
except the overhead of reopening connections on trunc totally kills it
"""
@@ -62,12 +62,12 @@ except:
else:
libfuse = "apt install libfuse\n modprobe fuse"
print(
"\n could not import fuse; these may help:"
+ "\n python3 -m pip install --user fusepy\n "
+ libfuse
+ "\n"
)
m = """\033[33m
could not import fuse; these may help:
{} -m pip install --user fusepy
{}
\033[0m"""
print(m.format(sys.executable, libfuse))
raise
@@ -154,7 +154,7 @@ def dewin(txt):
class RecentLog(object):
def __init__(self):
self.mtx = threading.Lock()
self.f = None # open("copyparty-fuse.log", "wb")
self.f = None # open("partyfuse.log", "wb")
self.q = []
thr = threading.Thread(target=self.printer)
@@ -185,9 +185,9 @@ class RecentLog(object):
print("".join(q), end="")
# [windows/cmd/cpy3] python dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/copyparty-fuse.py q: http://192.168.1.159:1234/
# [windows/cmd/cpy3] python dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/partyfuse.py q: http://192.168.1.159:1234/
#
# [windows] find /q/music/albums/Phant*24bit -printf '%s %p\n' | sort -n | tail -n 8 | sed -r 's/^[0-9]+ //' | while IFS= read -r x; do dd if="$x" of=/dev/null bs=4k count=8192 & done
# [alpine] ll t; for x in t/2020_0724_16{2,3}*; do dd if="$x" of=/dev/null bs=4k count=10240 & done

View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python3
from __future__ import print_function, unicode_literals
"""copyparty-fuse: remote copyparty as a local filesystem"""
"""partyfuse: remote copyparty as a local filesystem"""
__author__ = "ed <copyparty@ocv.me>"
__copyright__ = 2019
__license__ = "MIT"
@@ -12,7 +12,7 @@ __url__ = "https://github.com/9001/copyparty/"
mount a copyparty server (local or remote) as a filesystem
usage:
python copyparty-fuse.py http://192.168.1.69:3923/ ./music
python partyfuse.py http://192.168.1.69:3923/ ./music
dependencies:
python3 -m pip install --user fusepy
@@ -74,12 +74,12 @@ except:
else:
libfuse = "apt install libfuse3-3\n modprobe fuse"
print(
"\n could not import fuse; these may help:"
+ "\n python3 -m pip install --user fusepy\n "
+ libfuse
+ "\n"
)
m = """\033[33m
could not import fuse; these may help:
{} -m pip install --user fusepy
{}
\033[0m"""
print(m.format(sys.executable, libfuse))
raise
@@ -166,7 +166,7 @@ def dewin(txt):
class RecentLog(object):
def __init__(self):
self.mtx = threading.Lock()
self.f = None # open("copyparty-fuse.log", "wb")
self.f = None # open("partyfuse.log", "wb")
self.q = []
thr = threading.Thread(target=self.printer)
@@ -197,9 +197,9 @@ class RecentLog(object):
print("".join(q), end="")
# [windows/cmd/cpy3] python dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\copyparty-fuse.py q: http://192.168.1.159:1234/
# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/copyparty-fuse.py q: http://192.168.1.159:1234/
# [windows/cmd/cpy3] python dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
# [windows/cmd/msys2] C:\msys64\mingw64\bin\python3 dev\copyparty\bin\partyfuse.py q: http://192.168.1.159:1234/
# [windows/mty/msys2] /mingw64/bin/python3 /c/Users/ed/dev/copyparty/bin/partyfuse.py q: http://192.168.1.159:1234/
#
# [windows] find /q/music/albums/Phant*24bit -printf '%s %p\n' | sort -n | tail -n 8 | sed -r 's/^[0-9]+ //' | while IFS= read -r x; do dd if="$x" of=/dev/null bs=4k count=8192 & done
# [alpine] ll t; for x in t/2020_0724_16{2,3}*; do dd if="$x" of=/dev/null bs=4k count=10240 & done
@@ -997,7 +997,7 @@ def main():
ap.add_argument(
"-cf", metavar="NUM_BLOCKS", type=int, default=nf, help="file cache"
)
ap.add_argument("-a", metavar="PASSWORD", help="password")
ap.add_argument("-a", metavar="PASSWORD", help="password or $filepath")
ap.add_argument("-d", action="store_true", help="enable debug")
ap.add_argument("-te", metavar="PEM_FILE", help="certificate to expect/verify")
ap.add_argument("-td", action="store_true", help="disable certificate check")

View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python3
from __future__ import print_function, unicode_literals
"""copyparty-fuseb: remote copyparty as a local filesystem"""
"""partyfuse2: remote copyparty as a local filesystem"""
__author__ = "ed <copyparty@ocv.me>"
__copyright__ = 2020
__license__ = "MIT"
@@ -32,9 +32,19 @@ try:
if not hasattr(fuse, "__version__"):
raise Exception("your fuse-python is way old")
except:
print(
"\n could not import fuse; these may help:\n python3 -m pip install --user fuse-python\n apt install libfuse\n modprobe fuse\n"
)
if WINDOWS:
libfuse = "install https://github.com/billziss-gh/winfsp/releases/latest"
elif MACOS:
libfuse = "install https://osxfuse.github.io/"
else:
libfuse = "apt install libfuse\n modprobe fuse"
m = """\033[33m
could not import fuse; these may help:
{} -m pip install --user fuse-python
{}
\033[0m"""
print(m.format(sys.executable, libfuse))
raise
@@ -42,13 +52,13 @@ except:
mount a copyparty server (local or remote) as a filesystem
usage:
python ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas
python ./partyfuse2.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas
dependencies:
sudo apk add fuse-dev python3-dev
python3 -m pip install --user fuse-python
fork of copyparty-fuse.py based on fuse-python which
fork of partyfuse.py based on fuse-python which
appears to be more compliant than fusepy? since this works with samba
(probably just my garbage code tbh)
"""
@@ -639,7 +649,7 @@ def main():
print(" need argument: mount-path")
print("example:")
print(
" ./copyparty-fuseb.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas"
" ./partyfuse2.py -f -o allow_other,auto_unmount,nonempty,pw=wark,url=http://192.168.1.69:3923 /mnt/nas"
)
sys.exit(1)

View File

@@ -4,7 +4,7 @@ set -e
# runs copyparty (or any other program really) in a chroot
#
# assumption: these directories, and everything within, are owned by root
sysdirs=( /bin /lib /lib32 /lib64 /sbin /usr )
sysdirs=( /bin /lib /lib32 /lib64 /sbin /usr /etc/alternatives )
# error-handler
@@ -97,9 +97,11 @@ done
cln() {
rv=$?
# cleanup if not in use
lsof "$jail" | grep -qF "$jail" &&
echo "chroot is in use, will not cleanup" ||
wait -f -p rv $p || true
cd /
echo "stopping chroot..."
lsof "$jail" | grep -F "$jail" &&
echo "chroot is in use; will not unmount" ||
{
mount | grep -F " on $jail" |
awk '{sub(/ type .*/,"");sub(/.* on /,"");print}' |
@@ -124,5 +126,6 @@ export LOGNAME="$USER"
#echo "cpp [$cpp]"
chroot --userspec=$uid:$gid "$jail" "$pybin" $pyarg "$cpp" "$@" &
p=$!
trap 'kill -USR1 $p' USR1
trap 'kill $p' INT TERM
wait

View File

@@ -3,14 +3,12 @@ from __future__ import print_function, unicode_literals
"""
up2k.py: upload to copyparty
2022-09-05, v0.19, ed <irc.rizon.net>, MIT-Licensed
2023-01-13, v1.2, ed <irc.rizon.net>, MIT-Licensed
https://github.com/9001/copyparty/blob/hovudstraum/bin/up2k.py
- dependencies: requests
- supports python 2.6, 2.7, and 3.3 through 3.11
- almost zero error-handling
- but if something breaks just try again and it'll autoresume
- supports python 2.6, 2.7, and 3.3 through 3.12
- if something breaks just try again and it'll autoresume
"""
import os
@@ -42,17 +40,17 @@ except ImportError:
m = "requests/2.18.4 urllib3/1.23 chardet/3.0.4 certifi/2020.4.5.1 idna/2.7"
m = [" https://pypi.org/project/" + x + "/#files" for x in m.split()]
m = "\n ERROR: need these:\n" + "\n".join(m) + "\n"
m += "\n for f in *.whl; do unzip $f; done; rm -r *.dist-info\n"
print(m.format(sys.executable))
sys.exit(1)
# from copyparty/__init__.py
PY2 = sys.version_info[0] == 2
PY2 = sys.version_info < (3,)
if PY2:
from Queue import Queue
from urllib import unquote
from urllib import quote
from urllib import quote, unquote
sys.dont_write_bytecode = True
bytes = str
@@ -69,6 +67,14 @@ VT100 = platform.system() != "Windows"
req_ses = requests.Session()
class Daemon(threading.Thread):
def __init__(self, target, name=None, a=None):
# type: (Any, Any, Any) -> None
threading.Thread.__init__(self, target=target, args=a or (), name=name)
self.daemon = True
self.start()
class File(object):
"""an up2k upload task; represents a single file"""
@@ -86,6 +92,7 @@ class File(object):
self.kchunks = {} # type: dict[str, tuple[int, int]] # hash: [ ofs, sz ]
# set by handshake
self.recheck = False # duplicate; redo handshake after all files done
self.ucids = [] # type: list[str] # chunks which need to be uploaded
self.wark = None # type: str
self.url = None # type: str
@@ -154,10 +161,7 @@ class MTHash(object):
self.done_q = Queue()
self.thrs = []
for _ in range(cores):
t = threading.Thread(target=self.worker)
t.daemon = True
t.start()
self.thrs.append(t)
self.thrs.append(Daemon(self.worker))
def hash(self, f, fsz, chunksz, pcb=None, pcb_opaque=None):
with self.omutex:
@@ -257,10 +261,10 @@ def termsize():
try:
import fcntl, termios, struct
cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
r = struct.unpack(b"hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, b"AAAA"))
return r[::-1]
except:
return
return cr
return None
cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
if not cr:
@@ -270,12 +274,11 @@ def termsize():
os.close(fd)
except:
pass
if not cr:
try:
cr = (env["LINES"], env["COLUMNS"])
except:
cr = (25, 80)
return int(cr[1]), int(cr[0])
try:
return cr or (int(env["COLUMNS"]), int(env["LINES"]))
except:
return 80, 25
class CTermsize(object):
@@ -290,9 +293,7 @@ class CTermsize(object):
except:
return
thr = threading.Thread(target=self.worker)
thr.daemon = True
thr.start()
Daemon(self.worker)
def worker(self):
while True:
@@ -359,26 +360,29 @@ def walkdir(err, top, seen):
seen = seen[:] + [atop]
for ap, inf in sorted(statdir(err, top)):
yield ap, inf
if stat.S_ISDIR(inf.st_mode):
try:
for x in walkdir(err, ap, seen):
yield x
except Exception as ex:
err.append((ap, str(ex)))
else:
yield ap, inf
def walkdirs(err, tops):
"""recursive statdir for a list of tops, yields [top, relpath, stat]"""
sep = "{0}".format(os.sep).encode("ascii")
for top in tops:
isdir = os.path.isdir(top)
if top[-1:] == sep:
stop = top.rstrip(sep)
yield stop, b"", os.stat(stop)
else:
stop = os.path.dirname(top)
stop, dn = os.path.split(top)
if isdir:
yield stop, dn, os.stat(stop)
if os.path.isdir(top):
if isdir:
for ap, inf in walkdir(err, top, []):
yield stop, ap[len(stop) :].lstrip(sep), inf
else:
@@ -420,7 +424,7 @@ def up2k_chunksize(filesize):
while True:
for mul in [1, 2]:
nchunks = math.ceil(filesize * 1.0 / chunksize)
if nchunks <= 256 or chunksize >= 32 * 1024 * 1024:
if nchunks <= 256 or (chunksize >= 32 * 1024 * 1024 and nchunks < 4096):
return chunksize
chunksize += stepsize
@@ -469,14 +473,17 @@ def get_hashlist(file, pcb, mth):
file.kchunks[k] = [v1, v2]
def handshake(req_ses, url, file, pw, search):
# type: (requests.Session, str, File, any, bool) -> list[str]
def handshake(ar, file, search):
# type: (argparse.Namespace, File, bool) -> tuple[list[str], bool]
"""
performs a handshake with the server; reply is:
if search, a list of search results
otherwise, a list of chunks to upload
"""
url = ar.url
pw = ar.a
req = {
"hash": [x[0] for x in file.cids],
"name": file.name,
@@ -485,22 +492,42 @@ def handshake(req_ses, url, file, pw, search):
}
if search:
req["srch"] = 1
elif ar.dr:
req["replace"] = True
headers = {"Content-Type": "text/plain"} # wtf ed
headers = {"Content-Type": "text/plain"} # <=1.5.1 compat
if pw:
headers["Cookie"] = "=".join(["cppwd", pw])
file.recheck = False
if file.url:
url = file.url
elif b"/" in file.rel:
url += quotep(file.rel.rsplit(b"/", 1)[0]).decode("utf-8", "replace")
while True:
sc = 600
txt = ""
try:
r = req_ses.post(url, headers=headers, json=req)
break
sc = r.status_code
txt = r.text
if sc < 400:
break
raise Exception("http {0}: {1}".format(sc, txt))
except Exception as ex:
em = str(ex).split("SSLError(")[-1]
em = str(ex).split("SSLError(")[-1].split("\nURL: ")[0].strip()
if sc == 422 or "<pre>partial upload exists at a different" in txt:
file.recheck = True
return [], False
elif sc == 409 or "<pre>upload rejected, file already exists" in txt:
return [], False
elif "<pre>you don't have " in txt:
raise
eprint("handshake failed, retrying: {0}\n {1}\n\n".format(file.name, em))
time.sleep(1)
@@ -525,8 +552,8 @@ def handshake(req_ses, url, file, pw, search):
return r["hash"], r["sprs"]
def upload(req_ses, file, cid, pw):
# type: (requests.Session, File, str, any) -> None
def upload(file, cid, pw):
# type: (File, str, str) -> None
"""upload one specific chunk, `cid` (a chunk-hash)"""
headers = {
@@ -548,35 +575,22 @@ def upload(req_ses, file, cid, pw):
f.f.close()
class Daemon(threading.Thread):
def __init__(self, *a, **ka):
threading.Thread.__init__(self, *a, **ka)
self.daemon = True
class Ctl(object):
"""
this will be the coordinator which runs everything in parallel
(hashing, handshakes, uploads) but right now it's p dumb
the coordinator which runs everything in parallel
(hashing, handshakes, uploads)
"""
def __init__(self, ar):
self.ar = ar
ar.files = [
os.path.abspath(os.path.realpath(x.encode("utf-8")))
+ (x[-1:] if x[-1:] == os.sep else "").encode("utf-8")
for x in ar.files
]
ar.url = ar.url.rstrip("/") + "/"
if "://" not in ar.url:
ar.url = "http://" + ar.url
def _scan(self):
ar = self.ar
eprint("\nscanning {0} locations\n".format(len(ar.files)))
nfiles = 0
nbytes = 0
err = []
for _, _, inf in walkdirs(err, ar.files):
if stat.S_ISDIR(inf.st_mode):
continue
nfiles += 1
nbytes += inf.st_size
@@ -598,8 +612,15 @@ class Ctl(object):
return
eprint("found {0} files, {1}\n\n".format(nfiles, humansize(nbytes)))
self.nfiles = nfiles
self.nbytes = nbytes
return nfiles, nbytes
def __init__(self, ar, stats=None):
self.ar = ar
self.stats = stats or self._scan()
if not self.stats:
return
self.nfiles, self.nbytes = self.stats
if ar.td:
requests.packages.urllib3.disable_warnings()
@@ -629,8 +650,8 @@ class Ctl(object):
self.mutex = threading.Lock()
self.q_handshake = Queue() # type: Queue[File]
self.q_recheck = Queue() # type: Queue[File] # partial upload exists [...]
self.q_upload = Queue() # type: Queue[tuple[File, str]]
self.recheck = [] # type: list[File]
self.st_hash = [None, "(idle, starting...)"] # type: tuple[File, int]
self.st_up = [None, "(idle, starting...)"] # type: tuple[File, int]
@@ -643,6 +664,9 @@ class Ctl(object):
"""minimal basic slow boring fallback codepath"""
search = self.ar.s
for nf, (top, rel, inf) in enumerate(self.filegen):
if stat.S_ISDIR(inf.st_mode) or not rel:
continue
file = File(top, rel, inf.st_size, inf.st_mtime)
upath = file.abs.decode("utf-8", "replace")
@@ -652,7 +676,7 @@ class Ctl(object):
burl = self.ar.url[:12] + self.ar.url[8:].split("/")[0] + "/"
while True:
print(" hs...")
hs, _ = handshake(req_ses, self.ar.url, file, self.ar.a, search)
hs, _ = handshake(self.ar, file, search)
if search:
if hs:
for hit in hs:
@@ -669,19 +693,28 @@ class Ctl(object):
ncs = len(hs)
for nc, cid in enumerate(hs):
print(" {0} up {1}".format(ncs - nc, cid))
upload(req_ses, file, cid, self.ar.a)
upload(file, cid, self.ar.a)
print(" ok!")
if file.recheck:
self.recheck.append(file)
if not self.recheck:
return
eprint("finalizing {0} duplicate files".format(len(self.recheck)))
for file in self.recheck:
handshake(self.ar, file, search)
def _fancy(self):
if VT100:
if VT100 and not self.ar.ns:
atexit.register(self.cleanup_vt100)
ss.scroll_region(3)
Daemon(target=self.hasher).start()
Daemon(self.hasher)
for _ in range(self.ar.j):
Daemon(target=self.handshaker).start()
Daemon(target=self.uploader).start()
Daemon(self.handshaker)
Daemon(self.uploader)
idles = 0
while idles < 3:
@@ -698,7 +731,7 @@ class Ctl(object):
else:
idles = 0
if VT100:
if VT100 and not self.ar.ns:
maxlen = ss.w - len(str(self.nfiles)) - 14
txt = "\033[s\033[{0}H".format(ss.g)
for y, k, st, f in [
@@ -738,11 +771,18 @@ class Ctl(object):
eta = str(datetime.timedelta(seconds=int(eta)))
sleft = humansize(self.nbytes - self.up_b)
nleft = self.nfiles - self.up_f
tail = "\033[K\033[u" if VT100 else "\r"
tail = "\033[K\033[u" if VT100 and not self.ar.ns else "\r"
t = "{0} eta @ {1}/s, {2}, {3}# left".format(eta, spd, sleft, nleft)
eprint(txt + "\033]0;{0}\033\\\r{0}{1}".format(t, tail))
if not self.recheck:
return
eprint("finalizing {0} duplicate files".format(len(self.recheck)))
for file in self.recheck:
handshake(self.ar, file, False)
def cleanup_vt100(self):
ss.scroll_region(None)
eprint("\033[J\033]0;\033\\")
@@ -754,8 +794,10 @@ class Ctl(object):
prd = None
ls = {}
for top, rel, inf in self.filegen:
if self.ar.z:
rd = os.path.dirname(rel)
isdir = stat.S_ISDIR(inf.st_mode)
if self.ar.z or self.ar.drd:
rd = rel if isdir else os.path.dirname(rel)
srd = rd.decode("utf-8", "replace").replace("\\", "/")
if prd != rd:
prd = rd
headers = {}
@@ -764,19 +806,37 @@ class Ctl(object):
ls = {}
try:
print(" ls ~{0}".format(rd.decode("utf-8", "replace")))
r = req_ses.get(
self.ar.url.encode("utf-8") + quotep(rd) + b"?ls",
headers=headers,
)
for f in r.json()["files"]:
rfn = f["href"].split("?")[0].encode("utf-8", "replace")
ls[unquote(rfn)] = f
except:
print(" mkdir ~{0}".format(rd.decode("utf-8", "replace")))
print(" ls ~{0}".format(srd))
zb = self.ar.url.encode("utf-8")
zb += quotep(rd.replace(b"\\", b"/"))
r = req_ses.get(zb + b"?ls&dots", headers=headers)
if not r:
raise Exception("HTTP {}".format(r.status_code))
j = r.json()
for f in j["dirs"] + j["files"]:
rfn = f["href"].split("?")[0].rstrip("/")
ls[unquote(rfn.encode("utf-8", "replace"))] = f
except Exception as ex:
print(" mkdir ~{0} ({1})".format(srd, ex))
if self.ar.drd:
dp = os.path.join(top, rd)
lnodes = set(os.listdir(dp))
bnames = [x for x in ls if x not in lnodes]
if bnames:
vpath = self.ar.url.split("://")[-1].split("/", 1)[-1]
names = [x.decode("utf-8", "replace") for x in bnames]
locs = [vpath + srd + "/" + x for x in names]
print("DELETING ~{0}/#{1}".format(srd, len(names)))
req_ses.post(self.ar.url + "?delete", json=locs)
if isdir:
continue
if self.ar.z:
rf = ls.get(os.path.basename(rel), None)
if rf and rf["sz"] == inf.st_size and abs(rf["ts"] - inf.st_mtime) <= 1:
if rf and rf["sz"] == inf.st_size and abs(rf["ts"] - inf.st_mtime) <= 2:
self.nfiles -= 1
self.nbytes -= inf.st_size
continue
@@ -785,15 +845,17 @@ class Ctl(object):
while True:
with self.mutex:
if (
self.hash_b - self.up_b < 1024 * 1024 * 128
and self.hash_c - self.up_c < 64
and (
not self.ar.nh
or (
self.q_upload.empty()
and self.q_handshake.empty()
and not self.uploader_busy
)
self.hash_f - self.up_f == 1
or (
self.hash_b - self.up_b < 1024 * 1024 * 1024
and self.hash_c - self.up_c < 512
)
) and (
not self.ar.nh
or (
self.q_upload.empty()
and self.q_handshake.empty()
and not self.uploader_busy
)
):
break
@@ -813,16 +875,10 @@ class Ctl(object):
def handshaker(self):
search = self.ar.s
q = self.q_handshake
burl = self.ar.url[:8] + self.ar.url[8:].split("/")[0] + "/"
while True:
file = q.get()
file = self.q_handshake.get()
if not file:
if q == self.q_handshake:
q = self.q_recheck
q.put(None)
continue
self.q_upload.put(None)
break
@@ -830,16 +886,7 @@ class Ctl(object):
self.handshaker_busy += 1
upath = file.abs.decode("utf-8", "replace")
try:
hs, sprs = handshake(req_ses, self.ar.url, file, self.ar.a, search)
except Exception as ex:
if q == self.q_handshake and "<pre>partial upload exists" in str(ex):
self.q_recheck.put(file)
hs = []
else:
raise
hs, sprs = handshake(self.ar, file, search)
if search:
if hs:
for hit in hs:
@@ -856,8 +903,11 @@ class Ctl(object):
continue
if file.recheck:
self.recheck.append(file)
with self.mutex:
if not sprs and not self.serialized:
if hs and not sprs and not self.serialized:
t = "server filesystem does not support sparse files; serializing uploads\n"
eprint(t)
self.serialized = True
@@ -869,6 +919,9 @@ class Ctl(object):
self.up_c += len(file.cids) - file.up_c
self.up_b += file.size - file.up_b
if not file.recheck:
self.up_done(file)
if hs and file.up_c:
# some chunks failed
self.up_c -= len(hs)
@@ -900,10 +953,10 @@ class Ctl(object):
file, cid = task
try:
upload(req_ses, file, cid, self.ar.a)
upload(file, cid, self.ar.a)
except:
eprint("upload failed, retrying: {0} #{1}\n".format(file.name, cid[:8]))
pass # handshake will fix it
# handshake will fix it
with self.mutex:
sz = file.kchunks[cid][1]
@@ -919,6 +972,10 @@ class Ctl(object):
self.up_c += 1
self.uploader_busy -= 1
def up_done(self, file):
if self.ar.dl:
os.unlink(file.abs)
class APF(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
pass
@@ -943,21 +1000,74 @@ source file/folder selection uses rsync syntax, meaning that:
ap.add_argument("url", type=unicode, help="server url, including destination folder")
ap.add_argument("files", type=unicode, nargs="+", help="files and/or folders to process")
ap.add_argument("-v", action="store_true", help="verbose")
ap.add_argument("-a", metavar="PASSWORD", help="password")
ap.add_argument("-a", metavar="PASSWORD", help="password or $filepath")
ap.add_argument("-s", action="store_true", help="file-search (disables upload)")
ap.add_argument("--ok", action="store_true", help="continue even if some local files are inaccessible")
ap = app.add_argument_group("compatibility")
ap.add_argument("--cls", action="store_true", help="clear screen before start")
ap.add_argument("--ws", action="store_true", help="copyparty is running on windows; wait before deleting files after uploading")
ap = app.add_argument_group("folder sync")
ap.add_argument("--dl", action="store_true", help="delete local files after uploading")
ap.add_argument("--dr", action="store_true", help="delete remote files which don't exist locally")
ap.add_argument("--drd", action="store_true", help="delete remote files during upload instead of afterwards; reduces peak disk space usage, but will reupload instead of detecting renames")
ap = app.add_argument_group("performance tweaks")
ap.add_argument("-j", type=int, metavar="THREADS", default=4, help="parallel connections")
ap.add_argument("-J", type=int, metavar="THREADS", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
ap.add_argument("-ns", action="store_true", help="no status panel (for slow consoles)")
ap.add_argument("--safe", action="store_true", help="use simple fallback approach")
ap.add_argument("-z", action="store_true", help="ZOOMIN' (skip uploading files if they exist at the destination with the ~same last-modified timestamp, so same as yolo / turbo with date-chk but even faster)")
ap = app.add_argument_group("tls")
ap.add_argument("-te", metavar="PEM_FILE", help="certificate to expect/verify")
ap.add_argument("-td", action="store_true", help="disable certificate check")
# fmt: on
Ctl(app.parse_args())
ar = app.parse_args()
if ar.drd:
ar.dr = True
for k in "dl dr drd".split():
errs = []
if ar.safe and getattr(ar, k):
errs.append(k)
if errs:
raise Exception("--safe is incompatible with " + str(errs))
ar.files = [
os.path.abspath(os.path.realpath(x.encode("utf-8")))
+ (x[-1:] if x[-1:] == os.sep else "").encode("utf-8")
for x in ar.files
]
ar.url = ar.url.rstrip("/") + "/"
if "://" not in ar.url:
ar.url = "http://" + ar.url
if ar.a and ar.a.startswith("$"):
fn = ar.a[1:]
print("reading password from file [{}]".format(fn))
with open(fn, "rb") as f:
ar.a = f.read().decode("utf-8").strip()
if ar.cls:
print("\x1b\x5b\x48\x1b\x5b\x32\x4a\x1b\x5b\x33\x4a", end="")
ctl = Ctl(ar)
if ar.dr and not ar.drd:
print("\npass 2/2: delete")
if getattr(ctl, "up_br") and ar.ws:
# wait for up2k to mtime if there was uploads
time.sleep(4)
ar.drd = True
ar.z = True
Ctl(ar, ctl.stats)
if __name__ == "__main__":

View File

@@ -27,7 +27,13 @@ however if your copyparty is behind a reverse-proxy, you may want to use [`share
### [`explorer-nothumbs-nofoldertypes.reg`](explorer-nothumbs-nofoldertypes.reg)
* disables thumbnails and folder-type detection in windows explorer
* makes it way faster (especially for slow/networked locations (such as copyparty-fuse))
* makes it way faster (especially for slow/networked locations (such as partyfuse))
### [`webdav-cfg.reg`](webdav-cfg.bat)
* improves the native webdav support in windows;
* removes the 47.6 MiB filesize limit when downloading from webdav
* optionally enables webdav basic-auth over plaintext http
* optionally helps disable wpad, removing the 10sec latency
### [`cfssl.sh`](cfssl.sh)
* creates CA and server certificates using cfssl

View File

@@ -0,0 +1,15 @@
# when running copyparty behind a reverse proxy,
# the following arguments are recommended:
#
# --http-only lower latency on initial connection
# -i 127.0.0.1 only accept connections from nginx
#
# if you are doing location-based proxying (such as `/stuff` below)
# you must run copyparty with --rp-loc=stuff
#
# on fedora/rhel, remember to setsebool -P httpd_can_network_connect 1
LoadModule proxy_module modules/mod_proxy.so
ProxyPass "/stuff" "http://127.0.0.1:3923/stuff"
# do not specify ProxyPassReverse
RequestHeader set "X-Forwarded-Proto" expr=%{REQUEST_SCHEME}

View File

@@ -1,15 +1,17 @@
# when running copyparty behind a reverse proxy,
# the following arguments are recommended:
#
# -nc 512 important, see next paragraph
# --http-only lower latency on initial connection
# -i 127.0.0.1 only accept connections from nginx
#
# -nc must match or exceed the webserver's max number of concurrent clients;
# copyparty default is 1024 if OS permits it (see "max clients:" on startup),
# nginx default is 512 (worker_processes 1, worker_connections 512)
#
# you may also consider adding -j0 for CPU-intensive configurations
# (not that i can really think of any good examples)
#
# on fedora/rhel, remember to setsebool -P httpd_can_network_connect 1
upstream cpp {
server 127.0.0.1:3923;

View File

@@ -14,5 +14,5 @@ name="$SVCNAME"
command_background=true
pidfile="/var/run/$SVCNAME.pid"
command="/usr/bin/python /usr/local/bin/copyparty-sfx.py"
command="/usr/bin/python3 /usr/local/bin/copyparty-sfx.py"
command_args="-q -v /mnt::rw"

View File

@@ -0,0 +1,57 @@
# Maintainer: icxes <dev.null@need.moe>
pkgname=copyparty
pkgver="1.6.4"
pkgrel=1
pkgdesc="Portable file sharing hub"
arch=("any")
url="https://github.com/9001/${pkgname}"
license=('MIT')
depends=("python" "lsof")
optdepends=("ffmpeg: thumbnails for videos, images (slower) and audio, music tags"
"python-jinja: faster html generator"
"python-mutagen: music tags (alternative)"
"python-pillow: thumbnails for images"
"python-pyvips: thumbnails for images (higher quality, faster, uses more ram)"
"libkeyfinder-git: detection of musical keys"
"qm-vamp-plugins: BPM detection"
"python-pyopenssl: ftps functionality"
"python-impacket-git: smb support (bad idea)"
)
source=("${url}/releases/download/v${pkgver}/${pkgname}-sfx.py"
"${pkgname}.conf"
"${pkgname}.service"
"prisonparty.service"
"index.md"
"https://raw.githubusercontent.com/9001/${pkgname}/v${pkgver}/bin/prisonparty.sh"
"https://raw.githubusercontent.com/9001/${pkgname}/v${pkgver}/LICENSE"
)
backup=("etc/${pkgname}.d/init" )
sha256sums=("d0447c7a8c4738d2f910f0287c66c70f48c6fae4cf941fb7227504e646fe3e78"
"b8565eba5e64dedba1cf6c7aac7e31c5a731ed7153d6810288a28f00a36c28b2"
"f65c207e0670f9d78ad2e399bda18d5502ff30d2ac79e0e7fc48e7fbdc39afdc"
"c4f396b083c9ec02ad50b52412c84d2a82be7f079b2d016e1c9fad22d68285ff"
"dba701de9fd584405917e923ea1e59dbb249b96ef23bad479cf4e42740b774c8"
"746971e95817c54445ce7f9c8406822dffc814cd5eb8113abd36dd472fd677d7"
"cb2ce3d6277bf2f5a82ecf336cc44963bc6490bcf496ffbd75fc9e21abaa75f3"
)
package() {
cd "${srcdir}/"
install -dm755 "${pkgdir}/etc/${pkgname}.d"
install -Dm755 "${pkgname}-sfx.py" "${pkgdir}/usr/bin/${pkgname}"
install -Dm755 "prisonparty.sh" "${pkgdir}/usr/bin/prisonparty"
install -Dm644 "${pkgname}.conf" "${pkgdir}/etc/${pkgname}.d/init"
install -Dm644 "${pkgname}.service" "${pkgdir}/usr/lib/systemd/system/${pkgname}.service"
install -Dm644 "prisonparty.service" "${pkgdir}/usr/lib/systemd/system/prisonparty.service"
install -Dm644 "index.md" "${pkgdir}/var/lib/${pkgname}-jail/README.md"
install -Dm644 "LICENSE" "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
find /etc/${pkgname}.d -iname '*.conf' 2>/dev/null | grep -qE . && return
echo "┏━━━━━━━━━━━━━━━──-"
echo "┃ Configure ${pkgname} by adding .conf files into /etc/${pkgname}.d/"
echo "┃ and maybe copy+edit one of the following to /etc/systemd/system/:"
echo "┣━♦ /usr/lib/systemd/system/${pkgname}.service (standard)"
echo "┣━♦ /usr/lib/systemd/system/prisonparty.service (chroot)"
echo "┗━━━━━━━━━━━━━━━──-"
}

View File

@@ -0,0 +1,7 @@
## import all *.conf files from the current folder (/etc/copyparty.d)
% ./
# add additional .conf files to this folder;
# see example config files for reference:
# https://github.com/9001/copyparty/blob/hovudstraum/docs/example.conf
# https://github.com/9001/copyparty/tree/hovudstraum/docs/copyparty.d

View File

@@ -0,0 +1,32 @@
# this will start `/usr/bin/copyparty-sfx.py`
# and read config from `/etc/copyparty.d/*.conf`
#
# you probably want to:
# change "User=cpp" and "/home/cpp/" to another user
#
# unless you add -q to disable logging, you may want to remove the
# following line to allow buffering (slightly better performance):
# Environment=PYTHONUNBUFFERED=x
[Unit]
Description=copyparty file server
[Service]
Type=notify
SyslogIdentifier=copyparty
Environment=PYTHONUNBUFFERED=x
WorkingDirectory=/var/lib/copyparty-jail
ExecReload=/bin/kill -s USR1 $MAINPID
# user to run as + where the TLS certificate is (if any)
User=cpp
Environment=XDG_CONFIG_HOME=/home/cpp/.config
# stop systemd-tmpfiles-clean.timer from deleting copyparty while it's running
ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
# run copyparty
ExecStart=/usr/bin/python3 /usr/bin/copyparty -c /etc/copyparty.d/init
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,3 @@
this is `/var/lib/copyparty-jail`, the fallback webroot when copyparty has not yet been configured
please add some `*.conf` files to `/etc/copyparty.d/`

View File

@@ -0,0 +1,31 @@
# this will start `/usr/bin/copyparty-sfx.py`
# in a chroot, preventing accidental access elsewhere
# and read config from `/etc/copyparty.d/*.conf`
#
# expose additional filesystem locations to copyparty
# by listing them between the last `1000` and `--`
#
# `1000 1000` = what user to run copyparty as
#
# unless you add -q to disable logging, you may want to remove the
# following line to allow buffering (slightly better performance):
# Environment=PYTHONUNBUFFERED=x
[Unit]
Description=copyparty file server
[Service]
SyslogIdentifier=prisonparty
Environment=PYTHONUNBUFFERED=x
WorkingDirectory=/var/lib/copyparty-jail
ExecReload=/bin/kill -s USR1 $MAINPID
# stop systemd-tmpfiles-clean.timer from deleting copyparty while it's running
ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
# run copyparty
ExecStart=/bin/bash /usr/bin/prisonparty /var/lib/copyparty-jail 1000 1000 /etc/copyparty.d -- \
/usr/bin/python3 /usr/bin/copyparty -c /etc/copyparty.d/init
[Install]
WantedBy=multi-user.target

View File

@@ -1,13 +1,22 @@
<!--
NOTE: DEPRECATED; please use the javascript version instead:
https://github.com/9001/copyparty/blob/hovudstraum/contrib/plugins/minimal-up2k.js
----
save this as .epilogue.html inside a write-only folder to declutter the UI, makes it look like
https://user-images.githubusercontent.com/241032/118311195-dd6ca380-b4ef-11eb-86f3-75a3ff2e1332.png
only works if you disable the prologue/epilogue sandbox with --no-sb-lg
which should probably be combined with --no-dot-ren to prevent damage
(`no_sb_lg` can also be set per-volume with volflags)
-->
<style>
/* make the up2k ui REALLY minimal by hiding a bunch of stuff: */
#ops, #tree, #path, #epi+h2, /* main tabs and navigators (tree/breadcrumbs) */
#ops, #tree, #path, #wfp, /* main tabs and navigators (tree/breadcrumbs) */
#u2conf tr:first-child>td[rowspan]:not(#u2btn_cw), /* most of the config options */

View File

@@ -17,7 +17,7 @@ almost the same as minimal-up2k.html except this one...:
var u2min = `
<style>
#ops, #path, #tree, #files, #epi+div+h2,
#ops, #path, #tree, #files, #wfp,
#u2conf td.c+.c, #u2cards, #srch_dz, #srch_zd {
display: none !important;
}
@@ -55,5 +55,5 @@ var u2min = `
if (!has(perms, 'read')) {
var e2 = mknod('div');
e2.innerHTML = u2min;
ebi('wrap').insertBefore(e2, QS('#epi+h2'));
ebi('wrap').insertBefore(e2, QS('#wfp'));
}

View File

@@ -6,12 +6,17 @@
# 1) put copyparty-sfx.py and prisonparty.sh in /usr/local/bin
# 2) cp -pv prisonparty.service /etc/systemd/system && systemctl enable --now prisonparty
#
# expose additional filesystem locations to copyparty
# by listing them between the last `1000` and `--`
#
# `1000 1000` = what user to run copyparty as
#
# you may want to:
# change '/mnt::rw' to another location or permission-set
# (remember to change the '/mnt' chroot arg too)
#
# enable line-buffering for realtime logging (slight performance cost):
# inside the [Service] block, add the following line:
# unless you add -q to disable logging, you may want to remove the
# following line to allow buffering (slightly better performance):
# Environment=PYTHONUNBUFFERED=x
[Unit]
@@ -19,7 +24,14 @@ Description=copyparty file server
[Service]
SyslogIdentifier=prisonparty
WorkingDirectory=/usr/local/bin
Environment=PYTHONUNBUFFERED=x
WorkingDirectory=/var/lib/copyparty-jail
ExecReload=/bin/kill -s USR1 $MAINPID
# stop systemd-tmpfiles-clean.timer from deleting copyparty while it's running
ExecStartPre=+/bin/bash -c 'mkdir -p /run/tmpfiles.d/ && echo "x /tmp/pe-copyparty*" > /run/tmpfiles.d/copyparty.conf'
# run copyparty
ExecStart=/bin/bash /usr/local/bin/prisonparty.sh /var/lib/copyparty-jail 1000 1000 /mnt -- \
/usr/bin/python3 /usr/local/bin/copyparty-sfx.py -q -v /mnt::rw

51
contrib/webdav-cfg.bat Normal file
View File

@@ -0,0 +1,51 @@
@echo off
rem removes the 47.6 MiB filesize limit when downloading from webdav
rem + optionally allows/enables password-auth over plaintext http
rem + optionally helps disable wpad, removing the 10sec latency
setlocal enabledelayedexpansion
net session >nul 2>&1
if %errorlevel% neq 0 (
echo sorry, you must run this as administrator
pause
exit /b
)
reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\WebClient\Parameters /v FileSizeLimitInBytes /t REG_DWORD /d 0xffffffff /f
reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\WebClient\Parameters /v FsCtlRequestTimeoutInSec /t REG_DWORD /d 0xffffffff /f
echo(
echo OK;
echo allow webdav basic-auth over plaintext http?
echo Y: login works, but the password will be visible in wireshark etc
echo N: login will NOT work unless you use https and valid certificates
set c=.
set /p "c=(Y/N): "
echo(
if /i not "!c!"=="y" goto :g1
reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\WebClient\Parameters /v BasicAuthLevel /t REG_DWORD /d 0x2 /f
rem default is 1 (require tls)
:g1
echo(
echo OK;
echo do you want to disable wpad?
echo can give a HUGE speed boost depending on network settings
set c=.
set /p "c=(Y/N): "
echo(
if /i not "!c!"=="y" goto :g2
echo(
echo i'm about to open the [Connections] tab in [Internet Properties] for you;
echo please click [LAN settings] and disable [Automatically detect settings]
echo(
pause
control inetcpl.cpl,,4
:g2
net stop webclient
net start webclient
echo(
echo OK; all done
pause

View File

@@ -7,16 +7,19 @@ import sys
import time
try:
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING
except:
TYPE_CHECKING = False
PY2 = sys.version_info[0] == 2
if PY2:
if True:
from typing import Any, Callable
PY2 = sys.version_info < (3,)
if not PY2:
unicode: Callable[[Any], str] = str
else:
sys.dont_write_bytecode = True
unicode = unicode # noqa: F821 # pylint: disable=undefined-variable,self-assigning-variable
else:
unicode = str
WINDOWS: Any = (
[int(x) for x in platform.version().split(".")]
@@ -40,8 +43,8 @@ except:
class EnvParams(object):
def __init__(self) -> None:
self.t0 = time.time()
self.mod = None
self.cfg = None
self.mod = ""
self.cfg = ""
self.ox = getattr(sys, "oxidized", None)

View File

@@ -9,41 +9,46 @@ __license__ = "MIT"
__url__ = "https://github.com/9001/copyparty/"
import argparse
import base64
import filecmp
import locale
import os
import re
import shutil
import socket
import sys
import threading
import time
import traceback
import uuid
from textwrap import dedent
from .__init__ import ANYWIN, CORES, PY2, VT100, WINDOWS, E, EnvParams, unicode
from .__version__ import CODENAME, S_BUILD_DT, S_VERSION
from .authsrv import re_vol
from .authsrv import expand_config_file, re_vol, split_cfg_ln, upgrade_cfg_fmt
from .cfg import flagcats, onedash
from .svchub import SvcHub
from .util import (
IMPLICATIONS,
JINJA_VER,
PYFTPD_VER,
SQLITE_VER,
UNPLICATIONS,
align_tab,
ansi_re,
is_exe,
min_ex,
py_desc,
pybin,
termsize,
wrap,
)
try:
if True: # pylint: disable=using-constant-test
from collections.abc import Callable
from types import FrameType
from typing import Any, Optional
except:
pass
try:
HAVE_SSL = True
@@ -51,7 +56,9 @@ try:
except:
HAVE_SSL = False
u = unicode
printed: list[str] = []
zsid = uuid.uuid4().urn[4:]
class RiceFormatter(argparse.HelpFormatter):
@@ -76,7 +83,11 @@ class RiceFormatter(argparse.HelpFormatter):
defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE]
if action.option_strings or action.nargs in defaulting_nargs:
ret += fmt
return ret
if not VT100:
ret = re.sub("\033\\[[0-9;]+m", "", ret)
return ret # type: ignore
def _fill_text(self, text: str, width: int, indent: str) -> str:
"""same as RawDescriptionHelpFormatter(HelpFormatter)"""
@@ -99,7 +110,7 @@ class RiceFormatter(argparse.HelpFormatter):
self.__add_whitespace(i, lWSpace, x)
for i, x in enumerate(wrap(line, width, width - 1))
]
textRows[idx] = lines
textRows[idx] = lines # type: ignore
return [item for sublist in textRows for item in sublist]
@@ -136,7 +147,7 @@ def init_E(E: EnvParams) -> None:
# __init__ runs 18 times when oxidized; do expensive stuff here
def get_unixdir() -> str:
paths: list[tuple[Callable[..., str], str]] = [
paths: list[tuple[Callable[..., Any], str]] = [
(os.environ.get, "XDG_CONFIG_HOME"),
(os.path.expanduser, "~/.config"),
(os.environ.get, "TMPDIR"),
@@ -158,7 +169,7 @@ def init_E(E: EnvParams) -> None:
if not os.path.isdir(p):
os.mkdir(p)
return p
return p # type: ignore
except:
pass
@@ -191,7 +202,8 @@ def init_E(E: EnvParams) -> None:
E.mod = _unpack()
if sys.platform == "win32":
E.cfg = os.path.normpath(os.environ["APPDATA"] + "/copyparty")
bdir = os.environ.get("APPDATA") or os.environ.get("TEMP")
E.cfg = os.path.normpath(bdir + "/copyparty")
elif sys.platform == "darwin":
E.cfg = os.path.expanduser("~/Library/Preferences/copyparty")
else:
@@ -205,19 +217,50 @@ def init_E(E: EnvParams) -> None:
raise
def get_srvname() -> str:
try:
ret: str = unicode(socket.gethostname()).split(".")[0]
except:
ret = ""
if ret not in ["", "localhost"]:
return ret
fp = os.path.join(E.cfg, "name.txt")
lprint("using hostname from {}\n".format(fp))
try:
with open(fp, "rb") as f:
ret = f.read().decode("utf-8", "replace").strip()
except:
ret = ""
namelen = 5
while len(ret) < namelen:
ret += base64.b32encode(os.urandom(4))[:7].decode("utf-8").lower()
ret = re.sub("[234567=]", "", ret)[:namelen]
with open(fp, "wb") as f:
f.write(ret.encode("utf-8") + b"\n")
return ret
def ensure_locale() -> None:
safe = "en_US.UTF-8"
for x in [
"en_US.UTF-8",
safe,
"English_United States.UTF8",
"English_United States.1252",
]:
try:
locale.setlocale(locale.LC_ALL, x)
lprint("Locale: {}\n".format(x))
break
if x != safe:
lprint("Locale: {}\n".format(x))
return
except:
continue
t = "setlocale {} failed,\n sorting and dates might get funky\n"
warn(t.format(safe))
def ensure_cert() -> None:
"""
@@ -235,8 +278,8 @@ def ensure_cert() -> None:
try:
if filecmp.cmp(cert_cfg, cert_insec):
lprint(
"\033[33m using default TLS certificate; https will be insecure."
+ "\033[36m\n certificate location: {}\033[0m\n".format(cert_cfg)
"\033[33musing default TLS certificate; https will be insecure."
+ "\033[36m\ncertificate location: {}\033[0m\n".format(cert_cfg)
)
except:
pass
@@ -313,27 +356,30 @@ def configure_ssl_ciphers(al: argparse.Namespace) -> None:
def args_from_cfg(cfg_path: str) -> list[str]:
lines: list[str] = []
expand_config_file(lines, cfg_path, "")
lines = upgrade_cfg_fmt(None, argparse.Namespace(vc=False), lines, "")
ret: list[str] = []
skip = False
with open(cfg_path, "rb") as f:
for ln in [x.decode("utf-8").strip() for x in f]:
if not ln:
skip = False
skip = True
for ln in lines:
sn = ln.split(" #")[0].strip()
if sn.startswith("["):
skip = True
if sn.startswith("[global]"):
skip = False
continue
if skip or not sn.split("#")[0].strip():
continue
for k, v in split_cfg_ln(sn).items():
k = k.lstrip("-")
if not k:
continue
if ln.startswith("#"):
continue
if not ln.startswith("-"):
continue
if skip:
continue
try:
ret.extend(ln.split(" ", 1))
except:
ret.append(ln)
prefix = "-" if k in onedash else "--"
if v is True:
ret.append(prefix + k)
else:
ret.append(prefix + k + "=" + v)
return ret
@@ -407,21 +453,8 @@ def showlic() -> None:
print(f.read().decode("utf-8", "replace"))
def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Namespace:
ap = argparse.ArgumentParser(
formatter_class=formatter,
prog="copyparty",
description="http file sharing hub v{} ({})".format(S_VERSION, S_BUILD_DT),
)
try:
fk_salt = unicode(os.path.getmtime(os.path.join(E.cfg, "cert.pem")))
except:
fk_salt = "hunter2"
hcores = min(CORES, 3) # 4% faster than 4+ on py3.9 @ r5-4500U
sects = [
def get_sects():
return [
[
"accounts",
"accounts and volumes",
@@ -438,8 +471,9 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
"m" (move): move files and folders; need "w" at destination
"d" (delete): permanently delete files and folders
"g" (get): download files, but cannot see folder contents
"G" (upget): "get", but can see filekeys of their own uploads
too many volflags to list here, see the other sections
too many volflags to list here, see --help-flags
example:\033[35m
-a ed:hunter2 -v .::r:rw,ed -v ../inc:dump:w:rw,ed:c,nodupe \033[36m
@@ -466,63 +500,53 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
"""
volflags are appended to volume definitions, for example,
to create a write-only volume with the \033[33mnodupe\033[0m and \033[32mnosub\033[0m flags:
\033[35m-v /mnt/inc:/inc:w\033[33m:c,nodupe\033[32m:c,nosub
\033[35m-v /mnt/inc:/inc:w\033[33m:c,nodupe\033[32m:c,nosub"""
)
+ build_flags_desc(),
],
[
"hooks",
"execute commands before/after various events",
dedent(
"""
execute a command (a program or script) before or after various events;
\033[36mxbu\033[35m executes CMD before a file upload starts
\033[36mxau\033[35m executes CMD after a file upload finishes
\033[36mxbr\033[35m executes CMD before a file rename/move
\033[36mxar\033[35m executes CMD after a file rename/move
\033[36mxbd\033[35m executes CMD before a file delete
\033[36mxad\033[35m executes CMD after a file delete
\033[36mxm\033[35m executes CMD on message
\033[0m
can be defined as --args or volflags; for example \033[36m
--xau notify-send
-v .::r:c,xau=notify-send
\033[0m
commands specified as --args are appended to volflags;
each --arg and volflag can be specified multiple times,
each command will execute in order unless one returns non-zero
\033[0muploads, general:
\033[36mnodupe\033[35m rejects existing files (instead of symlinking them)
\033[36mnosub\033[35m forces all uploads into the top folder of the vfs
\033[36mmagic$\033[35m enables filetype detection for nameless uploads
\033[36mgz\033[35m allows server-side gzip of uploads with ?gz (also c,xz)
\033[36mpk\033[35m forces server-side compression, optional arg: xz,9
optionally prefix the command with comma-sep. flags similar to -mtp:
\033[0mupload rules:
\033[36mmaxn=250,600\033[35m max 250 uploads over 15min
\033[36mmaxb=1g,300\033[35m max 1 GiB over 5min (suffixes: b, k, m, g)
\033[36msz=1k-3m\033[35m allow filesizes between 1 KiB and 3MiB
\033[36mdf=1g\033[35m ensure 1 GiB free disk space
\033[36mf\033[35m forks the process, doesn't wait for completion
\033[36mc\033[35m checks return code, blocks the action if non-zero
\033[36mj\033[35m provides json with info as 1st arg instead of filepath
\033[36mwN\033[35m waits N sec after command has been started before continuing
\033[36mtN\033[35m sets an N sec timeout before the command is abandoned
\033[0mupload rotation:
(moves all uploads into the specified folder structure)
\033[36mrotn=100,3\033[35m 3 levels of subfolders with 100 entries in each
\033[36mrotf=%Y-%m/%d-%H\033[35m date-formatted organizing
\033[36mlifetime=3600\033[35m uploads are deleted after 1 hour
\033[36mkt\033[35m kills the entire process tree on timeout (default),
\033[36mkm\033[35m kills just the main process
\033[36mkn\033[35m lets it continue running until copyparty is terminated
\033[0mdatabase, general:
\033[36me2d\033[35m sets -e2d (all -e2* args can be set using ce2* volflags)
\033[36md2ts\033[35m disables metadata collection for existing files
\033[36md2ds\033[35m disables onboot indexing, overrides -e2ds*
\033[36md2t\033[35m disables metadata collection, overrides -e2t*
\033[36md2v\033[35m disables file verification, overrides -e2v*
\033[36md2d\033[35m disables all database stuff, overrides -e2*
\033[36mhist=/tmp/cdb\033[35m puts thumbnails and indexes at that location
\033[36mscan=60\033[35m scan for new files every 60sec, same as --re-maxage
\033[36mnohash=\\.iso$\033[35m skips hashing file contents if path matches *.iso
\033[36mnoidx=\\.iso$\033[35m fully ignores the contents at paths matching *.iso
\033[36mnoforget$\033[35m don't forget files when deleted from disk
\033[36mxdev\033[35m do not descend into other filesystems
\033[36mxvol\033[35m skip symlinks leaving the volume root
\033[0mdatabase, audio tags:
"mte", "mth", "mtp", "mtm" all work the same as -mte, -mth, ...
\033[36mmtp=.bpm=f,audio-bpm.py\033[35m uses the "audio-bpm.py" program to
generate ".bpm" tags from uploads (f = overwrite tags)
\033[36mmtp=ahash,vhash=media-hash.py\033[35m collects two tags at once
\033[0mthumbnails:
\033[36mdthumb\033[35m disables all thumbnails
\033[36mdvthumb\033[35m disables video thumbnails
\033[36mdathumb\033[35m disables audio thumbnails (spectrograms)
\033[36mdithumb\033[35m disables image thumbnails
\033[0mclient and ux:
\033[36mhtml_head=TXT\033[35m includes TXT in the <head>
\033[36mrobots\033[35m allows indexing by search engines (default)
\033[36mnorobots\033[35m kindly asks search engines to leave
\033[0mothers:
\033[36mfk=8\033[35m generates per-file accesskeys,
which will then be required at the "g" permission
\033[0m"""
\033[36mc0\033[35m show all process output (default)
\033[36mc1\033[35m show only stderr
\033[36mc2\033[35m show only stdout
\033[36mc3\033[35m mute all process otput
\033[0m
except for \033[36mxm\033[0m, only one hook / one action can run at a time,
so it's recommended to use the \033[36mf\033[0m flag unless you really need
to wait for the hook to finish before continuing (without \033[36mf\033[0m
the upload speed can easily drop to 10% for small files)"""
),
],
[
@@ -558,79 +582,226 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
"""
),
],
[
"dbd",
"database durability profiles",
dedent(
"""
mainly affects uploads of many small files on slow HDDs; speeds measured uploading 520 files on a WD20SPZX (SMR 2.5" 5400rpm 4kb)
\033[32macid\033[0m = extremely safe but slow; the old default. Should never lose any data no matter what
\033[32mswal\033[0m = 2.4x faster uploads yet 99.9%% as safe -- theoretical chance of losing metadata for the ~200 most recently uploaded files if there's a power-loss or your OS crashes
\033[32mwal\033[0m = another 21x faster on HDDs yet 90%% as safe; same pitfall as \033[33mswal\033[0m except more likely
\033[32myolo\033[0m = another 1.5x faster, and removes the occasional sudden upload-pause while the disk syncs, but now you're at risk of losing the entire database in a powerloss / OS-crash
profiles can be set globally (--dbd=yolo), or per-volume with volflags: -v ~/Music:music:r:c,dbd=acid
"""
),
],
]
# fmt: off
u = unicode
def build_flags_desc():
ret = ""
for grp, flags in flagcats.items():
ret += "\n\n\033[0m" + grp
for k, v in flags.items():
v = v.replace("\n", "\n ")
ret += "\n \033[36m{}\033[35m {}".format(k, v)
return ret + "\033[0m"
# fmt: off
def add_general(ap, nc, srvname):
ap2 = ap.add_argument_group('general options')
ap2.add_argument("-c", metavar="PATH", type=u, action="append", help="add config file")
ap2.add_argument("-nc", metavar="NUM", type=int, default=64, help="max num clients")
ap2.add_argument("-nc", metavar="NUM", type=int, default=nc, help="max num clients")
ap2.add_argument("-j", metavar="CORES", type=int, default=1, help="max num cpu cores, 0=all")
ap2.add_argument("-a", metavar="ACCT", type=u, action="append", help="add account, USER:PASS; example [ed:wark]")
ap2.add_argument("-v", metavar="VOL", type=u, action="append", help="add volume, SRC:DST:FLAG; examples [.::r], [/mnt/nas/music:/music:r:aed]")
ap2.add_argument("-a", metavar="ACCT", type=u, action="append", help="add account, \033[33mUSER\033[0m:\033[33mPASS\033[0m; example [\033[32med:wark\033[0m]")
ap2.add_argument("-v", metavar="VOL", type=u, action="append", help="add volume, \033[33mSRC\033[0m:\033[33mDST\033[0m:\033[33mFLAG\033[0m; examples [\033[32m.::r\033[0m], [\033[32m/mnt/nas/music:/music:r:aed\033[0m]")
ap2.add_argument("-ed", action="store_true", help="enable the ?dots url parameter / client option which allows clients to see dotfiles / hidden files")
ap2.add_argument("-emp", action="store_true", help="enable markdown plugins -- neat but dangerous, big XSS risk")
ap2.add_argument("-mcr", metavar="SEC", type=int, default=60, help="md-editor mod-chk rate")
ap2.add_argument("--urlform", metavar="MODE", type=u, default="print,get", help="how to handle url-form POSTs; see --help-urlform")
ap2.add_argument("--wintitle", metavar="TXT", type=u, default="cpp @ $pub", help="window title, for example '$ip-10.1.2.' or '$ip-'")
ap2.add_argument("--wintitle", metavar="TXT", type=u, default="cpp @ $pub", help="window title, for example [\033[32m$ip-10.1.2.\033[0m] or [\033[32m$ip-]")
ap2.add_argument("--name", metavar="TXT", type=u, default=srvname, help="server name (displayed topleft in browser and in mDNS)")
ap2.add_argument("--license", action="store_true", help="show licenses and exit")
ap2.add_argument("--version", action="store_true", help="show versions and exit")
def add_qr(ap, tty):
ap2 = ap.add_argument_group('qr options')
ap2.add_argument("--qr", action="store_true", help="show http:// QR-code on startup")
ap2.add_argument("--qrs", action="store_true", help="show https:// QR-code on startup")
ap2.add_argument("--qrl", metavar="PATH", type=u, default="", help="location to include in the url, for example [\033[32mpriv/?pw=hunter2\033[0m]")
ap2.add_argument("--qri", metavar="PREFIX", type=u, default="", help="select IP which starts with PREFIX; [\033[32m.\033[0m] to force default IP when mDNS URL would have been used instead")
ap2.add_argument("--qr-fg", metavar="COLOR", type=int, default=0 if tty else 16, help="foreground; try [\033[32m0\033[0m] if the qr-code is unreadable")
ap2.add_argument("--qr-bg", metavar="COLOR", type=int, default=229, help="background (white=255)")
ap2.add_argument("--qrp", metavar="CELLS", type=int, default=4, help="padding (spec says 4 or more, but 1 is usually fine)")
ap2.add_argument("--qrz", metavar="N", type=int, default=0, help="[\033[32m1\033[0m]=1x, [\033[32m2\033[0m]=2x, [\033[32m0\033[0m]=auto (try [\033[32m2\033[0m] on broken fonts)")
def add_upload(ap):
ap2 = ap.add_argument_group('upload options')
ap2.add_argument("--dotpart", action="store_true", help="dotfile incomplete uploads, hiding them from clients unless -ed")
ap2.add_argument("--plain-ip", action="store_true", help="when avoiding filename collisions by appending the uploader's ip to the filename: append the plaintext ip instead of salting and hashing the ip")
ap2.add_argument("--unpost", metavar="SEC", type=int, default=3600*12, help="grace period where uploads can be deleted by the uploader, even without delete permissions; 0=disabled")
ap2.add_argument("--reg-cap", metavar="N", type=int, default=38400, help="max number of uploads to keep in memory when running without -e2d; roughly 1 MiB RAM per 600")
ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload")
ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even if copyparty thinks you're better off without -- probably useful on nfs and cow filesystems (zfs, btrfs)")
ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem)")
ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made")
ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead")
ap2.add_argument("--magic", action="store_true", help="enable filetype detection on nameless uploads")
ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload (very slow on windows)")
ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even when it might be dangerous (multiprocessing, filesystems lacking sparse-files support, ...)")
ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem) (volflag=hardlink)")
ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made (volflag=neversymlink)")
ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead (volflag=copydupes")
ap2.add_argument("--no-dupe", action="store_true", help="reject duplicate files during upload; only matches within the same volume (volflag=nodupe)")
ap2.add_argument("--no-snap", action="store_true", help="disable snapshots -- forget unfinished uploads on shutdown; don't create .hist/up2k.snap files -- abandoned/interrupted uploads must be cleaned up manually")
ap2.add_argument("--rand", action="store_true", help="force randomized filenames, --nrand chars long (volflag=rand)")
ap2.add_argument("--nrand", metavar="NUM", type=int, default=9, help="randomized filenames length (volflag=nrand)")
ap2.add_argument("--magic", action="store_true", help="enable filetype detection on nameless uploads (volflag=magic)")
ap2.add_argument("--df", metavar="GiB", type=float, default=0, help="ensure GiB free disk space by rejecting upload requests")
ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; 0 = off and warn if enabled, 1 = off, 2 = on, 3 = on and disable datecheck")
ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; s=smallest-first, n=alphabetical, fs=force-s, fn=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")
ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; [\033[32m0\033[0m] = off and warn if enabled, [\033[32m1\033[0m] = off, [\033[32m2\033[0m] = on, [\033[32m3\033[0m] = on and disable datecheck")
ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; [\033[32ms\033[0m]=smallest-first, [\033[32mn\033[0m]=alphabetical, [\033[32mfs\033[0m]=force-s, [\033[32mfn\033[0m]=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")
ap2.add_argument("--write-uplog", action="store_true", help="write POST reports to textfiles in working-directory")
def add_network(ap):
ap2 = ap.add_argument_group('network options')
ap2.add_argument("-i", metavar="IP", type=u, default="0.0.0.0", help="ip to bind (comma-sep.)")
ap2.add_argument("-i", metavar="IP", type=u, default="::", help="ip to bind (comma-sep.), default: all IPv4 and IPv6")
ap2.add_argument("-p", metavar="PORT", type=u, default="3923", help="ports to bind (comma/range)")
ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to keep; 0 = tcp, 1 = origin (first x-fwd), 2 = cloudflare, 3 = nginx, -1 = closest proxy")
ap2.add_argument("--ll", action="store_true", help="include link-local IPv4/IPv6 even if the NIC has routable IPs (breaks some mdns clients)")
ap2.add_argument("--rproxy", metavar="DEPTH", type=int, default=1, help="which ip to keep; [\033[32m0\033[0m]=tcp, [\033[32m1\033[0m]=origin (first x-fwd), [\033[32m2\033[0m]=cloudflare, [\033[32m3\033[0m]=nginx, [\033[32m-1\033[0m]=closest proxy")
ap2.add_argument("--rp-loc", metavar="PATH", type=u, default="", help="if reverse-proxying on a location instead of a dedicated domain/subdomain, provide the base location here (eg. /foo/bar)")
if ANYWIN:
ap2.add_argument("--reuseaddr", action="store_true", help="set reuseaddr on listening sockets on windows; allows rapid restart of copyparty at the expense of being able to accidentally start multiple instances")
else:
ap2.add_argument("--freebind", action="store_true", help="allow listening on IPs which do not yet exist, for example if the network interfaces haven't finished going up. Only makes sense for IPs other than '0.0.0.0', '127.0.0.1', '::', and '::1'. May require running as root (unless net.ipv6.ip_nonlocal_bind)")
ap2.add_argument("--s-wr-sz", metavar="B", type=int, default=256*1024, help="socket write size in bytes")
ap2.add_argument("--s-wr-slp", metavar="SEC", type=float, default=0, help="debug: socket write delay in seconds")
ap2.add_argument("--rsp-slp", metavar="SEC", type=float, default=0, help="debug: response delay in seconds")
def add_tls(ap):
ap2 = ap.add_argument_group('SSL/TLS options')
ap2.add_argument("--http-only", action="store_true", help="disable ssl/tls -- force plaintext")
ap2.add_argument("--https-only", action="store_true", help="disable plaintext -- force tls")
ap2.add_argument("--ssl-ver", metavar="LIST", type=u, help="set allowed ssl/tls versions; [help] shows available versions; default is what your python version considers safe")
ap2.add_argument("--ciphers", metavar="LIST", type=u, help="set allowed ssl/tls ciphers; [help] shows available ciphers")
ap2.add_argument("--ssl-ver", metavar="LIST", type=u, help="set allowed ssl/tls versions; [\033[32mhelp\033[0m] shows available versions; default is what your python version considers safe")
ap2.add_argument("--ciphers", metavar="LIST", type=u, help="set allowed ssl/tls ciphers; [\033[32mhelp\033[0m] shows available ciphers")
ap2.add_argument("--ssl-dbg", action="store_true", help="dump some tls info")
ap2.add_argument("--ssl-log", metavar="PATH", type=u, help="log master secrets for later decryption in wireshark")
ap2 = ap.add_argument_group('FTP options')
ap2.add_argument("--ftp", metavar="PORT", type=int, help="enable FTP server on PORT, for example 3921")
ap2.add_argument("--ftps", metavar="PORT", type=int, help="enable FTPS server on PORT, for example 3990")
ap2.add_argument("--ftp-dbg", action="store_true", help="enable debug logging")
ap2.add_argument("--ftp-nat", metavar="ADDR", type=u, help="the NAT address to use for passive connections")
ap2.add_argument("--ftp-pr", metavar="P-P", type=u, help="the range of TCP ports to use for passive connections, for example 12000-13000")
def add_zeroconf(ap):
ap2 = ap.add_argument_group("Zeroconf options")
ap2.add_argument("-z", action="store_true", help="enable all zeroconf backends (mdns, ssdp)")
ap2.add_argument("--z-on", metavar="NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes\n └─example: \033[32meth0, wlo1, virhost0, 192.168.123.0/24, fd00:fda::/96\033[0m")
ap2.add_argument("--z-off", metavar="NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--z-chk", metavar="SEC", type=int, default=10, help="check for network changes every SEC seconds (0=disable)")
ap2.add_argument("-zv", action="store_true", help="verbose all zeroconf backends")
ap2.add_argument("--mc-hop", metavar="SEC", type=int, default=0, help="rejoin multicast groups every SEC seconds (workaround for some switches/routers which cause mDNS to suddenly stop working after some time); try [\033[32m300\033[0m] or [\033[32m180\033[0m]")
def add_zc_mdns(ap):
ap2 = ap.add_argument_group("Zeroconf-mDNS options")
ap2.add_argument("--zm", action="store_true", help="announce the enabled protocols over mDNS (multicast DNS-SD) -- compatible with KDE, gnome, macOS, ...")
ap2.add_argument("--zm-on", metavar="NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--zm-off", metavar="NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--zm4", action="store_true", help="IPv4 only -- try this if some clients can't connect")
ap2.add_argument("--zm6", action="store_true", help="IPv6 only")
ap2.add_argument("--zmv", action="store_true", help="verbose mdns")
ap2.add_argument("--zmvv", action="store_true", help="verboser mdns")
ap2.add_argument("--zms", metavar="dhf", type=u, default="", help="list of services to announce -- d=webdav h=http f=ftp s=smb -- lowercase=plaintext uppercase=TLS -- default: all enabled services except http/https (\033[32mDdfs\033[0m if \033[33m--ftp\033[0m and \033[33m--smb\033[0m is set)")
ap2.add_argument("--zm-ld", metavar="PATH", type=u, default="", help="link a specific folder for webdav shares")
ap2.add_argument("--zm-lh", metavar="PATH", type=u, default="", help="link a specific folder for http shares")
ap2.add_argument("--zm-lf", metavar="PATH", type=u, default="", help="link a specific folder for ftp shares")
ap2.add_argument("--zm-ls", metavar="PATH", type=u, default="", help="link a specific folder for smb shares")
ap2.add_argument("--zm-mnic", action="store_true", help="merge NICs which share subnets; assume that same subnet means same network")
ap2.add_argument("--zm-msub", action="store_true", help="merge subnets on each NIC -- always enabled for ipv6 -- reduces network load, but gnome-gvfs clients may stop working")
ap2.add_argument("--zm-noneg", action="store_true", help="disable NSEC replies -- try this if some clients don't see copyparty")
def add_zc_ssdp(ap):
ap2 = ap.add_argument_group("Zeroconf-SSDP options")
ap2.add_argument("--zs", action="store_true", help="announce the enabled protocols over SSDP -- compatible with Windows")
ap2.add_argument("--zs-on", metavar="NETS", type=u, default="", help="enable zeroconf ONLY on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--zs-off", metavar="NETS", type=u, default="", help="disable zeroconf on the comma-separated list of subnets and/or interface names/indexes")
ap2.add_argument("--zsv", action="store_true", help="verbose SSDP")
ap2.add_argument("--zsl", metavar="PATH", type=u, default="/?hc", help="location to include in the url (or a complete external URL), for example [\033[32mpriv/?pw=hunter2\033[0m] (goes directly to /priv/ with password hunter2) or [\033[32m?hc=priv&pw=hunter2\033[0m] (shows mounting options for /priv/ with password)")
ap2.add_argument("--zsid", metavar="UUID", type=u, default=zsid, help="USN (device identifier) to announce")
def add_ftp(ap):
ap2 = ap.add_argument_group('FTP options')
ap2.add_argument("--ftp", metavar="PORT", type=int, help="enable FTP server on PORT, for example \033[32m3921")
ap2.add_argument("--ftps", metavar="PORT", type=int, help="enable FTPS server on PORT, for example \033[32m3990")
ap2.add_argument("--ftpv", action="store_true", help="verbose")
ap2.add_argument("--ftp-wt", metavar="SEC", type=int, default=7, help="grace period for resuming interrupted uploads (any client can write to any file last-modified more recently than SEC seconds ago)")
ap2.add_argument("--ftp-nat", metavar="ADDR", type=u, help="the NAT address to use for passive connections")
ap2.add_argument("--ftp-pr", metavar="P-P", type=u, help="the range of TCP ports to use for passive connections, for example \033[32m12000-13000")
def add_webdav(ap):
ap2 = ap.add_argument_group('WebDAV options')
ap2.add_argument("--daw", action="store_true", help="enable full write support. \033[1;31mWARNING:\033[0m This has side-effects -- PUT-operations will now \033[1;31mOVERWRITE\033[0m existing files, rather than inventing new filenames to avoid loss of data. You might want to instead set this as a volflag where needed. By not setting this flag, uploaded files can get written to a filename which the client does not expect (which might be okay, depending on client)")
ap2.add_argument("--dav-inf", action="store_true", help="allow depth:infinite requests (recursive file listing); extremely server-heavy but required for spec compliance -- luckily few clients rely on this")
ap2.add_argument("--dav-mac", action="store_true", help="disable apple-garbage filter -- allow macos to create junk files (._* and .DS_Store, .Spotlight-*, .fseventsd, .Trashes, .AppleDouble, __MACOS)")
def add_smb(ap):
ap2 = ap.add_argument_group('SMB/CIFS options')
ap2.add_argument("--smb", action="store_true", help="enable smb (read-only) -- this requires running copyparty as root on linux and macos unless --smb-port is set above 1024 and your OS does port-forwarding from 445 to that.\n\033[1;31mWARNING:\033[0m this protocol is dangerous! Never expose to the internet. Account permissions are coalesced; if one account has write-access to a volume, then all accounts do.")
ap2.add_argument("--smbw", action="store_true", help="enable write support (please dont)")
ap2.add_argument("--smb1", action="store_true", help="disable SMBv2, only enable SMBv1 (CIFS)")
ap2.add_argument("--smb-port", metavar="PORT", type=int, default=445, help="port to listen on -- if you change this value, you must NAT from TCP:445 to this port using iptables or similar")
ap2.add_argument("--smb-nwa-1", action="store_true", help="disable impacket#1433 workaround (truncate directory listings to 64kB)")
ap2.add_argument("--smb-nwa-2", action="store_true", help="disable impacket workaround for filecopy globs")
ap2.add_argument("--smbv", action="store_true", help="verbose")
ap2.add_argument("--smbvv", action="store_true", help="verboser")
ap2.add_argument("--smbvvv", action="store_true", help="verbosest")
def add_hooks(ap):
ap2 = ap.add_argument_group('event hooks (see --help-hooks)')
ap2.add_argument("--xbu", metavar="CMD", type=u, action="append", help="execute CMD before a file upload starts")
ap2.add_argument("--xau", metavar="CMD", type=u, action="append", help="execute CMD after a file upload finishes")
ap2.add_argument("--xbr", metavar="CMD", type=u, action="append", help="execute CMD before a file move/rename")
ap2.add_argument("--xar", metavar="CMD", type=u, action="append", help="execute CMD after a file move/rename")
ap2.add_argument("--xbd", metavar="CMD", type=u, action="append", help="execute CMD before a file delete")
ap2.add_argument("--xad", metavar="CMD", type=u, action="append", help="execute CMD after a file delete")
ap2.add_argument("--xm", metavar="CMD", type=u, action="append", help="execute CMD on message")
def add_yolo(ap):
ap2 = ap.add_argument_group('yolo options')
ap2.add_argument("--allow-csrf", action="store_true", help="disable csrf protections; let other domains/sites impersonate you through cross-site requests")
ap2.add_argument("--getmod", action="store_true", help="permit ?move=[...] and ?delete as GET")
def add_optouts(ap):
ap2 = ap.add_argument_group('opt-outs')
ap2.add_argument("-nw", action="store_true", help="never write anything to disk (debug/benchmark)")
ap2.add_argument("--keep-qem", action="store_true", help="do not disable quick-edit-mode on windows (it is disabled to avoid accidental text selection which will deadlock copyparty)")
ap2.add_argument("--no-dav", action="store_true", help="disable webdav support")
ap2.add_argument("--no-del", action="store_true", help="disable delete operations")
ap2.add_argument("--no-mv", action="store_true", help="disable move/rename operations")
ap2.add_argument("-nih", action="store_true", help="no info hostname -- don't show in UI")
ap2.add_argument("-nid", action="store_true", help="no info disk-usage -- don't show in UI")
ap2.add_argument("-nb", action="store_true", help="no powered-by-copyparty branding in UI")
ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
ap2.add_argument("--no-lifetime", action="store_true", help="disable automatic deletion of uploads after a certain time (lifetime volflag)")
ap2.add_argument("--no-lifetime", action="store_true", help="disable automatic deletion of uploads after a certain time (as specified by the 'lifetime' volflag)")
def add_safety(ap, fk_salt):
ap2 = ap.add_argument_group('safety options')
ap2.add_argument("-s", action="count", default=0, help="increase safety: Disable thumbnails / potentially dangerous software (ffmpeg/pillow/vips), hide partial uploads, avoid crawlers.\n └─Alias of\033[32m --dotpart --no-thumb --no-mtag-ff --no-robots --force-js")
ap2.add_argument("-ss", action="store_true", help="further increase safety: Prevent js-injection, accidental move/delete, broken symlinks, 404 on 403, ban on excessive 404s.\n └─Alias of\033[32m -s --no-dot-mv --no-dot-ren --unpost=0 --no-del --no-mv --hardlink --vague-403 --ban-404=50,60,1440 -nih")
ap2.add_argument("-sss", action="store_true", help="further increase safety: Enable logging to disk, scan for dangerous symlinks.\n └─Alias of\033[32m -ss -lo=cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz --ls=**,*,ln,p,r")
ap2.add_argument("--ls", metavar="U[,V[,F]]", type=u, help="do a sanity/safety check of all volumes on startup; arguments USER,VOL,FLAGS; example [**,*,ln,p,r]")
ap2.add_argument("-ss", action="store_true", help="further increase safety: Prevent js-injection, accidental move/delete, broken symlinks, webdav, 404 on 403, ban on excessive 404s.\n └─Alias of\033[32m -s --unpost=0 --no-del --no-mv --hardlink --vague-403 --ban-404=50,60,1440 -nih")
ap2.add_argument("-sss", action="store_true", help="further increase safety: Enable logging to disk, scan for dangerous symlinks.\n └─Alias of\033[32m -ss --no-dav --no-logues --no-readme -lo=cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz --ls=**,*,ln,p,r")
ap2.add_argument("--ls", metavar="U[,V[,F]]", type=u, help="do a sanity/safety check of all volumes on startup; arguments \033[33mUSER\033[0m,\033[33mVOL\033[0m,\033[33mFLAGS\033[0m; example [\033[32m**,*,ln,p,r\033[0m]")
ap2.add_argument("--salt", type=u, default="hunter2", help="up2k file-hash salt; used to generate unpredictable internal identifiers for uploads -- doesn't really matter")
ap2.add_argument("--fk-salt", metavar="SALT", type=u, default=fk_salt, help="per-file accesskey salt; used to generate unpredictable URLs for hidden files -- this one DOES matter")
ap2.add_argument("--no-dot-mv", action="store_true", help="disallow moving dotfiles; makes it impossible to move folders containing dotfiles")
@@ -639,34 +810,46 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
ap2.add_argument("--no-readme", action="store_true", help="disable rendering readme.md into directory listings")
ap2.add_argument("--vague-403", action="store_true", help="send 404 instead of 403 (security through ambiguity, very enterprise)")
ap2.add_argument("--force-js", action="store_true", help="don't send folder listings as HTML, force clients to use the embedded json instead -- slight protection against misbehaving search engines which ignore --no-robots")
ap2.add_argument("--no-robots", action="store_true", help="adds http and html headers asking search engines to not index anything")
ap2.add_argument("--logout", metavar="H", type=float, default="8086", help="logout clients after H hours of inactivity (0.0028=10sec, 0.1=6min, 24=day, 168=week, 720=month, 8760=year)")
ap2.add_argument("--ban-pw", metavar="N,W,B", type=u, default="9,60,1440", help="more than N wrong passwords in W minutes = ban for B minutes (disable with \"no\")")
ap2.add_argument("--ban-404", metavar="N,W,B", type=u, default="no", help="hitting more than N 404's in W minutes = ban for B minutes (disabled by default since turbo-up2k counts as 404s)")
ap2.add_argument("--no-robots", action="store_true", help="adds http and html headers asking search engines to not index anything (volflag=norobots)")
ap2.add_argument("--logout", metavar="H", type=float, default="8086", help="logout clients after H hours of inactivity; [\033[32m0.0028\033[0m]=10sec, [\033[32m0.1\033[0m]=6min, [\033[32m24\033[0m]=day, [\033[32m168\033[0m]=week, [\033[32m720\033[0m]=month, [\033[32m8760\033[0m]=year)")
ap2.add_argument("--ban-pw", metavar="N,W,B", type=u, default="9,60,1440", help="more than \033[33mN\033[0m wrong passwords in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes; disable with [\033[32mno\033[0m]")
ap2.add_argument("--ban-404", metavar="N,W,B", type=u, default="no", help="hitting more than \033[33mN\033[0m 404's in \033[33mW\033[0m minutes = ban for \033[33mB\033[0m minutes (disabled by default since turbo-up2k counts as 404s)")
ap2.add_argument("--aclose", metavar="MIN", type=int, default=10, help="if a client maxes out the server connection limit, downgrade it from connection:keep-alive to connection:close for MIN minutes (and also kill its active connections) -- disable with 0")
ap2.add_argument("--loris", metavar="B", type=int, default=60, help="if a client maxes out the server connection limit without sending headers, ban it for B minutes; disable with [\033[32m0\033[0m]")
ap2.add_argument("--acao", metavar="V[,V]", type=u, default="*", help="Access-Control-Allow-Origin; list of origins (domains/IPs without port) to accept requests from; [\033[32mhttps://1.2.3.4\033[0m]. Default [\033[32m*\033[0m] allows requests from all sites but removes cookies and http-auth; only ?pw=hunter2 survives")
ap2.add_argument("--acam", metavar="V[,V]", type=u, default="GET,HEAD", help="Access-Control-Allow-Methods; list of methods to accept from offsite ('*' behaves like described in --acao)")
def add_shutdown(ap):
ap2 = ap.add_argument_group('shutdown options')
ap2.add_argument("--ign-ebind", action="store_true", help="continue running even if it's impossible to listen on some of the requested endpoints")
ap2.add_argument("--ign-ebind-all", action="store_true", help="continue running even if it's impossible to receive connections at all")
ap2.add_argument("--exit", metavar="WHEN", type=u, default="", help="shutdown after WHEN has finished; for example 'idx' will do volume indexing + metadata analysis")
ap2.add_argument("--exit", metavar="WHEN", type=u, default="", help="shutdown after WHEN has finished; [\033[32mcfg\033[0m] config parsing, [\033[32midx\033[0m] volscan + multimedia indexing")
def add_logging(ap):
ap2 = ap.add_argument_group('logging options')
ap2.add_argument("-q", action="store_true", help="quiet")
ap2.add_argument("-lo", metavar="PATH", type=u, help="logfile, example: cpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz")
ap2.add_argument("-lo", metavar="PATH", type=u, help="logfile, example: \033[32mcpp-%%Y-%%m%%d-%%H%%M%%S.txt.xz")
ap2.add_argument("--no-voldump", action="store_true", help="do not list volumes and permissions on startup")
ap2.add_argument("--log-conn", action="store_true", help="debug: print tcp-server msgs")
ap2.add_argument("--log-htp", action="store_true", help="debug: print http-server threadpool scaling")
ap2.add_argument("--ihead", metavar="HEADER", type=u, action='append', help="dump incoming header")
ap2.add_argument("--lf-url", metavar="RE", type=u, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching")
ap2.add_argument("--lf-url", metavar="RE", type=u, default=r"^/\.cpr/|\?th=[wj]$|/\.(_|ql_|DS_Store$|localized$)", help="dont log URLs matching")
def add_admin(ap):
ap2 = ap.add_argument_group('admin panel options')
ap2.add_argument("--no-reload", action="store_true", help="disable ?reload=cfg (reload users/volumes/volflags from config file)")
ap2.add_argument("--no-rescan", action="store_true", help="disable ?scan (volume reindexing)")
ap2.add_argument("--no-stack", action="store_true", help="disable ?stack (list all stacks)")
def add_thumbnail(ap):
ap2 = ap.add_argument_group('thumbnail options')
ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails")
ap2.add_argument("--no-athumb", action="store_true", help="disable audio thumbnails (spectrograms)")
ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails")
ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails (volflag=dthumb)")
ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails (volflag=dvthumb)")
ap2.add_argument("--no-athumb", action="store_true", help="disable audio thumbnails (spectrograms) (volflag=dathumb)")
ap2.add_argument("--th-size", metavar="WxH", default="320x256", help="thumbnail res")
ap2.add_argument("--th-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for generating thumbnails")
ap2.add_argument("--th-convt", metavar="SEC", type=int, default=60, help="conversion timeout in seconds")
@@ -683,36 +866,46 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
# https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
# https://github.com/libvips/libvips
# ffmpeg -hide_banner -demuxers | awk '/^ D /{print$2}' | while IFS= read -r x; do ffmpeg -hide_banner -h demuxer=$x; done | grep -E '^Demuxer |extensions:'
ap2.add_argument("--th-r-pil", metavar="T,T", type=u, default="bmp,dib,gif,icns,ico,jpg,jpeg,jp2,jpx,pcx,png,pbm,pgm,ppm,pnm,sgi,tga,tif,tiff,webp,xbm,dds,xpm,heif,heifs,heic,heics,avif,avifs", help="image formats to decode using pillow")
ap2.add_argument("--th-r-vips", metavar="T,T", type=u, default="jpg,jpeg,jp2,jpx,jxl,tif,tiff,png,webp,heic,avif,fit,fits,fts,exr,svg,hdr,ppm,pgm,pfm,gif,nii", help="image formats to decode using pyvips")
ap2.add_argument("--th-r-ffi", metavar="T,T", type=u, default="apng,avif,avifs,bmp,dds,dib,fit,fits,fts,gif,heic,heics,heif,heifs,icns,ico,jp2,jpeg,jpg,jpx,jxl,pbm,pcx,pfm,pgm,png,pnm,ppm,psd,sgi,tga,tif,tiff,webp,xbm,xpm", help="image formats to decode using ffmpeg")
ap2.add_argument("--th-r-ffv", metavar="T,T", type=u, default="av1,asf,avi,flv,m4v,mkv,mjpeg,mjpg,mpg,mpeg,mpg2,mpeg2,h264,avc,mts,h265,hevc,mov,3gp,mp4,ts,mpegts,nut,ogv,ogm,rm,vob,webm,wmv", help="video formats to decode using ffmpeg")
ap2.add_argument("--th-r-ffa", metavar="T,T", type=u, default="aac,m4a,ogg,opus,flac,alac,mp3,mp2,ac3,dts,wma,ra,wav,aif,aiff,au,alaw,ulaw,mulaw,amr,gsm,ape,tak,tta,wv,mpc", help="audio formats to decode using ffmpeg")
ap2.add_argument("--th-r-pil", metavar="T,T", type=u, default="avif,avifs,blp,bmp,dcx,dds,dib,emf,eps,fits,flc,fli,fpx,gif,heic,heics,heif,heifs,icns,ico,im,j2p,j2k,jp2,jpeg,jpg,jpx,pbm,pcx,pgm,png,pnm,ppm,psd,sgi,spi,tga,tif,tiff,webp,wmf,xbm,xpm", help="image formats to decode using pillow")
ap2.add_argument("--th-r-vips", metavar="T,T", type=u, default="avif,exr,fit,fits,fts,gif,hdr,heic,jp2,jpeg,jpg,jpx,jxl,nii,pfm,pgm,png,ppm,svg,tif,tiff,webp", help="image formats to decode using pyvips")
ap2.add_argument("--th-r-ffi", metavar="T,T", type=u, default="apng,avif,avifs,bmp,dds,dib,fit,fits,fts,gif,hdr,heic,heics,heif,heifs,icns,ico,jp2,jpeg,jpg,jpx,jxl,pbm,pcx,pfm,pgm,png,pnm,ppm,psd,sgi,tga,tif,tiff,webp,xbm,xpm", help="image formats to decode using ffmpeg")
ap2.add_argument("--th-r-ffv", metavar="T,T", type=u, default="3gp,asf,av1,avc,avi,flv,h264,h265,hevc,m4v,mjpeg,mjpg,mkv,mov,mp4,mpeg,mpeg2,mpegts,mpg,mpg2,mts,nut,ogm,ogv,rm,ts,vob,webm,wmv", help="video formats to decode using ffmpeg")
ap2.add_argument("--th-r-ffa", metavar="T,T", type=u, default="aac,ac3,aif,aiff,alac,alaw,amr,apac,ape,au,bonk,dfpwm,dts,flac,gsm,ilbc,it,m4a,mo3,mod,mp2,mp3,mpc,mptm,mt2,mulaw,ogg,okt,opus,ra,s3m,tak,tta,ulaw,wav,wma,wv,xm,xpk", help="audio formats to decode using ffmpeg")
def add_transcoding(ap):
ap2 = ap.add_argument_group('transcoding options')
ap2.add_argument("--no-acode", action="store_true", help="disable audio transcoding")
ap2.add_argument("--ac-maxage", metavar="SEC", type=int, default=86400, help="delete cached transcode output after SEC seconds")
def add_db_general(ap, hcores):
ap2 = ap.add_argument_group('general db options')
ap2.add_argument("-e2d", action="store_true", help="enable up2k database, making files searchable + enables upload deduplocation")
ap2.add_argument("-e2d", action="store_true", help="enable up2k database, making files searchable + enables upload deduplication")
ap2.add_argument("-e2ds", action="store_true", help="scan writable folders for new files on startup; sets -e2d")
ap2.add_argument("-e2dsa", action="store_true", help="scans all folders on startup; sets -e2ds")
ap2.add_argument("-e2v", action="store_true", help="verify file integrity; rehash all files and compare with db")
ap2.add_argument("-e2vu", action="store_true", help="on hash mismatch: update the database with the new hash")
ap2.add_argument("-e2vp", action="store_true", help="on hash mismatch: panic and quit copyparty")
ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume data (db, thumbs)")
ap2.add_argument("--no-hash", metavar="PTN", type=u, help="regex: disable hashing of matching paths during e2ds folder scans")
ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans")
ap2.add_argument("--hist", metavar="PATH", type=u, help="where to store volume data (db, thumbs) (volflag=hist)")
ap2.add_argument("--no-hash", metavar="PTN", type=u, help="regex: disable hashing of matching paths during e2ds folder scans (volflag=nohash)")
ap2.add_argument("--no-idx", metavar="PTN", type=u, help="regex: disable indexing of matching paths during e2ds folder scans (volflag=noidx)")
ap2.add_argument("--no-dhash", action="store_true", help="disable rescan acceleration; do full database integrity check -- makes the db ~5%% smaller and bootup/rescans 3~10x slower")
ap2.add_argument("--no-forget", action="store_true", help="never forget indexed files, even when deleted from disk -- makes it impossible to ever upload the same file twice")
ap2.add_argument("--xdev", action="store_true", help="do not descend into other filesystems (symlink or bind-mount to another HDD, ...)")
ap2.add_argument("--xvol", action="store_true", help="skip symlinks leaving the volume root")
ap2.add_argument("--re-dhash", action="store_true", help="rebuild the cache if it gets out of sync (for example crash on startup during metadata scanning)")
ap2.add_argument("--no-forget", action="store_true", help="never forget indexed files, even when deleted from disk -- makes it impossible to ever upload the same file twice (volflag=noforget)")
ap2.add_argument("--dbd", metavar="PROFILE", default="wal", help="database durability profile; sets the tradeoff between robustness and speed, see --help-dbd (volflag=dbd)")
ap2.add_argument("--xlink", action="store_true", help="on upload: check all volumes for dupes, not just the target volume (volflag=xlink)")
ap2.add_argument("--xdev", action="store_true", help="do not descend into other filesystems (symlink or bind-mount to another HDD, ...) (volflag=xdev)")
ap2.add_argument("--xvol", action="store_true", help="skip symlinks leaving the volume root (volflag=xvol)")
ap2.add_argument("--hash-mt", metavar="CORES", type=int, default=hcores, help="num cpu cores to use for file hashing; set 0 or 1 for single-core hashing")
ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off, can be set per-volume with the 'scan' volflag")
ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="disk rescan volume interval, 0=off (volflag=scan)")
ap2.add_argument("--db-act", metavar="SEC", type=float, default=10, help="defer any scheduled volume reindexing until SEC seconds after last db write (uploads, renames, ...)")
ap2.add_argument("--srch-time", metavar="SEC", type=int, default=45, help="search deadline -- terminate searches running for more than SEC seconds")
ap2.add_argument("--srch-hits", metavar="N", type=int, default=7999, help="max search results to allow clients to fetch; 125 results will be shown initially")
ap2.add_argument("--dotsrch", action="store_true", help="show dotfiles in search results (volflags: dotsrch | nodotsrch)")
def add_db_metadata(ap):
ap2 = ap.add_argument_group('metadata db options')
ap2.add_argument("-e2t", action="store_true", help="enable metadata indexing; makes it possible to search for artist/title/codec/resolution/...")
ap2.add_argument("-e2ts", action="store_true", help="scan existing files on startup; sets -e2t")
@@ -722,7 +915,7 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
ap2.add_argument("--mtag-to", metavar="SEC", type=int, default=60, help="timeout for ffprobe tag-scan")
ap2.add_argument("--mtag-mt", metavar="CORES", type=int, default=CORES, help="num cpu cores to use for tag scanning")
ap2.add_argument("--mtag-v", action="store_true", help="verbose tag scanning; print errors from mtp subprocesses and such")
ap2.add_argument("--mtag-vv", action="store_true", help="debug mtp settings")
ap2.add_argument("--mtag-vv", action="store_true", help="debug mtp settings and mutagen/ffprobe parsers")
ap2.add_argument("-mtm", metavar="M=t,t,t", type=u, action="append", help="add/replace metadata mapping")
ap2.add_argument("-mte", metavar="M,M,M", type=u, help="tags to index/display (comma-sep.)",
default="circle,album,.tn,artist,title,.bpm,key,.dur,.q,.vq,.aq,vc,ac,fmt,res,.fps,ahash,vhash")
@@ -730,32 +923,111 @@ def run_argparse(argv: list[str], formatter: Any, retry: bool) -> argparse.Names
default=".vq,.aq,vc,ac,fmt,res,.fps")
ap2.add_argument("-mtp", metavar="M=[f,]BIN", type=u, action="append", help="read tag M using program BIN to parse the file")
def add_ui(ap, retry):
ap2 = ap.add_argument_group('ui options')
ap2.add_argument("--lang", metavar="LANG", type=u, default="eng", help="language")
ap2.add_argument("--theme", metavar="NUM", type=int, default=0, help="default theme to use")
ap2.add_argument("--themes", metavar="NUM", type=int, default=8, help="number of themes installed")
ap2.add_argument("--favico", metavar="TXT", type=u, default="c 000 none" if retry else "🎉 000 none", help="favicon text [ foreground [ background ] ], set blank to disable")
ap2.add_argument("--favico", metavar="TXT", type=u, default="c 000 none" if retry else "🎉 000 none", help="\033[33mfavicon-text\033[0m [ \033[33mforeground\033[0m [ \033[33mbackground\033[0m ] ], set blank to disable")
ap2.add_argument("--mpmc", metavar="URL", type=u, default="", help="change the mediaplayer-toggle mouse cursor; URL to a folder with {2..5}.png inside (or disable with [\033[32m.\033[0m])")
ap2.add_argument("--js-browser", metavar="L", type=u, help="URL to additional JS to include")
ap2.add_argument("--css-browser", metavar="L", type=u, help="URL to additional CSS to include")
ap2.add_argument("--html-head", metavar="TXT", type=u, default="", help="text to append to the <head> of all HTML pages")
ap2.add_argument("--textfiles", metavar="CSV", type=u, default="txt,nfo,diz,cue,readme", help="file extensions to present as plaintext")
ap2.add_argument("--txt-max", metavar="KiB", type=int, default=64, help="max size of embedded textfiles on ?doc= (anything bigger will be lazy-loaded by JS)")
ap2.add_argument("--doctitle", metavar="TXT", type=u, default="copyparty", help="title / service-name to show in html documents")
ap2.add_argument("--pb-url", metavar="URL", type=u, default="https://github.com/9001/copyparty", help="powered-by link; disable with -np")
ap2.add_argument("--ver", action="store_true", help="show version on the control panel (incompatible by -np)")
ap2.add_argument("--md-sbf", metavar="FLAGS", type=u, default="downloads forms popups scripts top-navigation-by-user-activation", help="list of capabilities to ALLOW for README.md docs (volflag=md_sbf); see https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe#attr-sandbox")
ap2.add_argument("--lg-sbf", metavar="FLAGS", type=u, default="downloads forms popups scripts top-navigation-by-user-activation", help="list of capabilities to ALLOW for prologue/epilogue docs (volflag=lg_sbf)")
ap2.add_argument("--no-sb-md", action="store_true", help="don't sandbox README.md documents (volflags: no_sb_md | sb_md)")
ap2.add_argument("--no-sb-lg", action="store_true", help="don't sandbox prologue/epilogue docs (volflags: no_sb_lg | sb_lg); enables non-js support")
def add_debug(ap):
ap2 = ap.add_argument_group('debug options')
ap2.add_argument("--vc", action="store_true", help="verbose config file parser (explain config)")
ap2.add_argument("--cgen", action="store_true", help="generate config file from current config (best-effort; probably buggy)")
ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile; instead using a traditional file read loop")
ap2.add_argument("--no-scandir", action="store_true", help="disable scandir; instead using listdir + stat on each file")
ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing before starting the httpd")
ap2.add_argument("--no-htp", action="store_true", help="disable httpserver threadpool, create threads as-needed instead")
ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second, for example --stackmon=./st/%%Y-%%m/%%d/%%H%%M.xz,60")
ap2.add_argument("--rclone-mdns", action="store_true", help="use mdns-domain instead of server-ip on /?hc")
ap2.add_argument("--stackmon", metavar="P,S", type=u, help="write stacktrace to Path every S second, for example --stackmon=\033[32m./st/%%Y-%%m/%%d/%%H%%M.xz,60")
ap2.add_argument("--log-thrs", metavar="SEC", type=float, help="list active threads every SEC")
ap2.add_argument("--log-fk", metavar="REGEX", type=u, default="", help="log filekey params for files where path matches REGEX; '.' (a single dot) = all files")
# fmt: on
ap2.add_argument("--log-fk", metavar="REGEX", type=u, default="", help="log filekey params for files where path matches REGEX; [\033[32m.\033[0m] (a single dot) = all files")
ap2.add_argument("--bak-flips", action="store_true", help="[up2k] if a client uploads a bitflipped/corrupted chunk, store a copy according to --bf-nc and --bf-dir")
ap2.add_argument("--bf-nc", metavar="NUM", type=int, default=200, help="bak-flips: stop if there's more than NUM files at --kf-dir already; default: 6.3 GiB max (200*32M)")
ap2.add_argument("--bf-dir", metavar="PATH", type=u, default="bf", help="bak-flips: store corrupted chunks at PATH; default: folder named 'bf' wherever copyparty was started")
# fmt: on
def run_argparse(
argv: list[str], formatter: Any, retry: bool, nc: int
) -> argparse.Namespace:
ap = argparse.ArgumentParser(
formatter_class=formatter,
prog="copyparty",
description="http file sharing hub v{} ({})".format(S_VERSION, S_BUILD_DT),
)
try:
fk_salt = unicode(os.path.getmtime(os.path.join(E.cfg, "cert.pem")))
except:
fk_salt = "hunter2"
hcores = min(CORES, 4) # optimal on py3.11 @ r5-4500U
tty = os.environ.get("TERM", "").lower() == "linux"
srvname = get_srvname()
add_general(ap, nc, srvname)
add_network(ap)
add_tls(ap)
add_qr(ap, tty)
add_zeroconf(ap)
add_zc_mdns(ap)
add_zc_ssdp(ap)
add_upload(ap)
add_db_general(ap, hcores)
add_db_metadata(ap)
add_thumbnail(ap)
add_transcoding(ap)
add_ftp(ap)
add_webdav(ap)
add_smb(ap)
add_safety(ap, fk_salt)
add_optouts(ap)
add_shutdown(ap)
add_yolo(ap)
add_hooks(ap)
add_ui(ap, retry)
add_admin(ap)
add_logging(ap)
add_debug(ap)
ap2 = ap.add_argument_group("help sections")
sects = get_sects()
for k, h, _ in sects:
ap2.add_argument("--help-" + k, action="store_true", help=h)
try:
if not retry:
raise Exception()
for x in ap._actions:
if not x.help:
continue
a = ["ascii", "replace"]
x.help = x.help.encode(*a).decode(*a) + "\033[0m"
except:
pass
ret = ap.parse_args(args=argv[1:])
for k, h, t in sects:
k2 = "help_" + k.replace("-", "_")
@@ -781,7 +1053,7 @@ def main(argv: Optional[list[str]] = None) -> None:
S_VERSION,
CODENAME,
S_BUILD_DT,
py_desc().replace("[", "\033[1;30m["),
py_desc().replace("[", "\033[90m["),
SQLITE_VER,
JINJA_VER,
PYFTPD_VER,
@@ -795,12 +1067,21 @@ def main(argv: Optional[list[str]] = None) -> None:
showlic()
sys.exit(0)
if is_exe:
print("pybin: {}\n".format(pybin), end="")
ensure_locale()
if HAVE_SSL:
ensure_cert()
for k, v in zip(argv[1:], argv[2:]):
if k == "-c":
if k == "-c" and os.path.isfile(v):
supp = args_from_cfg(v)
argv.extend(supp)
for k in argv[1:]:
v = k[2:]
if k.startswith("-c") and v and os.path.isfile(v):
supp = args_from_cfg(v)
argv.extend(supp)
@@ -816,24 +1097,48 @@ def main(argv: Optional[list[str]] = None) -> None:
argv[idx] = nk
time.sleep(2)
da = len(argv) == 1
try:
if len(argv) == 1 and (ANYWIN or not os.geteuid()):
argv.extend(["-p80,443,3923", "--ign-ebind"])
if da:
argv.extend(["--qr"])
if ANYWIN or not os.geteuid():
# win10 allows symlinks if admin; can be unexpected
argv.extend(["-p80,443,3923", "--ign-ebind", "--no-dedup"])
except:
pass
if da:
t = "no arguments provided; will use {}\n"
lprint(t.format(" ".join(argv[1:])))
nc = 1024
try:
import resource
_, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
if hard > 0: # -1 == infinite
nc = min(nc, hard // 4)
except:
nc = 512
retry = False
for fmtr in [RiceFormatter, RiceFormatter, Dodge11874, BasicDodge11874]:
try:
al = run_argparse(argv, fmtr, retry)
al = run_argparse(argv, fmtr, retry, nc)
dal = run_argparse([], fmtr, retry, nc)
break
except SystemExit:
raise
except:
retry = True
lprint("\n[ {} ]:\n{}\n".format(fmtr, min_ex()))
assert al
al.E = E # __init__ is not shared when oxidized
try:
assert al # type: ignore
assert dal # type: ignore
al.E = E # __init__ is not shared when oxidized
except:
sys.exit(1)
if WINDOWS and not al.keep_qem:
try:
@@ -860,7 +1165,7 @@ def main(argv: Optional[list[str]] = None) -> None:
if re.match("c[^,]", opt):
mod = True
na.append("c," + opt[1:])
elif re.sub("^[rwmdg]*", "", opt) and "," not in opt:
elif re.sub("^[rwmdgG]*", "", opt) and "," not in opt:
mod = True
perm = opt[0]
if perm == "a":
@@ -885,6 +1190,11 @@ def main(argv: Optional[list[str]] = None) -> None:
if getattr(al, k1):
setattr(al, k2, True)
# propagate unplications
for k1, k2 in UNPLICATIONS:
if getattr(al, k1):
setattr(al, k2, False)
al.i = al.i.split(",")
try:
if "-" in al.p:
@@ -901,6 +1211,12 @@ def main(argv: Optional[list[str]] = None) -> None:
zs = "argument {} cannot be '{}'; try one of these: {}"
raise Exception(zs.format(arg, val, okays))
if not al.qrs and [k for k in argv if k.startswith("--qr")]:
al.qr = True
if al.ihead:
al.ihead = [x.lower() for x in al.ihead]
if HAVE_SSL:
if al.ssl_ver:
configure_ssl_ver(al)
@@ -916,12 +1232,16 @@ def main(argv: Optional[list[str]] = None) -> None:
+ " (if you crash with codec errors then that is why)"
)
if PY2 and al.smb:
print("error: python2 cannot --smb")
return
if sys.version_info < (3, 6):
al.no_scandir = True
# signal.signal(signal.SIGINT, sighandler)
SvcHub(al, argv, "".join(printed)).run()
SvcHub(al, dal, argv, "".join(printed)).run()
if __name__ == "__main__":

View File

@@ -1,8 +1,8 @@
# coding: utf-8
VERSION = (1, 4, 0)
CODENAME = "mostly reliable"
BUILD_DT = (2022, 9, 23)
VERSION = (1, 6, 5)
CODENAME = "cors k"
BUILD_DT = (2023, 2, 12)
S_VERSION = ".".join(map(str, VERSION))
S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

View File

@@ -14,13 +14,15 @@ from datetime import datetime
from .__init__ import ANYWIN, TYPE_CHECKING, WINDOWS
from .bos import bos
from .cfg import flagdescs, permdescs, vf_bmap, vf_cmap, vf_vmap
from .util import (
IMPLICATIONS,
META_NOBOTS,
SQLITE_VER,
UNPLICATIONS,
Pebkac,
absreal,
fsenc,
afsenc,
get_df,
humansize,
relchk,
@@ -30,15 +32,12 @@ from .util import (
unhumanize,
)
try:
if True: # pylint: disable=using-constant-test
from collections.abc import Iterable
import typing
from typing import Any, Generator, Optional, Union
from .util import RootLogger
except:
pass
from .util import NamedLogger, RootLogger
if TYPE_CHECKING:
pass
@@ -58,18 +57,20 @@ class AXS(object):
umove: Optional[Union[list[str], set[str]]] = None,
udel: Optional[Union[list[str], set[str]]] = None,
uget: Optional[Union[list[str], set[str]]] = None,
upget: Optional[Union[list[str], set[str]]] = None,
) -> None:
self.uread: set[str] = set(uread or [])
self.uwrite: set[str] = set(uwrite or [])
self.umove: set[str] = set(umove or [])
self.udel: set[str] = set(udel or [])
self.uget: set[str] = set(uget or [])
self.upget: set[str] = set(upget or [])
def __repr__(self) -> str:
return "AXS({})".format(
", ".join(
"{}={!r}".format(k, self.__dict__[k])
for k in "uread uwrite umove udel uget".split()
for k in "uread uwrite umove udel uget upget".split()
)
)
@@ -293,6 +294,7 @@ class VFS(object):
self.amove: dict[str, list[str]] = {}
self.adel: dict[str, list[str]] = {}
self.aget: dict[str, list[str]] = {}
self.apget: dict[str, list[str]] = {}
if realpath:
self.histpath = os.path.join(realpath, ".hist") # db / thumbcache
@@ -369,7 +371,6 @@ class VFS(object):
def _find(self, vpath: str) -> tuple["VFS", str]:
"""return [vfs,remainder]"""
vpath = undot(vpath)
if vpath == "":
return self, ""
@@ -380,13 +381,15 @@ class VFS(object):
rem = ""
if name in self.nodes:
return self.nodes[name]._find(rem)
return self.nodes[name]._find(undot(rem))
return self, vpath
def can_access(self, vpath: str, uname: str) -> tuple[bool, bool, bool, bool, bool]:
"""can Read,Write,Move,Delete,Get"""
vn, _ = self._find(vpath)
def can_access(
self, vpath: str, uname: str
) -> tuple[bool, bool, bool, bool, bool, bool]:
"""can Read,Write,Move,Delete,Get,Upget"""
vn, _ = self._find(undot(vpath))
c = vn.axs
return (
uname in c.uread or "*" in c.uread,
@@ -394,6 +397,7 @@ class VFS(object):
uname in c.umove or "*" in c.umove,
uname in c.udel or "*" in c.udel,
uname in c.uget or "*" in c.uget,
uname in c.upget or "*" in c.upget,
)
def get(
@@ -405,6 +409,7 @@ class VFS(object):
will_move: bool = False,
will_del: bool = False,
will_get: bool = False,
err: int = 403,
) -> tuple["VFS", str]:
"""returns [vfsnode,fs_remainder] if user has the requested permissions"""
if ANYWIN:
@@ -414,7 +419,7 @@ class VFS(object):
self.log("vfs", "invalid relpath [{}]".format(vpath))
raise Pebkac(404)
vn, rem = self._find(vpath)
vn, rem = self._find(undot(vpath))
c: AXS = vn.axs
for req, d, msg in [
@@ -426,7 +431,7 @@ class VFS(object):
]:
if req and (uname not in d and "*" not in d) and uname != LEELOO_DALLAS:
t = "you don't have {}-access for this location"
raise Pebkac(403, t.format(msg))
raise Pebkac(err, t.format(msg))
return vn, rem
@@ -441,11 +446,20 @@ class VFS(object):
def canonical(self, rem: str, resolve: bool = True) -> str:
"""returns the canonical path (fully-resolved absolute fs path)"""
rp = self.realpath
ap = self.realpath
if rem:
rp += "/" + rem
ap += "/" + rem
return absreal(rp) if resolve else rp
return absreal(ap) if resolve else ap
def dcanonical(self, rem: str) -> str:
"""resolves until the final component (filename)"""
ap = self.realpath
if rem:
ap += "/" + rem
ad, fn = os.path.split(ap)
return os.path.join(absreal(ad), fn)
def ls(
self,
@@ -562,14 +576,21 @@ class VFS(object):
yield x
def zipgen(
self, vrem: str, flt: set[str], uname: str, dots: bool, scandir: bool
self,
vrem: str,
flt: set[str],
uname: str,
dots: bool,
dirs: bool,
scandir: bool,
wrap: bool = True,
) -> Generator[dict[str, Any], None, None]:
# if multiselect: add all items to archive root
# if single folder: the folder itself is the top-level item
folder = "" if flt else (vrem.split("/")[-1] or "top")
folder = "" if flt or not wrap else (vrem.split("/")[-1].lstrip(".") or "top")
g = self.walk(folder, vrem, [], uname, [[True]], dots, scandir, False)
g = self.walk(folder, vrem, [], uname, [[True, False]], dots, scandir, False)
for _, _, vpath, apath, files, rd, vd in g:
if flt:
files = [x for x in files if x[0] in flt]
@@ -603,6 +624,21 @@ class VFS(object):
for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in ret]:
yield f
if not dirs:
continue
ts = int(time.time())
st = os.stat_result((16877, -1, -1, 1, 1000, 1000, 8, ts, ts, ts))
dnames = [n[0] for n in rd]
dstats = [n[1] for n in rd]
dnames += list(vd.keys())
dstats += [st] * len(vd)
vpaths = [vpath + "/" + n for n in dnames] if vpath else dnames
apaths = [os.path.join(apath, n) for n in dnames]
ret2 = list(zip(vpaths, apaths, dstats))
for d in [{"vp": v, "ap": a, "st": n} for v, a, n in ret2]:
yield d
if WINDOWS:
re_vol = re.compile(r"^([a-zA-Z]:[\\/][^:]*|[^:]*):([^:]*):(.*)$")
@@ -618,11 +654,15 @@ class AuthSrv(object):
args: argparse.Namespace,
log_func: Optional["RootLogger"],
warn_anonwrite: bool = True,
dargs: Optional[argparse.Namespace] = None,
) -> None:
self.args = args
self.dargs = dargs or args
self.log_func = log_func
self.warn_anonwrite = warn_anonwrite
self.line_ctr = 0
self.indent = ""
self.desc = []
self.mutex = threading.Lock()
self.reload()
@@ -655,80 +695,195 @@ class AuthSrv(object):
raise Exception("invalid config")
if src in mount.values():
t = "warning: filesystem-path [{}] mounted in multiple locations:"
t = "filesystem-path [{}] mounted in multiple locations:"
t = t.format(src)
for v in [k for k, v in mount.items() if v == src] + [dst]:
t += "\n /{}".format(v)
self.log(t, c=3)
raise Exception("invalid config")
if not bos.path.isdir(src):
self.log("warning: filesystem-path does not exist: {}".format(src), 3)
mount[dst] = src
daxs[dst] = AXS()
mflags[dst] = {}
def _e(self, desc: Optional[str] = None) -> None:
if not self.args.vc or not self.line_ctr:
return
if not desc and not self.indent:
self.log("")
return
desc = desc or ""
desc = desc.replace("[", "[\033[0m").replace("]", "\033[90m]")
self.log(" >>> {}{}".format(self.indent, desc), "90")
def _l(self, ln: str, c: int, desc: str) -> None:
if not self.args.vc or not self.line_ctr:
return
if c < 10:
c += 30
t = "\033[97m{:4} \033[{}m{}{}"
if desc:
t += " \033[0;90m# {}\033[0m"
desc = desc.replace("[", "[\033[0m").replace("]", "\033[90m]")
self.log(t.format(self.line_ctr, c, self.indent, ln, desc))
def _parse_config_file(
self,
fd: typing.BinaryIO,
fp: str,
cfg_lines: list[str],
acct: dict[str, str],
daxs: dict[str, AXS],
mflags: dict[str, dict[str, Any]],
mount: dict[str, str],
) -> None:
skip = False
vol_src = None
vol_dst = None
self.desc = []
self.line_ctr = 0
for ln in [x.decode("utf-8").strip() for x in fd]:
expand_config_file(cfg_lines, fp, "")
if self.args.vc:
lns = ["{:4}: {}".format(n, s) for n, s in enumerate(cfg_lines, 1)]
self.log("expanded config file (unprocessed):\n" + "\n".join(lns))
cfg_lines = upgrade_cfg_fmt(self.log, self.args, cfg_lines, fp)
cat = ""
catg = "[global]"
cata = "[accounts]"
catx = "accs:"
catf = "flags:"
ap: Optional[str] = None
vp: Optional[str] = None
for ln in cfg_lines:
self.line_ctr += 1
if not ln and vol_src is not None:
vol_src = None
vol_dst = None
if skip:
if not ln:
skip = False
ln = ln.split(" #")[0].strip()
if not ln.split("#")[0].strip():
continue
if not ln or ln.startswith("#"):
continue
subsection = ln in (catx, catf)
if ln.startswith("[") or subsection:
self._e()
if ap is None and vp is not None:
t = "the first line after [/{}] must be a filesystem path to share on that volume"
raise Exception(t.format(vp))
if vol_src is None:
if ln.startswith("u "):
u, p = ln[2:].split(":", 1)
acct[u] = p
elif ln.startswith("-"):
skip = True # argv
cat = ln
if not subsection:
ap = vp = None
self.indent = ""
else:
vol_src = ln
self.indent = " "
if ln == catg:
t = "begin commandline-arguments (anything from --help; dashes are optional)"
self._l(ln, 6, t)
elif ln == cata:
self._l(ln, 5, "begin user-accounts section")
elif ln.startswith("[/"):
vp = ln[1:-1].strip("/")
self._l(ln, 2, "define volume at URL [/{}]".format(vp))
elif subsection:
if ln == catx:
self._l(ln, 5, "volume access config:")
else:
t = "volume-specific config (anything from --help-flags)"
self._l(ln, 6, t)
else:
raise Exception("invalid section header")
self.indent = " " if subsection else " "
continue
if vol_src and vol_dst is None:
vol_dst = ln
if not vol_dst.startswith("/"):
raise Exception('invalid mountpoint "{}"'.format(vol_dst))
# cfg files override arguments and previous files
vol_src = absreal(vol_src)
vol_dst = vol_dst.strip("/")
self._map_volume(vol_src, vol_dst, mount, daxs, mflags)
if cat == catg:
self._l(ln, 6, "")
zt = split_cfg_ln(ln)
for zs, za in zt.items():
zs = zs.lstrip("-")
if za is True:
self._e("└─argument [{}]".format(zs))
else:
self._e("└─argument [{}] with value [{}]".format(zs, za))
continue
try:
lvl, uname = ln.split(" ", 1)
except:
lvl = ln
uname = "*"
if cat == cata:
try:
u, p = [zs.strip() for zs in ln.split(":", 1)]
self._l(ln, 5, "account [{}], password [{}]".format(u, p))
acct[u] = p
except:
t = 'lines inside the [accounts] section must be "username: password"'
raise Exception(t)
continue
if lvl == "a":
t = "WARNING (config-file): permission flag 'a' is deprecated; please use 'rw' instead"
self.log(t, 1)
if vp is not None and ap is None:
ap = ln
if ap.startswith("~"):
ap = os.path.expanduser(ap)
self._read_vol_str(lvl, uname, daxs[vol_dst], mflags[vol_dst])
ap = absreal(ap)
self._l(ln, 2, "bound to filesystem-path [{}]".format(ap))
self._map_volume(ap, vp, mount, daxs, mflags)
continue
if cat == catx:
err = ""
try:
self._l(ln, 5, "volume access config:")
sk, sv = ln.split(":")
if re.sub("[rwmdgG]", "", sk) or not sk:
err = "invalid accs permissions list; "
raise Exception(err)
if " " in re.sub(", *", "", sv).strip():
err = "list of users is not comma-separated; "
raise Exception(err)
self._read_vol_str(sk, sv.replace(" ", ""), daxs[vp], mflags[vp])
continue
except:
err += "accs entries must be 'rwmdgG: user1, user2, ...'"
raise Exception(err)
if cat == catf:
err = ""
try:
self._l(ln, 6, "volume-specific config:")
zd = split_cfg_ln(ln)
fstr = ""
for sk, sv in zd.items():
bad = re.sub(r"[a-z0-9_]", "", sk)
if bad:
err = "bad characters [{}] in volflag name [{}]; "
err = err.format(bad, sk)
raise Exception(err)
if sv is True:
fstr += "," + sk
else:
fstr += ",{}={}".format(sk, sv)
self._read_vol_str("c", fstr[1:], daxs[vp], mflags[vp])
fstr = ""
if fstr:
self._read_vol_str("c", fstr[1:], daxs[vp], mflags[vp])
continue
except:
err += "flags entries (volflags) must be one of the following:\n 'flag1, flag2, ...'\n 'key: value'\n 'flag1, flag2, key: value'"
raise Exception(err)
raise Exception("unprocessable line in config")
self._e()
self.line_ctr = 0
def _read_vol_str(
self, lvl: str, uname: str, axs: AXS, flags: dict[str, Any]
) -> None:
if lvl.strip("crwmdg"):
if lvl.strip("crwmdgG"):
raise Exception("invalid volflag: {},{}".format(lvl, uname))
if lvl == "c":
@@ -758,8 +913,17 @@ class AuthSrv(object):
("m", axs.umove),
("d", axs.udel),
("g", axs.uget),
]:
("G", axs.uget),
("G", axs.upget),
]: # b bb bbb
if ch in lvl:
if un == "*":
t = "└─add permission [{0}] for [everyone] -- {2}"
else:
t = "└─add permission [{0}] for user [{1}] -- {2}"
desc = permdescs.get(ch, "?")
self._e(t.format(ch, un, desc))
al.add(un)
def _read_volflag(
@@ -769,7 +933,13 @@ class AuthSrv(object):
value: Union[str, bool, list[str]],
is_list: bool,
) -> None:
if name not in ["mtp"]:
desc = flagdescs.get(name, "?").replace("\n", " ")
if name not in ["mtp", "xbu", "xau", "xbr", "xar", "xbd", "xad", "xm"]:
if value is True:
t = "└─add volflag [{}] = {} ({})"
else:
t = "└─add volflag [{}] = [{}] ({})"
self._e(t.format(name, value, desc))
flags[name] = value
return
@@ -782,6 +952,7 @@ class AuthSrv(object):
vals += [value]
flags[name] = vals
self._e("volflag [{}] += {} ({})".format(name, vals, desc))
def reload(self) -> None:
"""
@@ -808,7 +979,7 @@ class AuthSrv(object):
if self.args.v:
# list of src:dst:permset:permset:...
# permset is <rwmdg>[,username][,username] or <c>,<flag>[=args]
# permset is <rwmdgG>[,username][,username] or <c>,<flag>[=args]
for v_str in self.args.v:
m = re_vol.match(v_str)
if not m:
@@ -829,13 +1000,28 @@ class AuthSrv(object):
if self.args.c:
for cfg_fn in self.args.c:
with open(cfg_fn, "rb") as f:
try:
self._parse_config_file(f, acct, daxs, mflags, mount)
except:
t = "\n\033[1;31m\nerror in config file {} on line {}:\n\033[0m"
self.log(t.format(cfg_fn, self.line_ctr), 1)
raise
lns: list[str] = []
try:
self._parse_config_file(cfg_fn, lns, acct, daxs, mflags, mount)
zs = "#\033[36m cfg files in "
zst = [x[len(zs) :] for x in lns if x.startswith(zs)]
for zs in list(set(zst)):
self.log("discovered config files in " + zs, 6)
zs = "#\033[36m opening cfg file"
zstt = [x.split(" -> ") for x in lns if x.startswith(zs)]
zst = [(max(0, len(x) - 2) * " ") + "" + x[-1] for x in zstt]
t = "loaded {} config files:\n{}"
self.log(t.format(len(zst), "\n".join(zst)))
except:
lns = lns[: self.line_ctr]
slns = ["{:4}: {}".format(n, s) for n, s in enumerate(lns, 1)]
t = "\033[1;31m\nerror @ line {}, included from {}\033[0m"
t = t.format(self.line_ctr, cfg_fn)
self.log("\n{0}\n{1}{0}".format(t, "\n".join(slns)))
raise
# case-insensitive; normalize
if WINDOWS:
@@ -870,10 +1056,11 @@ class AuthSrv(object):
zv.flags = mflags[dst]
zv.dbv = None
assert vfs
vfs.all_vols = {}
vfs.get_all_vols(vfs.all_vols)
for perm in "read write move del get".split():
for perm in "read write move del get pget".split():
axs_key = "u" + perm
unames = ["*"] + list(acct.keys())
umap: dict[str, list[str]] = {x: [] for x in unames}
@@ -888,7 +1075,7 @@ class AuthSrv(object):
all_users = {}
missing_users = {}
for axs in daxs.values():
for d in [axs.uread, axs.uwrite, axs.umove, axs.udel, axs.uget]:
for d in [axs.uread, axs.uwrite, axs.umove, axs.udel, axs.uget, axs.upget]:
for usr in d:
all_users[usr] = 1
if usr != "*" and usr not in acct:
@@ -908,7 +1095,7 @@ class AuthSrv(object):
promote = []
demote = []
for vol in vfs.all_vols.values():
zb = hashlib.sha512(fsenc(vol.realpath)).digest()
zb = hashlib.sha512(afsenc(vol.realpath)).digest()
hid = base64.b32encode(zb).decode("ascii").lower()
vflag = vol.flags.get("hist")
if vflag == "-":
@@ -927,7 +1114,7 @@ class AuthSrv(object):
except:
owner = None
me = fsenc(vol.realpath).rstrip()
me = afsenc(vol.realpath).rstrip()
if owner not in [None, me]:
continue
@@ -1067,18 +1254,44 @@ class AuthSrv(object):
if ptn:
vol.flags[vf] = re.compile(ptn)
for k in ["e2t", "e2ts", "e2tsr", "e2v", "e2vu", "e2vp", "xdev", "xvol"]:
if getattr(self.args, k):
vol.flags[k] = True
for ga, vf in [["no_forget", "noforget"], ["magic", "magic"]]:
for ga, vf in vf_bmap().items():
if getattr(self.args, ga):
vol.flags[vf] = True
for ve, vd in (
("nodotsrch", "dotsrch"),
("sb_lg", "no_sb_lg"),
("sb_md", "no_sb_md"),
):
if ve in vol.flags:
vol.flags.pop(vd, None)
for ga, vf in vf_vmap().items():
if vf not in vol.flags:
vol.flags[vf] = getattr(self.args, ga)
for k in ("nrand",):
if k not in vol.flags:
vol.flags[k] = getattr(self.args, k)
for k in ("nrand",):
if k in vol.flags:
vol.flags[k] = int(vol.flags[k])
for k1, k2 in IMPLICATIONS:
if k1 in vol.flags:
vol.flags[k2] = True
for k1, k2 in UNPLICATIONS:
if k1 in vol.flags:
vol.flags[k2] = False
dbds = "acid|swal|wal|yolo"
vol.flags["dbd"] = dbd = vol.flags.get("dbd") or self.args.dbd
if dbd not in dbds.split("|"):
t = "invalid dbd [{}]; must be one of [{}]"
raise Exception(t.format(dbd, dbds))
# default tag cfgs if unset
if "mte" not in vol.flags:
vol.flags["mte"] = self.args.mte
@@ -1089,8 +1302,32 @@ class AuthSrv(object):
if "mth" not in vol.flags:
vol.flags["mth"] = self.args.mth
# append parsers from argv to volflags
self._read_volflag(vol.flags, "mtp", self.args.mtp, True)
# append additive args from argv to volflags
hooks = "xbu xau xbr xar xbd xad xm".split()
for name in ["mtp"] + hooks:
self._read_volflag(vol.flags, name, getattr(self.args, name), True)
for hn in hooks:
cmds = vol.flags.get(hn)
if not cmds:
continue
ncmds = []
for cmd in cmds:
hfs = []
ocmd = cmd
while "," in cmd[:6]:
zs, cmd = cmd.split(",", 1)
hfs.append(zs)
if "c" in hfs and "f" in hfs:
t = "cannot combine flags c and f; removing f from eventhook [{}]"
self.log(t.format(ocmd), 1)
hfs = [x for x in hfs if x != "f"]
ocmd = ",".join(hfs + [cmd])
ncmds.append(ocmd)
vol.flags[hn] = ncmds
# d2d drops all database features for a volume
for grp, rm in [["d2d", "e2d"], ["d2t", "e2t"], ["d2d", "e2v"]]:
@@ -1131,6 +1368,9 @@ class AuthSrv(object):
self.log(t.format(vol.vpath), 1)
del vol.flags["lifetime"]
if vol.flags.get("neversymlink") and not vol.flags.get("hardlink"):
vol.flags["copydupes"] = True
# verify tags mentioned by -mt[mp] are used by -mte
local_mtp = {}
local_only_mtp = {}
@@ -1175,6 +1415,18 @@ class AuthSrv(object):
self.log(t.format(mtp), 1)
errors = True
have_daw = False
for vol in vfs.all_vols.values():
daw = vol.flags.get("daw") or self.args.daw
if daw:
vol.flags["daw"] = True
have_daw = True
if have_daw and self.args.no_dav:
t = 'volume "/{}" has volflag "daw" (webdav write-access), but --no-dav is set'
self.log(t, 1)
errors = True
if errors:
sys.exit(1)
@@ -1193,6 +1445,7 @@ class AuthSrv(object):
[" move", "umove"],
["delete", "udel"],
[" get", "uget"],
[" upget", "upget"],
]:
u = list(sorted(getattr(zv.axs, attr)))
u = ", ".join("\033[35meverybody\033[0m" if x == "*" else x for x in u)
@@ -1288,10 +1541,11 @@ class AuthSrv(object):
raise Exception("volume not found: " + zs)
self.log(str({"users": users, "vols": vols, "flags": flags}))
t = "/{}: read({}) write({}) move({}) del({}) get({})"
t = "/{}: read({}) write({}) move({}) del({}) get({}) upget({})"
for k, zv in self.vfs.all_vols.items():
vc = zv.axs
self.log(t.format(k, vc.uread, vc.uwrite, vc.umove, vc.udel, vc.uget))
vs = [k, vc.uread, vc.uwrite, vc.umove, vc.udel, vc.uget, vc.upget]
self.log(t.format(*vs))
flag_v = "v" in flags
flag_ln = "ln" in flags
@@ -1316,7 +1570,7 @@ class AuthSrv(object):
"",
[],
u,
[[True]],
[[True, False]],
True,
not self.args.no_scandir,
False,
@@ -1360,3 +1614,294 @@ class AuthSrv(object):
if not flag_r:
sys.exit(0)
def cgen(self) -> None:
ret = [
"## WARNING:",
"## there will probably be mistakes in",
"## commandline-args (and maybe volflags)",
"",
]
csv = set("i p".split())
lst = set("c ihead mtm mtp xad xar xau xbd xbr xbu xm".split())
askip = set("a v c vc cgen".split())
# keymap from argv to vflag
amap = vf_bmap()
amap.update(vf_vmap())
amap.update(vf_cmap())
vmap = {v: k for k, v in amap.items()}
args = {k: v for k, v in vars(self.args).items()}
pops = []
for k1, k2 in IMPLICATIONS:
if args.get(k1):
pops.append(k2)
for pop in pops:
args.pop(pop, None)
if args:
ret.append("[global]")
for k, v in args.items():
if k in askip:
continue
if k in csv:
v = ", ".join([str(za) for za in v])
try:
v2 = getattr(self.dargs, k)
if v == v2:
continue
except:
continue
dk = " " + k.replace("_", "-")
if k in lst:
for ve in v:
ret.append("{}: {}".format(dk, ve))
else:
if v is True:
ret.append(dk)
elif v not in (False, None, ""):
ret.append("{}: {}".format(dk, v))
ret.append("")
if self.acct:
ret.append("[accounts]")
for u, p in self.acct.items():
ret.append(" {}: {}".format(u, p))
ret.append("")
for vol in self.vfs.all_vols.values():
ret.append("[/{}]".format(vol.vpath))
ret.append(" " + vol.realpath)
ret.append(" accs:")
perms = {
"r": "uread",
"w": "uwrite",
"m": "umove",
"d": "udel",
"g": "uget",
"G": "upget",
}
users = {}
for pkey in perms.values():
for uname in getattr(vol.axs, pkey):
try:
users[uname] += 1
except:
users[uname] = 1
lusers = [(v, k) for k, v in users.items()]
vperms = {}
for _, uname in sorted(lusers):
pstr = ""
for pchar, pkey in perms.items():
if uname in getattr(vol.axs, pkey):
pstr += pchar
if "g" in pstr and "G" in pstr:
pstr = pstr.replace("g", "")
try:
vperms[pstr].append(uname)
except:
vperms[pstr] = [uname]
for pstr, uname in vperms.items():
ret.append(" {}: {}".format(pstr, ", ".join(uname)))
trues = []
vals = []
for k, v in sorted(vol.flags.items()):
try:
ak = vmap[k]
if getattr(self.args, ak) is v:
continue
except:
pass
if k in lst:
for ve in v:
vals.append("{}: {}".format(k, ve))
elif v is True:
trues.append(k)
elif v is not False:
vals.append("{}: {}".format(k, v))
pops = []
for k1, k2 in IMPLICATIONS:
if k1 in trues:
pops.append(k2)
trues = [x for x in trues if x not in pops]
if trues:
vals.append(", ".join(trues))
if vals:
ret.append(" flags:")
for zs in vals:
ret.append(" " + zs)
ret.append("")
self.log("generated config:\n\n" + "\n".join(ret))
def split_cfg_ln(ln: str) -> dict[str, Any]:
# "a, b, c: 3" => {a:true, b:true, c:3}
ret = {}
while True:
ln = ln.strip()
if not ln:
break
ofs_sep = ln.find(",") + 1
ofs_var = ln.find(":") + 1
if not ofs_sep and not ofs_var:
ret[ln] = True
break
if ofs_sep and (ofs_sep < ofs_var or not ofs_var):
k, ln = ln.split(",", 1)
ret[k.strip()] = True
else:
k, ln = ln.split(":", 1)
ret[k.strip()] = ln.strip()
break
return ret
def expand_config_file(ret: list[str], fp: str, ipath: str) -> None:
"""expand all % file includes"""
fp = absreal(fp)
if len(ipath.split(" -> ")) > 64:
raise Exception("hit max depth of 64 includes")
if os.path.isdir(fp):
names = os.listdir(fp)
ret.append("#\033[36m cfg files in {} => {}\033[0m".format(fp, names))
for fn in sorted(names):
fp2 = os.path.join(fp, fn)
if not fp2.endswith(".conf") or fp2 in ipath:
continue
expand_config_file(ret, fp2, ipath)
return
ipath += " -> " + fp
ret.append("#\033[36m opening cfg file{}\033[0m".format(ipath))
with open(fp, "rb") as f:
for oln in [x.decode("utf-8").rstrip() for x in f]:
ln = oln.split(" #")[0].strip()
if ln.startswith("% "):
pad = " " * len(oln.split("%")[0])
fp2 = ln[1:].strip()
fp2 = os.path.join(os.path.dirname(fp), fp2)
ofs = len(ret)
expand_config_file(ret, fp2, ipath)
for n in range(ofs, len(ret)):
ret[n] = pad + ret[n]
continue
ret.append(oln)
ret.append("#\033[36m closed{}\033[0m".format(ipath))
def upgrade_cfg_fmt(
log: Optional["NamedLogger"], args: argparse.Namespace, orig: list[str], cfg_fp: str
) -> list[str]:
"""convert from v1 to v2 format"""
zst = [x.split("#")[0].strip() for x in orig]
zst = [x for x in zst if x]
if (
"[global]" in zst
or "[accounts]" in zst
or "accs:" in zst
or "flags:" in zst
or [x for x in zst if x.startswith("[/")]
or len(zst) == len([x for x in zst if x.startswith("%")])
):
return orig
zst = [x for x in orig if "#\033[36m opening cfg file" not in x]
incl = len(zst) != len(orig) - 1
t = "upgrading config file [{}] from v1 to v2"
if not args.vc:
t += ". Run with argument '--vc' to see the converted config if you want to upgrade"
if incl:
t += ". Please don't include v1 configs from v2 files or vice versa! Upgrade all of them at the same time."
if log:
log(t.format(cfg_fp), 3)
ret = []
vp = ""
ap = ""
cat = ""
catg = "[global]"
cata = "[accounts]"
catx = " accs:"
catf = " flags:"
for ln in orig:
sn = ln.strip()
if not sn:
cat = vp = ap = ""
if not sn.split("#")[0]:
ret.append(ln)
elif sn.startswith("-") and cat in ("", catg):
if cat != catg:
cat = catg
ret.append(cat)
sn = sn.lstrip("-")
zst = sn.split(" ", 1)
if len(zst) > 1:
sn = "{}: {}".format(zst[0], zst[1].strip())
ret.append(" " + sn)
elif sn.startswith("u ") and cat in ("", catg, cata):
if cat != cata:
cat = cata
ret.append(cat)
s1, s2 = sn[1:].split(":", 1)
ret.append(" {}: {}".format(s1.strip(), s2.strip()))
elif not ap:
ap = sn
elif not vp:
vp = "/" + sn.strip("/")
cat = "[{}]".format(vp)
ret.append(cat)
ret.append(" " + ap)
elif sn.startswith("c "):
if cat != catf:
cat = catf
ret.append(cat)
sn = sn[1:].strip()
if "=" in sn:
zst = sn.split("=", 1)
sn = zst[0].replace(",", ", ")
sn += ": " + zst[1]
else:
sn = sn.replace(",", ", ")
ret.append(" " + sn)
elif sn[:1] in "rwmdgG":
if cat != catx:
cat = catx
ret.append(cat)
zst = sn.split(" ")
zst = [x for x in zst if x]
if len(zst) == 1:
zst.append("*")
ret.append(" {}: {}".format(zst[0], ", ".join(zst[1:])))
else:
t = "did not understand line {} in the config"
t1 = t
n = 0
for ln in orig:
n += 1
t += "\n{:4} {}".format(n, ln)
if log:
log(t, 1)
else:
print("\033[31m" + t)
raise Exception(t1)
if args.vc and log:
t = "new config syntax (copy/paste this to upgrade your config):\n"
t += "\n# ======================[ begin upgraded config ]======================\n\n"
for ln in ret:
t += ln + "\n"
t += "\n# ======================[ end of upgraded config ]======================\n"
log(t)
return ret

View File

@@ -4,14 +4,13 @@ from __future__ import print_function, unicode_literals
import os
from ..util import SYMTIME, fsdec, fsenc
from . import path
from . import path as path
try:
from typing import Optional
except:
pass
if True: # pylint: disable=using-constant-test
from typing import Any, Optional
_ = (path,)
__all__ = ["path"]
# grep -hRiE '(^|[^a-zA-Z_\.-])os\.' . | gsed -r 's/ /\n/g;s/\(/(\n/g' | grep -hRiE '(^|[^a-zA-Z_\.-])os\.' | sort | uniq -c
# printf 'os\.(%s)' "$(grep ^def bos/__init__.py | gsed -r 's/^def //;s/\(.*//' | tr '\n' '|' | gsed -r 's/.$//')"
@@ -25,19 +24,25 @@ def listdir(p: str = ".") -> list[str]:
return [fsdec(x) for x in os.listdir(fsenc(p))]
def makedirs(name: str, mode: int = 0o755, exist_ok: bool = True) -> None:
def makedirs(name: str, mode: int = 0o755, exist_ok: bool = True) -> bool:
bname = fsenc(name)
try:
os.makedirs(bname, mode)
return True
except:
if not exist_ok or not os.path.isdir(bname):
raise
return False
def mkdir(p: str, mode: int = 0o755) -> None:
return os.mkdir(fsenc(p), mode)
def open(p: str, *a, **ka) -> int:
return os.open(fsenc(p), *a, **ka)
def rename(src: str, dst: str) -> None:
return os.rename(fsenc(src), fsenc(dst))

View File

@@ -3,21 +3,20 @@ from __future__ import print_function, unicode_literals
import threading
import time
import traceback
import queue
from .__init__ import CORES, TYPE_CHECKING
from .broker_mpw import MpWorker
from .broker_util import try_exec
from .util import mp
from .util import Daemon, mp
if TYPE_CHECKING:
from .svchub import SvcHub
try:
if True: # pylint: disable=using-constant-test
from typing import Any
except:
pass
class MProcess(mp.Process):
@@ -51,13 +50,7 @@ class BrokerMp(object):
q_yield: queue.Queue[tuple[int, str, list[Any]]] = mp.Queue(64)
proc = MProcess(q_pend, q_yield, MpWorker, (q_pend, q_yield, self.args, n))
thr = threading.Thread(
target=self.collector, args=(proc,), name="mp-sink-{}".format(n)
)
thr.daemon = True
thr.start()
Daemon(self.collector, "mp-sink-{}".format(n), (proc,))
self.procs.append(proc)
proc.start()
@@ -101,12 +94,15 @@ class BrokerMp(object):
else:
# new ipc invoking managed service in hub
obj = self.hub
for node in dest.split("."):
obj = getattr(obj, node)
try:
obj = self.hub
for node in dest.split("."):
obj = getattr(obj, node)
# TODO will deadlock if dest performs another ipc
rv = try_exec(retq_id, obj, *args)
# TODO will deadlock if dest performs another ipc
rv = try_exec(retq_id, obj, *args)
except:
rv = ["exception", "stack", traceback.format_exc()]
if retq_id:
proc.q_pend.put((retq_id, "retq", rv))
@@ -121,6 +117,10 @@ class BrokerMp(object):
for p in self.procs:
p.q_pend.put((0, dest, [args[0], len(self.procs)]))
elif dest == "set_netdevs":
for p in self.procs:
p.q_pend.put((0, dest, list(args)))
elif dest == "cb_httpsrv_up":
self.hub.cb_httpsrv_up()

View File

@@ -9,17 +9,16 @@ import threading
import queue
from .__init__ import ANYWIN
from .authsrv import AuthSrv
from .broker_util import BrokerCli, ExceptionalQueue
from .httpsrv import HttpSrv
from .util import FAKE_MP, HMaccas
from .util import FAKE_MP, Daemon, HMaccas
try:
if True: # pylint: disable=using-constant-test
from types import FrameType
from typing import Any, Optional, Union
except:
pass
class MpWorker(BrokerCli):
@@ -48,7 +47,11 @@ class MpWorker(BrokerCli):
# we inherited signal_handler from parent,
# replace it with something harmless
if not FAKE_MP:
for sig in [signal.SIGINT, signal.SIGTERM, signal.SIGUSR1]:
sigs = [signal.SIGINT, signal.SIGTERM]
if not ANYWIN:
sigs.append(signal.SIGUSR1)
for sig in sigs:
signal.signal(sig, self.signal_handler)
# starting to look like a good idea
@@ -60,10 +63,7 @@ class MpWorker(BrokerCli):
# on winxp and some other platforms,
# use thr.join() to block all signals
thr = threading.Thread(target=self.main, name="mpw-main")
thr.daemon = True
thr.start()
thr.join()
Daemon(self.main, "mpw-main").join()
def signal_handler(self, sig: Optional[int], frame: Optional[FrameType]) -> None:
# print('k')
@@ -97,6 +97,9 @@ class MpWorker(BrokerCli):
elif dest == "listen":
self.httpsrv.listen(args[0], args[1])
elif dest == "set_netdevs":
self.httpsrv.set_netdevs(args[0])
elif dest == "retq":
# response from previous ipc call
with self.retpend_mutex:

View File

@@ -12,10 +12,8 @@ from .util import HMaccas
if TYPE_CHECKING:
from .svchub import SvcHub
try:
if True: # pylint: disable=using-constant-test
from typing import Any
except:
pass
class BrokerThr(BrokerCli):
@@ -63,6 +61,10 @@ class BrokerThr(BrokerCli):
self.httpsrv.listen(args[0], 1)
return
if dest == "set_netdevs":
self.httpsrv.set_netdevs(args[0])
return
# new ipc invoking managed service in hub
obj = self.hub
for node in dest.split("."):

View File

@@ -10,12 +10,10 @@ from .__init__ import TYPE_CHECKING
from .authsrv import AuthSrv
from .util import HMaccas, Pebkac
try:
if True: # pylint: disable=using-constant-test
from typing import Any, Optional, Union
from .util import RootLogger
except:
pass
if TYPE_CHECKING:
from .httpsrv import HttpSrv
@@ -41,12 +39,14 @@ class BrokerCli(object):
for example resolving httpconn.* in httpcli -- see lines tagged #mypy404
"""
log: "RootLogger"
args: argparse.Namespace
asrv: AuthSrv
httpsrv: "HttpSrv"
iphash: HMaccas
def __init__(self) -> None:
self.log: "RootLogger" = None
self.args: argparse.Namespace = None
self.asrv: AuthSrv = None
self.httpsrv: "HttpSrv" = None
self.iphash: HMaccas = None
pass
def ask(self, dest: str, *args: Any) -> ExceptionalQueue:
return ExceptionalQueue(1)

149
copyparty/cfg.py Normal file
View File

@@ -0,0 +1,149 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
# awk -F\" '/add_argument\("-[^-]/{print(substr($2,2))}' copyparty/__main__.py | sort | tr '\n' ' '
zs = "a c e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vp e2vu ed emp i j lo mcr mte mth mtm mtp nb nc nid nih nw p q s ss sss v z zv"
onedash = set(zs.split())
def vf_bmap() -> dict[str, str]:
"""argv-to-volflag: simple bools"""
ret = {
"never_symlink": "neversymlink",
"no_dedup": "copydupes",
"no_dupe": "nodupe",
"no_forget": "noforget",
}
for k in (
"dotsrch",
"e2t",
"e2ts",
"e2tsr",
"e2v",
"e2vu",
"e2vp",
"hardlink",
"magic",
"no_sb_md",
"no_sb_lg",
"rand",
"xdev",
"xlink",
"xvol",
):
ret[k] = k
return ret
def vf_vmap() -> dict[str, str]:
"""argv-to-volflag: simple values"""
ret = {}
for k in ("lg_sbf", "md_sbf"):
ret[k] = k
return ret
def vf_cmap() -> dict[str, str]:
"""argv-to-volflag: complex/lists"""
ret = {}
for k in ("dbd", "html_head", "mte", "mth", "nrand"):
ret[k] = k
return ret
permdescs = {
"r": "read; list folder contents, download files",
"w": 'write; upload files; need "r" to see the uploads',
"m": 'move; move files and folders; need "w" at destination',
"d": "delete; permanently delete files and folders",
"g": "get; download files, but cannot see folder contents",
"G": 'upget; same as "g" but can see filekeys of their own uploads',
}
flagcats = {
"uploads, general": {
"nodupe": "rejects existing files (instead of symlinking them)",
"hardlink": "does dedup with hardlinks instead of symlinks",
"neversymlink": "disables symlink fallback; full copy instead",
"copydupes": "disables dedup, always saves full copies of dupes",
"daw": "enable full WebDAV write support (dangerous);\nPUT-operations will now \033[1;31mOVERWRITE\033[0;35m existing files",
"nosub": "forces all uploads into the top folder of the vfs",
"magic": "enables filetype detection for nameless uploads",
"gz": "allows server-side gzip of uploads with ?gz (also c,xz)",
"pk": "forces server-side compression, optional arg: xz,9",
},
"upload rules": {
"maxn=250,600": "max 250 uploads over 15min",
"maxb=1g,300": "max 1 GiB over 5min (suffixes: b, k, m, g)",
"rand": "force randomized filenames, 9 chars long by default",
"nrand=N": "randomized filenames are N chars long",
"sz=1k-3m": "allow filesizes between 1 KiB and 3MiB",
"df=1g": "ensure 1 GiB free disk space",
},
"upload rotation\n(moves all uploads into the specified folder structure)": {
"rotn=100,3": "3 levels of subfolders with 100 entries in each",
"rotf=%Y-%m/%d-%H": "date-formatted organizing",
"lifetime=3600": "uploads are deleted after 1 hour",
},
"database, general": {
"e2d": "enable database; makes files searchable + enables upload dedup",
"e2ds": "scan writable folders for new files on startup; also sets -e2d",
"e2dsa": "scans all folders for new files on startup; also sets -e2d",
"e2t": "enable multimedia indexing; makes it possible to search for tags",
"e2ts": "scan existing files for tags on startup; also sets -e2t",
"e2tsa": "delete all metadata from DB (full rescan); also sets -e2ts",
"d2ts": "disables metadata collection for existing files",
"d2ds": "disables onboot indexing, overrides -e2ds*",
"d2t": "disables metadata collection, overrides -e2t*",
"d2v": "disables file verification, overrides -e2v*",
"d2d": "disables all database stuff, overrides -e2*",
"hist=/tmp/cdb": "puts thumbnails and indexes at that location",
"scan=60": "scan for new files every 60sec, same as --re-maxage",
"nohash=\\.iso$": "skips hashing file contents if path matches *.iso",
"noidx=\\.iso$": "fully ignores the contents at paths matching *.iso",
"noforget": "don't forget files when deleted from disk",
"dbd=[acid|swal|wal|yolo]": "database speed-durability tradeoff",
"xlink": "cross-volume dupe detection / linking",
"xdev": "do not descend into other filesystems",
"xvol": "skip symlinks leaving the volume root",
"dotsrch": "show dotfiles in search results",
"nodotsrch": "hide dotfiles in search results (default)",
},
'database, audio tags\n"mte", "mth", "mtp", "mtm" all work the same as -mte, -mth, ...': {
"mtp=.bpm=f,audio-bpm.py": 'uses the "audio-bpm.py" program to\ngenerate ".bpm" tags from uploads (f = overwrite tags)',
"mtp=ahash,vhash=media-hash.py": "collects two tags at once",
},
"thumbnails": {
"dthumb": "disables all thumbnails",
"dvthumb": "disables video thumbnails",
"dathumb": "disables audio thumbnails (spectrograms)",
"dithumb": "disables image thumbnails",
},
"event hooks\n(better explained in --help-hooks)": {
"xbu=CMD": "execute CMD before a file upload starts",
"xau=CMD": "execute CMD after a file upload finishes",
"xbr=CMD": "execute CMD before a file rename/move",
"xar=CMD": "execute CMD after a file rename/move",
"xbd=CMD": "execute CMD before a file delete",
"xad=CMD": "execute CMD after a file delete",
"xm=CMD": "execute CMD on message",
},
"client and ux": {
"html_head=TXT": "includes TXT in the <head>",
"robots": "allows indexing by search engines (default)",
"norobots": "kindly asks search engines to leave",
"no_sb_md": "disable js sandbox for markdown files",
"no_sb_lg": "disable js sandbox for prologue/epilogue",
"sb_md": "enable js sandbox for markdown files (default)",
"sb_lg": "enable js sandbox for prologue/epilogue (default)",
"md_sbf": "list of markdown-sandbox safeguards to disable",
"lg_sbf": "list of *logue-sandbox safeguards to disable",
},
"others": {
"fk=8": 'generates per-file accesskeys,\nwhich will then be required at the "g" permission'
},
}
flagdescs = {k.split("=")[0]: v for tab in flagcats.values() for k, v in tab.items()}

72
copyparty/dxml.py Normal file
View File

@@ -0,0 +1,72 @@
import importlib
import sys
import xml.etree.ElementTree as ET
from .__init__ import PY2
if True: # pylint: disable=using-constant-test
from typing import Any, Optional
def get_ET() -> ET.XMLParser:
pn = "xml.etree.ElementTree"
cn = "_elementtree"
cmod = sys.modules.pop(cn, None)
if not cmod:
return ET.XMLParser # type: ignore
pmod = sys.modules.pop(pn)
sys.modules[cn] = None # type: ignore
ret = importlib.import_module(pn)
for name, mod in ((pn, pmod), (cn, cmod)):
if mod:
sys.modules[name] = mod
else:
sys.modules.pop(name, None)
sys.modules["xml.etree"].ElementTree = pmod # type: ignore
ret.ParseError = ET.ParseError # type: ignore
return ret.XMLParser # type: ignore
XMLParser: ET.XMLParser = get_ET()
class DXMLParser(XMLParser): # type: ignore
def __init__(self) -> None:
tb = ET.TreeBuilder()
super(DXMLParser, self).__init__(target=tb)
p = self._parser if PY2 else self.parser
p.StartDoctypeDeclHandler = self.nope
p.EntityDeclHandler = self.nope
p.UnparsedEntityDeclHandler = self.nope
p.ExternalEntityRefHandler = self.nope
def nope(self, *a: Any, **ka: Any) -> None:
raise BadXML("{}, {}".format(a, ka))
class BadXML(Exception):
pass
def parse_xml(txt: str) -> ET.Element:
parser = DXMLParser()
parser.feed(txt)
return parser.close() # type: ignore
def mktnod(name: str, text: str) -> ET.Element:
el = ET.Element(name)
el.text = text
return el
def mkenod(name: str, sub_el: Optional[ET.Element] = None) -> ET.Element:
el = ET.Element(name)
if sub_el is not None:
el.append(sub_el)
return el

View File

@@ -10,12 +10,10 @@ from .authsrv import AXS, VFS
from .bos import bos
from .util import chkcmd, min_ex
try:
if True: # pylint: disable=using-constant-test
from typing import Optional, Union
from .util import RootLogger
except:
pass
class Fstab(object):
@@ -28,7 +26,7 @@ class Fstab(object):
self.age = 0.0
def log(self, msg: str, c: Union[int, str] = 0) -> None:
self.log_func("fstab", msg + "\033[K", c)
self.log_func("fstab", msg, c)
def get(self, path: str) -> str:
if len(self.cache) > 9000:

View File

@@ -6,18 +6,26 @@ import logging
import os
import stat
import sys
import threading
import time
from pyftpdlib.authorizers import AuthenticationFailed, DummyAuthorizer
from pyftpdlib.filesystems import AbstractedFS, FilesystemError
from pyftpdlib.handlers import FTPHandler
from pyftpdlib.log import config_logging
from pyftpdlib.servers import FTPServer
from .__init__ import PY2, TYPE_CHECKING, E
from .__init__ import ANYWIN, PY2, TYPE_CHECKING, E
from .bos import bos
from .util import Pebkac, exclude_dotfiles, fsenc
from .util import (
Daemon,
Pebkac,
exclude_dotfiles,
fsenc,
ipnorm,
pybin,
relchk,
sanitize_fn,
vjoin,
)
try:
from pyftpdlib.ioloop import IOLoop
@@ -31,11 +39,9 @@ except ImportError:
if TYPE_CHECKING:
from .svchub import SvcHub
try:
if True: # pylint: disable=using-constant-test
import typing
from typing import Any, Optional
except:
pass
class FtpAuth(DummyAuthorizer):
@@ -46,21 +52,40 @@ class FtpAuth(DummyAuthorizer):
def validate_authentication(
self, username: str, password: str, handler: Any
) -> None:
handler.username = "{}:{}".format(username, password)
ip = handler.addr[0]
if ip.startswith("::ffff:"):
ip = ip[7:]
ip = ipnorm(ip)
bans = self.hub.bans
if ip in bans:
rt = bans[ip] - time.time()
if rt < 0:
logging.info("client unbanned")
del bans[ip]
else:
raise AuthenticationFailed("banned")
asrv = self.hub.asrv
if username == "anonymous":
password = ""
uname = "*"
else:
uname = asrv.iacct.get(password, "") or asrv.iacct.get(username, "") or "*"
uname = "*"
if password:
uname = asrv.iacct.get(password, "")
if not uname or not (asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)):
g = self.hub.gpwd
if g.lim:
bonk, ip = g.bonk(ip, handler.username)
if bonk:
logging.warning("client banned: invalid passwords")
bans[ip] = bonk
raise AuthenticationFailed("Authentication failed.")
handler.username = uname
if (password and not uname) or not (
asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)
):
raise AuthenticationFailed("Authentication failed.")
def get_home_dir(self, username: str) -> str:
return "/"
@@ -94,6 +119,9 @@ class FtpFs(AbstractedFS):
self.cwd = "/" # pyftpdlib convention of leading slash
self.root = "/var/lib/empty"
self.can_read = self.can_write = self.can_move = False
self.can_delete = self.can_get = self.can_upget = False
self.listdirinfo = self.listdir
self.chdir(".")
@@ -107,6 +135,13 @@ class FtpFs(AbstractedFS):
) -> str:
try:
vpath = vpath.replace("\\", "/").lstrip("/")
rd, fn = os.path.split(vpath)
if ANYWIN and relchk(rd):
logging.warning("malicious vpath: %s", vpath)
raise FilesystemError("unsupported characters in filepath")
fn = sanitize_fn(fn or "", "", [".prologue.html", ".epilogue.html"])
vpath = vjoin(rd, fn)
vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, r, w, m, d)
if not vfs.realpath:
raise FilesystemError("no filesystem mounted at this path")
@@ -145,16 +180,36 @@ class FtpFs(AbstractedFS):
w = "w" in mode or "a" in mode or "+" in mode
ap = self.rv2a(filename, r, w)
if w and bos.path.exists(ap):
raise FilesystemError("cannot open existing file for writing")
if w:
try:
st = bos.stat(ap)
td = time.time() - st.st_mtime
except:
td = 0
if td < -1 or td > self.args.ftp_wt:
raise FilesystemError("cannot open existing file for writing")
self.validpath(ap)
return open(fsenc(ap), mode)
def chdir(self, path: str) -> None:
self.cwd = join(self.cwd, path)
x = self.hub.asrv.vfs.can_access(self.cwd.lstrip("/"), self.h.username)
self.can_read, self.can_write, self.can_move, self.can_delete, self.can_get = x
nwd = join(self.cwd, path)
vfs, rem = self.hub.asrv.vfs.get(nwd, self.uname, False, False)
ap = vfs.canonical(rem)
if not bos.path.isdir(ap):
# returning 550 is library-default and suitable
raise FilesystemError("Failed to change directory")
self.cwd = nwd
(
self.can_read,
self.can_write,
self.can_move,
self.can_delete,
self.can_get,
self.can_upget,
) = self.hub.asrv.vfs.can_access(self.cwd.lstrip("/"), self.h.username)
def mkdir(self, path: str) -> None:
ap = self.rv2a(path, w=True)
@@ -166,7 +221,10 @@ class FtpFs(AbstractedFS):
vfs, rem = self.hub.asrv.vfs.get(vpath, self.uname, True, False)
fsroot, vfs_ls1, vfs_virt = vfs.ls(
rem, self.uname, not self.args.no_scandir, [[True], [False, True]]
rem,
self.uname,
not self.args.no_scandir,
[[True, False], [False, True]],
)
vfs_ls = [x[0] for x in vfs_ls1]
vfs_ls.extend(vfs_virt.keys())
@@ -195,7 +253,7 @@ class FtpFs(AbstractedFS):
vp = join(self.cwd, path).lstrip("/")
try:
self.hub.up2k.handle_rm(self.uname, self.h.remote_ip, [vp])
self.hub.up2k.handle_rm(self.uname, self.h.remote_ip, [vp], [])
except Exception as ex:
raise FilesystemError(str(ex))
@@ -235,11 +293,14 @@ class FtpFs(AbstractedFS):
def lstat(self, path: str) -> os.stat_result:
ap = self.rv2a(path)
return bos.lstat(ap)
return bos.stat(ap)
def isfile(self, path: str) -> bool:
st = self.stat(path)
return stat.S_ISREG(st.st_mode)
try:
st = self.stat(path)
return stat.S_ISREG(st.st_mode)
except:
return False # expected for mojibake in ftp_SIZE()
def islink(self, path: str) -> bool:
ap = self.rv2a(path)
@@ -276,8 +337,8 @@ class FtpFs(AbstractedFS):
class FtpHandler(FTPHandler):
abstracted_fs = FtpFs
hub: "SvcHub" = None
args: argparse.Namespace = None
hub: "SvcHub"
args: argparse.Namespace
def __init__(self, conn: Any, server: Any, ioloop: Any = None) -> None:
self.hub: "SvcHub" = FtpHandler.hub
@@ -291,6 +352,9 @@ class FtpHandler(FTPHandler):
# abspath->vpath mapping to resolve log_transfer paths
self.vfs_map: dict[str, str] = {}
# reduce non-debug logging
self.log_cmds_list = [x for x in self.log_cmds_list if x not in ("CWD", "XCWD")]
def ftp_STOR(self, file: str, mode: str = "w") -> Any:
# Optional[str]
vp = join(self.fs.cwd, file).lstrip("/")
@@ -355,7 +419,7 @@ class Ftpd(object):
h1 = SftpHandler
except:
t = "\nftps requires pyopenssl;\nplease run the following:\n\n {} -m pip install --user pyopenssl\n"
print(t.format(sys.executable))
print(t.format(pybin))
sys.exit(1)
h1.certfile = os.path.join(self.args.E.cfg, "cert.pem")
@@ -385,17 +449,23 @@ class Ftpd(object):
if self.args.ftp_nat:
h2.masquerade_address = self.args.ftp_nat
if self.args.ftp_dbg:
config_logging(level=logging.DEBUG)
lgr = logging.getLogger("pyftpdlib")
lgr.setLevel(logging.DEBUG if self.args.ftpv else logging.INFO)
ips = self.args.i
if "::" in ips:
ips.append("0.0.0.0")
ioloop = IOLoop()
for ip in self.args.i:
for ip in ips:
for h, lp in hs:
FTPServer((ip, int(lp)), h, ioloop)
try:
FTPServer((ip, int(lp)), h, ioloop)
except:
if ip != "0.0.0.0" or "::" not in ips:
raise
thr = threading.Thread(target=ioloop.loop, name="ftp")
thr.daemon = True
thr.start()
Daemon(ioloop.loop, "ftp")
def join(p1: str, p2: str) -> str:

File diff suppressed because it is too large Load Diff

View File

@@ -25,15 +25,16 @@ from .th_srv import HAVE_PIL, HAVE_VIPS
from .u2idx import U2idx
from .util import HMaccas, shut_socket
try:
if True: # pylint: disable=using-constant-test
from typing import Optional, Pattern, Union
except:
pass
if TYPE_CHECKING:
from .httpsrv import HttpSrv
PTN_HTTP = re.compile(br"[A-Z]{3}[A-Z ]")
class HttpConn(object):
"""
spawned by HttpSrv to handle an incoming client connection,
@@ -45,6 +46,7 @@ class HttpConn(object):
) -> None:
self.s = sck
self.sr: Optional[Util._Unrecv] = None
self.cli: Optional[HttpCli] = None
self.addr = addr
self.hsrv = hsrv
@@ -55,14 +57,17 @@ class HttpConn(object):
self.cert_path = hsrv.cert_path
self.u2fh: Util.FHC = hsrv.u2fh # mypy404
self.iphash: HMaccas = hsrv.broker.iphash
self.bans: dict[str, int] = hsrv.bans
self.aclose: dict[str, int] = hsrv.aclose
enth = (HAVE_PIL or HAVE_VIPS or HAVE_FFMPEG) and not self.args.no_thumb
self.thumbcli: Optional[ThumbCli] = ThumbCli(hsrv) if enth else None # mypy404
self.ico: Ico = Ico(self.args) # mypy404
self.t0: float = time.time() # mypy404
self.freshen_pwd: float = 0.0
self.stopping = False
self.nreq: int = 0 # mypy404
self.nreq: int = -1 # mypy404
self.nbyte: int = 0 # mypy404
self.u2idx: Optional[U2idx] = None
self.log_func: "Util.RootLogger" = hsrv.log # mypy404
@@ -134,9 +139,11 @@ class HttpConn(object):
self.s.send(b"HTTP/1.1 400 Bad Request\r\n\r\n" + err.encode("utf-8"))
return False
return method not in [None, b"GET ", b"HEAD", b"POST", b"PUT ", b"OPTI"]
return not method or not bool(PTN_HTTP.match(method))
def run(self) -> None:
self.s.settimeout(10)
self.sr = None
if self.args.https_only:
is_https = True
@@ -205,6 +212,6 @@ class HttpConn(object):
while not self.stopping:
self.nreq += 1
cli = HttpCli(self)
if not cli.run():
self.cli = HttpCli(self)
if not self.cli.run():
return

View File

@@ -28,13 +28,18 @@ except ImportError:
)
sys.exit(1)
from .__init__ import MACOS, TYPE_CHECKING, EnvParams
from .__init__ import ANYWIN, MACOS, TYPE_CHECKING, EnvParams
from .bos import bos
from .httpconn import HttpConn
from .util import (
E_SCK,
FHC,
Daemon,
Garda,
Magician,
Netdev,
NetMap,
ipnorm,
min_ex,
shut_socket,
spack,
@@ -44,11 +49,10 @@ from .util import (
if TYPE_CHECKING:
from .broker_util import BrokerCli
from .ssdp import SSDPr
try:
if True: # pylint: disable=using-constant-test
from typing import Any, Optional
except:
pass
class HttpSrv(object):
@@ -70,10 +74,14 @@ class HttpSrv(object):
nsuf = "-n{}-i{:x}".format(nid, os.getpid()) if nid else ""
self.magician = Magician()
self.bans: dict[str, int] = {}
self.nm = NetMap([], {})
self.ssdp: Optional["SSDPr"] = None
self.gpwd = Garda(self.args.ban_pw)
self.g404 = Garda(self.args.ban_404)
self.bans: dict[str, int] = {}
self.aclose: dict[str, int] = {}
self.bound: set[tuple[str, int]] = set()
self.name = "hsrv" + nsuf
self.mutex = threading.Lock()
self.stopping = False
@@ -96,13 +104,21 @@ class HttpSrv(object):
env = jinja2.Environment()
env.loader = jinja2.FileSystemLoader(os.path.join(self.E.mod, "web"))
self.j2 = {
x: env.get_template(x + ".html")
for x in ["splash", "browser", "browser2", "msg", "md", "mde", "cf"]
}
jn = ["splash", "svcs", "browser", "browser2", "msg", "md", "mde", "cf"]
self.j2 = {x: env.get_template(x + ".html") for x in jn}
zs = os.path.join(self.E.mod, "web", "deps", "prism.js.gz")
self.prism = os.path.exists(zs)
self.mallow = "GET HEAD POST PUT DELETE OPTIONS".split()
if not self.args.no_dav:
zs = "PROPFIND PROPPATCH LOCK UNLOCK MKCOL COPY MOVE"
self.mallow += zs.split()
if self.args.zs:
from .ssdp import SSDPr
self.ssdp = SSDPr(broker)
cert_path = os.path.join(self.E.cfg, "cert.pem")
if bos.path.exists(cert_path):
self.cert_path = cert_path
@@ -120,9 +136,7 @@ class HttpSrv(object):
start_log_thrs(self.log, self.args.log_thrs, nid)
self.th_cfg: dict[str, Any] = {}
t = threading.Thread(target=self.post_init, name="hsrv-init2")
t.daemon = True
t.start()
Daemon(self.post_init, "hsrv-init2")
def post_init(self) -> None:
try:
@@ -131,18 +145,20 @@ class HttpSrv(object):
except:
pass
def set_netdevs(self, netdevs: dict[str, Netdev]) -> None:
ips = set()
for ip, _ in self.bound:
ips.add(ip)
self.nm = NetMap(list(ips), netdevs)
def start_threads(self, n: int) -> None:
self.tp_nthr += n
if self.args.log_htp:
self.log(self.name, "workers += {} = {}".format(n, self.tp_nthr), 6)
for _ in range(n):
thr = threading.Thread(
target=self.thr_poolw,
name=self.name + "-poolw",
)
thr.daemon = True
thr.start()
Daemon(self.thr_poolw, self.name + "-poolw")
def stop_threads(self, n: int) -> None:
self.tp_nthr -= n
@@ -170,26 +186,28 @@ class HttpSrv(object):
def listen(self, sck: socket.socket, nlisteners: int) -> None:
if self.args.j != 1:
# lost in the pickle; redefine
sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sck.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
sck.settimeout(None) # < does not inherit, ^ does
if not ANYWIN or self.args.reuseaddr:
sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
ip, port = sck.getsockname()
sck.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
sck.settimeout(None) # < does not inherit, ^ opts above do
ip, port = sck.getsockname()[:2]
self.srvs.append(sck)
self.bound.add((ip, port))
self.nclimax = math.ceil(self.args.nc * 1.0 / nlisteners)
t = threading.Thread(
target=self.thr_listen,
args=(sck,),
name="httpsrv-n{}-listen-{}-{}".format(self.nid or "0", ip, port),
Daemon(
self.thr_listen,
"httpsrv-n{}-listen-{}-{}".format(self.nid or "0", ip, port),
(sck,),
)
t.daemon = True
t.start()
def thr_listen(self, srv_sck: socket.socket) -> None:
"""listens on a shared tcp server"""
ip, port = srv_sck.getsockname()
ip, port = srv_sck.getsockname()[:2]
fno = srv_sck.fileno()
msg = "subscribed @ {}:{} f{} p{}".format(ip, port, fno, os.getpid())
hip = "[{}]".format(ip) if ":" in ip else ip
msg = "subscribed @ {}:{} f{} p{}".format(hip, port, fno, os.getpid())
self.log(self.name, msg)
def fun() -> None:
@@ -199,19 +217,80 @@ class HttpSrv(object):
while not self.stopping:
if self.args.log_conn:
self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="1;30")
self.log(self.name, "|%sC-ncli" % ("-" * 1,), c="90")
if self.ncli >= self.nclimax:
self.log(self.name, "at connection limit; waiting", 3)
while self.ncli >= self.nclimax:
time.sleep(0.1)
spins = 0
while self.ncli >= self.nclimax:
if not spins:
self.log(self.name, "at connection limit; waiting", 3)
spins += 1
time.sleep(0.1)
if spins != 50 or not self.args.aclose:
continue
ipfreq: dict[str, int] = {}
with self.mutex:
for c in self.clients:
ip = ipnorm(c.ip)
try:
ipfreq[ip] += 1
except:
ipfreq[ip] = 1
ip, n = sorted(ipfreq.items(), key=lambda x: x[1], reverse=True)[0]
if n < self.nclimax / 2:
continue
self.aclose[ip] = int(time.time() + self.args.aclose * 60)
nclose = 0
nloris = 0
nconn = 0
with self.mutex:
for c in self.clients:
cip = ipnorm(c.ip)
if ip != cip:
continue
nconn += 1
try:
if (
c.nreq >= 1
or not c.cli
or c.cli.in_hdr_recv
or c.cli.keepalive
):
Daemon(c.shutdown)
nclose += 1
if c.nreq <= 0 and (not c.cli or c.cli.in_hdr_recv):
nloris += 1
except:
pass
t = "{} downgraded to connection:close for {} min; dropped {}/{} connections"
self.log(self.name, t.format(ip, self.args.aclose, nclose, nconn), 1)
if nloris < nconn / 2:
continue
t = "slowloris (idle-conn): {} banned for {} min"
self.log(self.name, t.format(ip, self.args.loris, nclose), 1)
self.bans[ip] = int(time.time() + self.args.loris * 60)
if self.args.log_conn:
self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="1;30")
self.log(self.name, "|%sC-acc1" % ("-" * 2,), c="90")
try:
sck, addr = srv_sck.accept()
sck, saddr = srv_sck.accept()
cip, cport = saddr[:2]
if cip.startswith("::ffff:"):
cip = cip[7:]
addr = (cip, cport)
except (OSError, socket.error) as ex:
if self.stopping:
break
self.log(self.name, "accept({}): {}".format(fno, ex), c=6)
time.sleep(0.02)
continue
@@ -220,7 +299,7 @@ class HttpSrv(object):
t = "|{}C-acc2 \033[0;36m{} \033[3{}m{}".format(
"-" * 3, ip, port % 8, port
)
self.log("%s %s" % addr, t, c="1;30")
self.log("%s %s" % addr, t, c="90")
self.accept(sck, addr)
@@ -241,10 +320,7 @@ class HttpSrv(object):
if self.nid:
name += "-{}".format(self.nid)
thr = threading.Thread(target=self.periodic, name=name)
self.t_periodic = thr
thr.daemon = True
thr.start()
self.t_periodic = Daemon(self.periodic, name)
if self.tp_q:
self.tp_time = self.tp_time or now
@@ -259,13 +335,11 @@ class HttpSrv(object):
t = "looks like the httpserver threadpool died; please make an issue on github and tell me the story of how you pulled that off, thanks and dog bless\n"
self.log(self.name, t, 1)
thr = threading.Thread(
target=self.thr_client,
args=(sck, addr),
name="httpconn-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
Daemon(
self.thr_client,
"httpconn-{}-{}".format(addr[0].split(".", 2)[-1][-6:], addr[1]),
(sck, addr),
)
thr.daemon = True
thr.start()
def thr_poolw(self) -> None:
assert self.tp_q
@@ -324,15 +398,16 @@ class HttpSrv(object):
with self.mutex:
self.clients.add(cli)
# print("{}\n".format(len(self.clients)), end="")
fno = sck.fileno()
try:
if self.args.log_conn:
self.log("%s %s" % addr, "|%sC-crun" % ("-" * 4,), c="1;30")
self.log("%s %s" % addr, "|%sC-crun" % ("-" * 4,), c="90")
cli.run()
except (OSError, socket.error) as ex:
if ex.errno not in [10038, 10054, 107, 57, 49, 9]:
if ex.errno not in E_SCK:
self.log(
"%s %s" % addr,
"run({}): {}".format(fno, ex),
@@ -342,7 +417,7 @@ class HttpSrv(object):
finally:
sck = cli.s
if self.args.log_conn:
self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 5,), c="1;30")
self.log("%s %s" % addr, "|%sC-cdone" % ("-" * 5,), c="90")
try:
fno = sck.fileno()
@@ -352,15 +427,9 @@ class HttpSrv(object):
self.log(
"%s %s" % addr,
"shut({}): {}".format(fno, ex),
c="1;30",
c="90",
)
if ex.errno not in [10038, 10054, 107, 57, 49, 9]:
# 10038 No longer considered a socket
# 10054 Foribly closed by remote
# 107 Transport endpoint not connected
# 57 Socket is not connected
# 49 Can't assign requested address (wifi down)
# 9 Bad file descriptor
if ex.errno not in E_SCK:
raise
finally:
with self.mutex:

538
copyparty/mdns.py Normal file
View File

@@ -0,0 +1,538 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import random
import select
import socket
import time
from ipaddress import IPv4Network, IPv6Network
from .__init__ import TYPE_CHECKING
from .__init__ import unicode as U
from .multicast import MC_Sck, MCast
from .stolen.dnslib import AAAA
from .stolen.dnslib import CLASS as DC
from .stolen.dnslib import (
NSEC,
PTR,
QTYPE,
RR,
SRV,
TXT,
A,
DNSHeader,
DNSQuestion,
DNSRecord,
)
from .util import CachedSet, Daemon, Netdev, list_ips, min_ex
if TYPE_CHECKING:
from .svchub import SvcHub
if True: # pylint: disable=using-constant-test
from typing import Any, Optional, Union
MDNS4 = "224.0.0.251"
MDNS6 = "ff02::fb"
class MDNS_Sck(MC_Sck):
def __init__(
self,
sck: socket.socket,
nd: Netdev,
grp: str,
ip: str,
net: Union[IPv4Network, IPv6Network],
):
super(MDNS_Sck, self).__init__(sck, nd, grp, ip, net)
self.bp_probe = b""
self.bp_ip = b""
self.bp_svc = b""
self.bp_bye = b""
self.last_tx = 0.0
self.tx_ex = False
class MDNS(MCast):
def __init__(self, hub: "SvcHub", ngen: int) -> None:
al = hub.args
grp4 = "" if al.zm6 else MDNS4
grp6 = "" if al.zm4 else MDNS6
super(MDNS, self).__init__(
hub, MDNS_Sck, al.zm_on, al.zm_off, grp4, grp6, 5353, hub.args.zmv
)
self.srv: dict[socket.socket, MDNS_Sck] = {}
self.logsrc = "mDNS-{}".format(ngen)
self.ngen = ngen
self.ttl = 300
zs = self.args.name + ".local."
zs = zs.encode("ascii", "replace").decode("ascii", "replace")
self.hn = "-".join(x for x in zs.split("?") if x) or (
"vault-{}".format(random.randint(1, 255))
)
self.lhn = self.hn.lower()
# requester ip -> (response deadline, srv, body):
self.q: dict[str, tuple[float, MDNS_Sck, bytes]] = {}
self.rx4 = CachedSet(0.42) # 3 probes @ 250..500..750 => 500ms span
self.rx6 = CachedSet(0.42)
self.svcs, self.sfqdns = self.build_svcs()
self.lsvcs = {k.lower(): v for k, v in self.svcs.items()}
self.lsfqdns = set([x.lower() for x in self.sfqdns])
self.probing = 0.0
self.unsolicited: list[float] = [] # scheduled announces on all nics
self.defend: dict[MDNS_Sck, float] = {} # server -> deadline
def log(self, msg: str, c: Union[int, str] = 0) -> None:
self.log_func(self.logsrc, msg, c)
def build_svcs(self) -> tuple[dict[str, dict[str, Any]], set[str]]:
zms = self.args.zms
http = {"port": 80 if 80 in self.args.p else self.args.p[0]}
https = {"port": 443 if 443 in self.args.p else self.args.p[0]}
webdav = http.copy()
webdavs = https.copy()
webdav["u"] = webdavs["u"] = "u" # KDE requires username
ftp = {"port": (self.args.ftp if "f" in zms else self.args.ftps)}
smb = {"port": self.args.smb_port}
# some gvfs require path
zs = self.args.zm_ld or "/"
if zs:
webdav["path"] = zs
webdavs["path"] = zs
if self.args.zm_lh:
http["path"] = self.args.zm_lh
https["path"] = self.args.zm_lh
if self.args.zm_lf:
ftp["path"] = self.args.zm_lf
if self.args.zm_ls:
smb["path"] = self.args.zm_ls
svcs: dict[str, dict[str, Any]] = {}
if "d" in zms:
svcs["_webdav._tcp.local."] = webdav
if "D" in zms:
svcs["_webdavs._tcp.local."] = webdavs
if "h" in zms:
svcs["_http._tcp.local."] = http
if "H" in zms:
svcs["_https._tcp.local."] = https
if "f" in zms.lower():
svcs["_ftp._tcp.local."] = ftp
if "s" in zms.lower():
svcs["_smb._tcp.local."] = smb
sfqdns: set[str] = set()
for k, v in svcs.items():
name = "{}-c-{}".format(self.args.name, k.split(".")[0][1:])
v["name"] = name
sfqdns.add("{}.{}".format(name, k))
return svcs, sfqdns
def build_replies(self) -> None:
for srv in self.srv.values():
probe = DNSRecord(DNSHeader(0, 0), q=DNSQuestion(self.hn, QTYPE.ANY))
areply = DNSRecord(DNSHeader(0, 0x8400))
sreply = DNSRecord(DNSHeader(0, 0x8400))
bye = DNSRecord(DNSHeader(0, 0x8400))
have4 = have6 = False
for s2 in self.srv.values():
if srv.idx != s2.idx:
continue
if s2.v6:
have6 = True
else:
have4 = True
for ip in srv.ips:
if ":" in ip:
qt = QTYPE.AAAA
ar = {"rclass": DC.F_IN, "rdata": AAAA(ip)}
else:
qt = QTYPE.A
ar = {"rclass": DC.F_IN, "rdata": A(ip)}
r0 = RR(self.hn, qt, ttl=0, **ar)
r120 = RR(self.hn, qt, ttl=120, **ar)
# rfc-10:
# SHOULD rr ttl 120sec for A/AAAA/SRV
# (and recommend 75min for all others)
probe.add_auth(r120)
areply.add_answer(r120)
sreply.add_answer(r120)
bye.add_answer(r0)
for sclass, props in self.svcs.items():
sname = props["name"]
sport = props["port"]
sfqdn = sname + "." + sclass
k = "_services._dns-sd._udp.local."
r = RR(k, QTYPE.PTR, DC.IN, 4500, PTR(sclass))
sreply.add_answer(r)
r = RR(sclass, QTYPE.PTR, DC.IN, 4500, PTR(sfqdn))
sreply.add_answer(r)
r = RR(sfqdn, QTYPE.SRV, DC.F_IN, 120, SRV(0, 0, sport, self.hn))
sreply.add_answer(r)
areply.add_answer(r)
r = RR(sfqdn, QTYPE.SRV, DC.F_IN, 0, SRV(0, 0, sport, self.hn))
bye.add_answer(r)
txts = []
for k in ("u", "path"):
if k not in props:
continue
zb = "{}={}".format(k, props[k]).encode("utf-8")
if len(zb) > 255:
t = "value too long for mdns: [{}]"
raise Exception(t.format(props[k]))
txts.append(zb)
# gvfs really wants txt even if they're empty
r = RR(sfqdn, QTYPE.TXT, DC.F_IN, 4500, TXT(txts))
sreply.add_answer(r)
if not (have4 and have6) and not self.args.zm_noneg:
ns = NSEC(self.hn, ["AAAA" if have6 else "A"])
r = RR(self.hn, QTYPE.NSEC, DC.F_IN, 120, ns)
areply.add_ar(r)
if len(sreply.pack()) < 1400:
sreply.add_ar(r)
srv.bp_probe = probe.pack()
srv.bp_ip = areply.pack()
srv.bp_svc = sreply.pack()
srv.bp_bye = bye.pack()
# since all replies are small enough to fit in one packet,
# always send full replies rather than just a/aaaa records
srv.bp_ip = srv.bp_svc
def send_probes(self) -> None:
slp = random.random() * 0.25
for _ in range(3):
time.sleep(slp)
slp = 0.25
if not self.running:
break
if self.args.zmv:
self.log("sending hostname probe...")
# ipv4: need to probe each ip (each server)
# ipv6: only need to probe each set of looped nics
probed6: set[str] = set()
for srv in self.srv.values():
if srv.ip in probed6:
continue
try:
srv.sck.sendto(srv.bp_probe, (srv.grp, 5353))
if srv.v6:
for ip in srv.ips:
probed6.add(ip)
except Exception as ex:
self.log("sendto failed: {} ({})".format(srv.ip, ex), "90")
def run(self) -> None:
try:
bound = self.create_servers()
except:
t = "no server IP matches the mdns config\n{}"
self.log(t.format(min_ex()), 1)
bound = []
if not bound:
self.log("failed to announce copyparty services on the network", 3)
return
self.build_replies()
Daemon(self.send_probes)
zf = time.time() + 2
self.probing = zf # cant unicast so give everyone an extra sec
self.unsolicited = [zf, zf + 1, zf + 3, zf + 7] # rfc-8.3
last_hop = time.time()
ihop = self.args.mc_hop
while self.running:
timeout = (
0.02 + random.random() * 0.07
if self.probing or self.q or self.defend or self.unsolicited
else (last_hop + ihop if ihop else 180)
)
rdy = select.select(self.srv, [], [], timeout)
rx: list[socket.socket] = rdy[0] # type: ignore
self.rx4.cln()
self.rx6.cln()
buf = b""
addr = ("0", 0)
for sck in rx:
try:
buf, addr = sck.recvfrom(4096)
self.eat(buf, addr, sck)
except:
if not self.running:
self.log("stopped", 2)
return
t = "{} {} \033[33m|{}| {}\n{}".format(
self.srv[sck].name, addr, len(buf), repr(buf)[2:-1], min_ex()
)
self.log(t, 6)
if not self.probing:
self.process()
continue
if self.probing < time.time():
t = "probe ok; announcing [{}]"
self.log(t.format(self.hn[:-1]), 2)
self.probing = 0
self.log("stopped", 2)
def stop(self, panic=False) -> None:
self.running = False
for srv in self.srv.values():
try:
if panic:
srv.sck.close()
else:
srv.sck.sendto(srv.bp_bye, (srv.grp, 5353))
except:
pass
self.srv = {}
def eat(self, buf: bytes, addr: tuple[str, int], sck: socket.socket) -> None:
cip = addr[0]
v6 = ":" in cip
if (cip.startswith("169.254") and not self.ll_ok) or (
v6 and not cip.startswith("fe80")
):
return
cache = self.rx6 if v6 else self.rx4
if buf in cache.c:
return
srv: Optional[MDNS_Sck] = self.srv[sck] if v6 else self.map_client(cip) # type: ignore
if not srv:
return
cache.add(buf)
now = time.time()
if self.args.zmv and cip != srv.ip and cip not in srv.ips:
t = "{} [{}] \033[36m{} \033[0m|{}|"
self.log(t.format(srv.name, srv.ip, cip, len(buf)), "90")
p = DNSRecord.parse(buf)
if self.args.zmvv:
self.log(str(p))
# check for incoming probes for our hostname
cips = [U(x.rdata) for x in p.auth if U(x.rname).lower() == self.lhn]
if cips and self.sips.isdisjoint(cips):
if not [x for x in cips if x not in ("::1", "127.0.0.1")]:
# avahi broadcasting 127.0.0.1-only packets
return
self.log("someone trying to steal our hostname: {}".format(cips), 3)
# immediately unicast
if not self.probing:
srv.sck.sendto(srv.bp_ip, (cip, 5353))
# and schedule multicast
self.defend[srv] = self.defend.get(srv, now + 0.1)
return
# check for someone rejecting our probe / hijacking our hostname
cips = [
U(x.rdata)
for x in p.rr
if U(x.rname).lower() == self.lhn and x.rclass == DC.F_IN
]
if cips and self.sips.isdisjoint(cips):
if not [x for x in cips if x not in ("::1", "127.0.0.1")]:
# avahi broadcasting 127.0.0.1-only packets
return
# check if we've been given additional IPs
for ip in list_ips():
if ip in cips:
self.sips.add(ip)
if not self.sips.isdisjoint(cips):
return
t = "mdns zeroconf: "
if self.probing:
t += "Cannot start; hostname '{}' is occupied"
else:
t += "Emergency stop; hostname '{}' got stolen"
t += " on {}! Use --name to set another hostname.\n\nName taken by {}\n\nYour IPs: {}\n"
self.log(t.format(self.args.name, srv.name, cips, list(self.sips)), 1)
self.stop(True)
return
# then rfc-6.7; dns pretending to be mdns (android...)
if p.header.id or addr[1] != 5353:
rsp: Optional[DNSRecord] = None
for r in p.questions:
try:
lhn = U(r.qname).lower()
except:
self.log("invalid question: {}".format(r))
continue
if lhn != self.lhn:
continue
if p.header.id and r.qtype in (QTYPE.A, QTYPE.AAAA):
rsp = rsp or DNSRecord(DNSHeader(p.header.id, 0x8400))
rsp.add_question(r)
for ip in srv.ips:
qt = r.qtype
v6 = ":" in ip
if v6 == (qt == QTYPE.AAAA):
rd = AAAA(ip) if v6 else A(ip)
rr = RR(self.hn, qt, DC.IN, 10, rd)
rsp.add_answer(rr)
if rsp:
srv.sck.sendto(rsp.pack(), addr[:2])
# but don't return in case it's a differently broken client
# then a/aaaa records
for r in p.questions:
try:
lhn = U(r.qname).lower()
except:
self.log("invalid question: {}".format(r))
continue
if lhn != self.lhn:
continue
# gvfs keeps repeating itself
found = False
unicast = False
for rr in p.rr:
try:
rname = U(rr.rname).lower()
except:
self.log("invalid rr: {}".format(rr))
continue
if rname == self.lhn:
if rr.ttl > 60:
found = True
if rr.rclass == DC.F_IN:
unicast = True
if unicast:
# spec-compliant mDNS-over-unicast
srv.sck.sendto(srv.bp_ip, (cip, 5353))
elif addr[1] != 5353:
# just in case some clients use (and want us to use) invalid ports
srv.sck.sendto(srv.bp_ip, addr[:2])
if not found:
self.q[cip] = (0, srv, srv.bp_ip)
return
deadline = now + (0.5 if p.header.tc else 0.02) # rfc-7.2
# and service queries
for r in p.questions:
if not r or not r.qname:
continue
qname = U(r.qname).lower()
if qname in self.lsvcs or qname == "_services._dns-sd._udp.local.":
self.q[cip] = (deadline, srv, srv.bp_svc)
break
# heed rfc-7.1 if there was an announce in the past 12sec
# (workaround gvfs race-condition where it occasionally
# doesn't read/decode the full response...)
if now < srv.last_tx + 12:
for rr in p.rr:
if not rr.rdata:
continue
rdata = U(rr.rdata).lower()
if rdata in self.lsfqdns:
if rr.ttl > 2250:
self.q.pop(cip, None)
break
def process(self) -> None:
tx = set()
now = time.time()
cooldown = 0.9 # rfc-6: 1
if self.unsolicited and self.unsolicited[0] < now:
self.unsolicited.pop(0)
cooldown = 0.1
for srv in self.srv.values():
tx.add(srv)
for srv, deadline in list(self.defend.items()):
if now < deadline:
continue
if self._tx(srv, srv.bp_ip, 0.02): # rfc-6: 0.25
self.defend.pop(srv)
for cip, (deadline, srv, msg) in list(self.q.items()):
if now < deadline:
continue
self.q.pop(cip)
self._tx(srv, msg, cooldown)
for srv in tx:
self._tx(srv, srv.bp_svc, cooldown)
def _tx(self, srv: MDNS_Sck, msg: bytes, cooldown: float) -> bool:
now = time.time()
if now < srv.last_tx + cooldown:
return False
try:
srv.sck.sendto(msg, (srv.grp, 5353))
srv.last_tx = now
except Exception as ex:
if srv.tx_ex:
return True
srv.tx_ex = True
t = "tx({},|{}|,{}): {}"
self.log(t.format(srv.ip, len(msg), cooldown, ex), 3)
return True

View File

@@ -10,27 +10,36 @@ import sys
from .__init__ import PY2, WINDOWS, E, unicode
from .bos import bos
from .util import REKOBO_LKEY, fsenc, min_ex, retchk, runcmd, uncyg
from .util import (
FFMPEG_URL,
REKOBO_LKEY,
fsenc,
is_exe,
min_ex,
pybin,
retchk,
runcmd,
sfsenc,
uncyg,
)
try:
if True: # pylint: disable=using-constant-test
from typing import Any, Union
from .util import RootLogger
except:
pass
def have_ff(cmd: str) -> bool:
def have_ff(scmd: str) -> bool:
if PY2:
print("# checking {}".format(cmd))
cmd = (cmd + " -version").encode("ascii").split(b" ")
print("# checking {}".format(scmd))
acmd = (scmd + " -version").encode("ascii").split(b" ")
try:
sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE).communicate()
sp.Popen(acmd, stdout=sp.PIPE, stderr=sp.PIPE).communicate()
return True
except:
return False
else:
return bool(shutil.which(cmd))
return bool(shutil.which(scmd))
HAVE_FFMPEG = have_ff("ffmpeg")
@@ -269,7 +278,7 @@ class MTag(object):
if self.backend == "mutagen":
self.get = self.get_mutagen
try:
import mutagen # noqa: F401 # pylint: disable=unused-import,import-outside-toplevel
from mutagen import version # noqa: F401
except:
self.log("could not load Mutagen, trying FFprobe instead", c=3)
self.backend = "ffprobe"
@@ -287,9 +296,14 @@ class MTag(object):
self.log(msg, c=3)
if not self.usable:
if is_exe:
t = "copyparty.exe cannot use mutagen; need ffprobe.exe to read media tags: "
self.log(t + FFMPEG_URL)
return
msg = "need Mutagen{} to read media tags so please run this:\n{}{} -m pip install --user mutagen\n"
pybin = os.path.basename(sys.executable)
self.log(msg.format(or_ffprobe, " " * 37, pybin), c=1)
pyname = os.path.basename(pybin)
self.log(msg.format(or_ffprobe, " " * 37, pyname), c=1)
return
# https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
@@ -381,20 +395,26 @@ class MTag(object):
parser_output[alias] = (priority, tv[0])
# take first value (lowest priority / most preferred)
ret = {sk: unicode(tv[1]).strip() for sk, tv in parser_output.items()}
ret: dict[str, Union[str, float]] = {
sk: unicode(tv[1]).strip() for sk, tv in parser_output.items()
}
# track 3/7 => track 3
for sk, tv in ret.items():
for sk, zv in ret.items():
if sk[0] == ".":
sv = str(tv).split("/")[0].strip().lstrip("0")
sv = str(zv).split("/")[0].strip().lstrip("0")
ret[sk] = sv or 0
# normalize key notation to rkeobo
okey = ret.get("key")
if okey:
key = okey.replace(" ", "").replace("maj", "").replace("min", "m")
key = str(okey).replace(" ", "").replace("maj", "").replace("min", "m")
ret["key"] = REKOBO_LKEY.get(key.lower(), okey)
if self.args.mtag_vv:
zl = " ".join("\033[36m{} \033[33m{}".format(k, v) for k, v in ret.items())
self.log("norm: {}\033[0m".format(zl), "90")
return ret
def compare(self, abspath: str) -> dict[str, Union[str, float]]:
@@ -441,10 +461,15 @@ class MTag(object):
if not bos.path.isfile(abspath):
return {}
import mutagen
from mutagen import File
try:
md = mutagen.File(fsenc(abspath), easy=True)
md = File(fsenc(abspath), easy=True)
assert md
if self.args.mtag_vv:
for zd in (md.info.__dict__, dict(md.tags)):
zl = ["\033[36m{} \033[33m{}".format(k, v) for k, v in zd.items()]
self.log("mutagen: {}\033[0m".format(" ".join(zl)), "90")
if not md.info.length and not md.info.codec:
raise Exception()
except:
@@ -494,6 +519,12 @@ class MTag(object):
return {}
ret, md = ffprobe(abspath, self.args.mtag_to)
if self.args.mtag_vv:
for zd in (ret, dict(md)):
zl = ["\033[36m{} \033[33m{}".format(k, v) for k, v in zd.items()]
self.log("ffprobe: {}\033[0m".format(" ".join(zl)), "90")
return self.normalize_tags(ret, md)
def get_bin(
@@ -504,12 +535,15 @@ class MTag(object):
env = os.environ.copy()
try:
if is_exe:
raise Exception()
pypath = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
zsl = [str(pypath)] + [str(x) for x in sys.path if x]
pypath = str(os.pathsep.join(zsl))
env["PYTHONPATH"] = pypath
except:
if not E.ox:
if not E.ox and not is_exe:
raise
ret: dict[str, Any] = {}
@@ -517,7 +551,7 @@ class MTag(object):
try:
cmd = [parser.bin, abspath]
if parser.bin.endswith(".py"):
cmd = [sys.executable] + cmd
cmd = [pybin] + cmd
args = {
"env": env,
@@ -536,7 +570,7 @@ class MTag(object):
else:
cmd = ["nice"] + cmd
bcmd = [fsenc(x) for x in cmd]
bcmd = [sfsenc(x) for x in cmd[:-1]] + [fsenc(cmd[-1])]
rc, v, err = runcmd(bcmd, **args) # type: ignore
retchk(rc, bcmd, err, self.log, 5, self.args.mtag_v)
v = v.strip()

370
copyparty/multicast.py Normal file
View File

@@ -0,0 +1,370 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import socket
import time
import ipaddress
from ipaddress import (
IPv4Address,
IPv4Network,
IPv6Address,
IPv6Network,
ip_address,
ip_network,
)
from .__init__ import MACOS, TYPE_CHECKING
from .util import Netdev, find_prefix, min_ex, spack
if TYPE_CHECKING:
from .svchub import SvcHub
if True: # pylint: disable=using-constant-test
from typing import Optional, Union
if not hasattr(socket, "IPPROTO_IPV6"):
setattr(socket, "IPPROTO_IPV6", 41)
class NoIPs(Exception):
pass
class MC_Sck(object):
"""there is one socket for each server ip"""
def __init__(
self,
sck: socket.socket,
nd: Netdev,
grp: str,
ip: str,
net: Union[IPv4Network, IPv6Network],
):
self.sck = sck
self.idx = nd.idx
self.name = nd.name
self.grp = grp
self.mreq = b""
self.ip = ip
self.net = net
self.ips = {ip: net}
self.v6 = ":" in ip
self.have4 = ":" not in ip
self.have6 = ":" in ip
class MCast(object):
def __init__(
self,
hub: "SvcHub",
Srv: type[MC_Sck],
on: list[str],
off: list[str],
mc_grp_4: str,
mc_grp_6: str,
port: int,
vinit: bool,
) -> None:
"""disable ipv%d by setting mc_grp_%d empty"""
self.hub = hub
self.Srv = Srv
self.args = hub.args
self.asrv = hub.asrv
self.log_func = hub.log
self.on = on
self.off = off
self.grp4 = mc_grp_4
self.grp6 = mc_grp_6
self.port = port
self.vinit = vinit
self.srv: dict[socket.socket, MC_Sck] = {} # listening sockets
self.sips: set[str] = set() # all listening ips (including failed attempts)
self.ll_ok: set[str] = set() # fallback linklocal IPv4 and IPv6 addresses
self.b2srv: dict[bytes, MC_Sck] = {} # binary-ip -> server socket
self.b4: list[bytes] = [] # sorted list of binary-ips
self.b6: list[bytes] = [] # sorted list of binary-ips
self.cscache: dict[str, Optional[MC_Sck]] = {} # client ip -> server cache
self.running = True
def log(self, msg: str, c: Union[int, str] = 0) -> None:
self.log_func("multicast", msg, c)
def create_servers(self) -> list[str]:
bound: list[str] = []
netdevs = self.hub.tcpsrv.netdevs
ips = [x[0] for x in self.hub.tcpsrv.bound]
if "::" in ips:
ips = [x for x in ips if x != "::"] + list(
[x.split("/")[0] for x in netdevs if ":" in x]
)
ips.append("0.0.0.0")
if "0.0.0.0" in ips:
ips = [x for x in ips if x != "0.0.0.0"] + list(
[x.split("/")[0] for x in netdevs if ":" not in x]
)
ips = [x for x in ips if x not in ("::1", "127.0.0.1")]
ips = find_prefix(ips, netdevs)
on = self.on[:]
off = self.off[:]
for lst in (on, off):
for av in list(lst):
try:
arg_net = ip_network(av, False)
except:
arg_net = None
for sk, sv in netdevs.items():
if arg_net:
net_ip = ip_address(sk.split("/")[0])
if net_ip in arg_net and sk not in lst:
lst.append(sk)
if (av == str(sv.idx) or av == sv.name) and sk not in lst:
lst.append(sk)
if on:
ips = [x for x in ips if x in on]
elif off:
ips = [x for x in ips if x not in off]
if not self.grp4:
ips = [x for x in ips if ":" in x]
if not self.grp6:
ips = [x for x in ips if ":" not in x]
ips = list(set(ips))
all_selected = ips[:]
# discard non-linklocal ipv6
ips = [x for x in ips if ":" not in x or x.startswith("fe80")]
if not ips:
raise NoIPs()
for ip in ips:
v6 = ":" in ip
netdev = netdevs[ip]
if not netdev.idx:
t = "using INADDR_ANY for ip [{}], netdev [{}]"
if not self.srv and ip not in ["::", "0.0.0.0"]:
self.log(t.format(ip, netdev), 3)
ipv = socket.AF_INET6 if v6 else socket.AF_INET
sck = socket.socket(ipv, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
sck.settimeout(None)
sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sck.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except:
pass
# most ipv6 clients expect multicast on linklocal ip only;
# add a/aaaa records for the other nic IPs
other_ips: set[str] = set()
if v6:
for nd in netdevs.values():
if nd.idx == netdev.idx and nd.ip in all_selected and ":" in nd.ip:
other_ips.add(nd.ip)
net = ipaddress.ip_network(ip, False)
ip = ip.split("/")[0]
srv = self.Srv(sck, netdev, self.grp6 if ":" in ip else self.grp4, ip, net)
for oth_ip in other_ips:
srv.ips[oth_ip.split("/")[0]] = ipaddress.ip_network(oth_ip, False)
# gvfs breaks if a linklocal ip appears in a dns reply
ll = {
k: v
for k, v in srv.ips.items()
if k.startswith("169.254") or k.startswith("fe80")
}
rt = {k: v for k, v in srv.ips.items() if k not in ll}
if self.args.ll or not rt:
self.ll_ok.update(list(ll))
if not self.args.ll:
srv.ips = rt or ll
if not srv.ips:
self.log("no IPs on {}; skipping [{}]".format(netdev, ip), 3)
continue
try:
self.setup_socket(srv)
self.srv[sck] = srv
bound.append(ip)
except:
t = "announce failed on {} [{}]:\n{}"
self.log(t.format(netdev, ip, min_ex()), 3)
if self.args.zm_msub:
for s1 in self.srv.values():
for s2 in self.srv.values():
if s1.idx != s2.idx:
continue
if s1.ip not in s2.ips:
s2.ips[s1.ip] = s1.net
if self.args.zm_mnic:
for s1 in self.srv.values():
for s2 in self.srv.values():
for ip1, net1 in list(s1.ips.items()):
for ip2, net2 in list(s2.ips.items()):
if net1 == net2 and ip1 != ip2:
s1.ips[ip2] = net2
self.sips = set([x.split("/")[0] for x in all_selected])
for srv in self.srv.values():
assert srv.ip in self.sips
return bound
def setup_socket(self, srv: MC_Sck) -> None:
sck = srv.sck
if srv.v6:
if self.vinit:
zsl = list(srv.ips.keys())
self.log("v6({}) idx({}) {}".format(srv.ip, srv.idx, zsl), 6)
for ip in srv.ips:
bip = socket.inet_pton(socket.AF_INET6, ip)
self.b2srv[bip] = srv
self.b6.append(bip)
grp = self.grp6 if srv.idx else ""
try:
if MACOS:
raise Exception()
sck.bind((grp, self.port, 0, srv.idx))
except:
sck.bind(("", self.port, 0, srv.idx))
bgrp = socket.inet_pton(socket.AF_INET6, self.grp6)
dev = spack(b"@I", srv.idx)
srv.mreq = bgrp + dev
if srv.idx != socket.INADDR_ANY:
sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, dev)
try:
sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS, 255)
sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_LOOP, 1)
except:
# macos
t = "failed to set IPv6 TTL/LOOP; announcements may not survive multiple switches/routers"
self.log(t, 3)
else:
if self.vinit:
self.log("v4({}) idx({})".format(srv.ip, srv.idx), 6)
bip = socket.inet_aton(srv.ip)
self.b2srv[bip] = srv
self.b4.append(bip)
grp = self.grp4 if srv.idx else ""
try:
if MACOS:
raise Exception()
sck.bind((grp, self.port))
except:
sck.bind(("", self.port))
bgrp = socket.inet_aton(self.grp4)
dev = (
spack(b"=I", socket.INADDR_ANY)
if srv.idx == socket.INADDR_ANY
else socket.inet_aton(srv.ip)
)
srv.mreq = bgrp + dev
if srv.idx != socket.INADDR_ANY:
sck.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, dev)
try:
sck.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 255)
sck.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, 1)
except:
# probably can't happen but dontcare if it does
t = "failed to set IPv4 TTL/LOOP; announcements may not survive multiple switches/routers"
self.log(t, 3)
self.hop(srv)
self.b4.sort(reverse=True)
self.b6.sort(reverse=True)
def hop(self, srv: MC_Sck) -> None:
"""rejoin to keepalive on routers/switches without igmp-snooping"""
sck = srv.sck
req = srv.mreq
if ":" in srv.ip:
try:
sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_LEAVE_GROUP, req)
# linux does leaves/joins twice with 0.2~1.05s spacing
time.sleep(1.2)
except:
pass
sck.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, req)
else:
try:
sck.setsockopt(socket.IPPROTO_IP, socket.IP_DROP_MEMBERSHIP, req)
time.sleep(1.2)
except:
pass
# t = "joining {} from ip {} idx {} with mreq {}"
# self.log(t.format(srv.grp, srv.ip, srv.idx, repr(srv.mreq)), 6)
sck.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, req)
def map_client(self, cip: str) -> Optional[MC_Sck]:
try:
return self.cscache[cip]
except:
pass
ret: Optional[MC_Sck] = None
v6 = ":" in cip
ci = IPv6Address(cip) if v6 else IPv4Address(cip)
for x in self.b6 if v6 else self.b4:
srv = self.b2srv[x]
if any([x for x in srv.ips.values() if ci in x]):
ret = srv
break
if not ret and cip in ("127.0.0.1", "::1"):
# just give it something
ret = list(self.srv.values())[0]
if not ret and cip.startswith("169.254"):
# idk how to map LL IPv4 msgs to nics;
# just pick one and hope for the best
lls = (
x
for x in self.srv.values()
if next((y for y in x.ips if y in self.ll_ok), None)
)
ret = next(lls, None)
if ret:
t = "new client on {} ({}): {}"
self.log(t.format(ret.name, ret.net, cip), 6)
else:
t = "could not map client {} to known subnet; maybe forwarded from another network?"
self.log(t.format(cip), 3)
if len(self.cscache) > 9000:
self.cscache = {}
self.cscache[cip] = ret
return ret

325
copyparty/smbd.py Normal file
View File

@@ -0,0 +1,325 @@
# coding: utf-8
import inspect
import logging
import os
import random
import stat
import sys
import time
from types import SimpleNamespace
from .__init__ import ANYWIN, TYPE_CHECKING
from .authsrv import LEELOO_DALLAS, VFS
from .bos import bos
from .util import Daemon, is_exe, min_ex, pybin
if True: # pylint: disable=using-constant-test
from typing import Any
if TYPE_CHECKING:
from .svchub import SvcHub
lg = logging.getLogger("smb")
debug, info, warning, error = (lg.debug, lg.info, lg.warning, lg.error)
class SMB(object):
def __init__(self, hub: "SvcHub") -> None:
self.hub = hub
self.args = hub.args
self.asrv = hub.asrv
self.log = hub.log
self.files: dict[int, tuple[float, str]] = {}
lg.setLevel(logging.DEBUG if self.args.smbvvv else logging.INFO)
for x in ["impacket", "impacket.smbserver"]:
lgr = logging.getLogger(x)
lgr.setLevel(logging.DEBUG if self.args.smbvv else logging.INFO)
try:
from impacket import smbserver
from impacket.ntlm import compute_lmhash, compute_nthash
except ImportError:
if is_exe:
print("copyparty.exe cannot do SMB")
sys.exit(1)
m = "\033[36m\n{}\033[31m\n\nERROR: need 'impacket'; please run this command:\033[33m\n {} -m pip install --user impacket\n\033[0m"
print(m.format(min_ex(), pybin))
sys.exit(1)
# patch vfs into smbserver.os
fos = SimpleNamespace()
for k in os.__dict__:
try:
setattr(fos, k, getattr(os, k))
except:
pass
fos.close = self._close
fos.listdir = self._listdir
fos.mkdir = self._mkdir
fos.open = self._open
fos.remove = self._unlink
fos.rename = self._rename
fos.stat = self._stat
fos.unlink = self._unlink
fos.utime = self._utime
smbserver.os = fos
# ...and smbserver.os.path
fop = SimpleNamespace()
for k in os.path.__dict__:
try:
setattr(fop, k, getattr(os.path, k))
except:
pass
fop.exists = self._p_exists
fop.getsize = self._p_getsize
fop.isdir = self._p_isdir
smbserver.os.path = fop
if not self.args.smb_nwa_2:
fop.join = self._p_join
# other patches
smbserver.isInFileJail = self._is_in_file_jail
self._disarm()
ip = next((x for x in self.args.i if ":" not in x), None)
if not ip:
self.log("smb", "IPv6 not supported for SMB; listening on 0.0.0.0", 3)
ip = "0.0.0.0"
port = int(self.args.smb_port)
srv = smbserver.SimpleSMBServer(listenAddress=ip, listenPort=port)
ro = "no" if self.args.smbw else "yes" # (does nothing)
srv.addShare("A", "/", readOnly=ro)
srv.setSMB2Support(not self.args.smb1)
for name, pwd in self.asrv.acct.items():
for u, p in ((name, pwd), (pwd, "k")):
lmhash = compute_lmhash(p)
nthash = compute_nthash(p)
srv.addCredential(u, 0, lmhash, nthash)
chi = [random.randint(0, 255) for x in range(8)]
cha = "".join(["{:02x}".format(x) for x in chi])
srv.setSMBChallenge(cha)
self.srv = srv
self.stop = srv.stop
self.log("smb", "listening @ {}:{}".format(ip, port))
def start(self) -> None:
Daemon(self.srv.start)
def _v2a(self, caller: str, vpath: str, *a: Any) -> tuple[VFS, str]:
vpath = vpath.replace("\\", "/").lstrip("/")
# cf = inspect.currentframe().f_back
# c1 = cf.f_back.f_code.co_name
# c2 = cf.f_code.co_name
debug('%s("%s", %s)\033[K\033[0m', caller, vpath, str(a))
# TODO find a way to grab `identity` in smbComSessionSetupAndX and smb2SessionSetup
vfs, rem = self.asrv.vfs.get(vpath, LEELOO_DALLAS, True, True)
return vfs, vfs.canonical(rem)
def _listdir(self, vpath: str, *a: Any, **ka: Any) -> list[str]:
vpath = vpath.replace("\\", "/").lstrip("/")
# caller = inspect.currentframe().f_back.f_code.co_name
debug('listdir("%s", %s)\033[K\033[0m', vpath, str(a))
vfs, rem = self.asrv.vfs.get(vpath, LEELOO_DALLAS, False, False)
_, vfs_ls, vfs_virt = vfs.ls(
rem, LEELOO_DALLAS, not self.args.no_scandir, [[False, False]]
)
dirs = [x[0] for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)]
fils = [x[0] for x in vfs_ls if x[0] not in dirs]
ls = list(vfs_virt.keys()) + dirs + fils
if self.args.smb_nwa_1:
return ls
# clients crash somewhere around 65760 byte
ret = []
sz = 112 * 2 # ['.', '..']
for n, fn in enumerate(ls):
if sz >= 64000:
t = "listing only %d of %d files (%d byte); see impacket#1433"
warning(t, n, len(ls), sz)
break
nsz = len(fn.encode("utf-16", "replace"))
nsz = ((nsz + 7) // 8) * 8
sz += 104 + nsz
ret.append(fn)
return ret
def _open(
self, vpath: str, flags: int, *a: Any, chmod: int = 0o777, **ka: Any
) -> Any:
f_ro = os.O_RDONLY
if ANYWIN:
f_ro |= os.O_BINARY
wr = flags != f_ro
if wr and not self.args.smbw:
yeet("blocked write (no --smbw): " + vpath)
vfs, ap = self._v2a("open", vpath, *a)
if wr and not vfs.axs.uwrite:
yeet("blocked write (no-write-acc): " + vpath)
ret = bos.open(ap, flags, *a, mode=chmod, **ka)
if wr:
now = time.time()
nf = len(self.files)
if nf > 9000:
oldest = min([x[0] for x in self.files.values()])
cutoff = oldest + (now - oldest) / 2
self.files = {k: v for k, v in self.files.items() if v[0] > cutoff}
info("was tracking %d files, now %d", nf, len(self.files))
vpath = vpath.replace("\\", "/").lstrip("/")
self.files[ret] = (now, vpath)
return ret
def _close(self, fd: int) -> None:
os.close(fd)
if fd not in self.files:
return
_, vp = self.files.pop(fd)
vp, fn = os.path.split(vp)
vfs, rem = self.hub.asrv.vfs.get(vp, LEELOO_DALLAS, False, True)
vfs, rem = vfs.get_dbv(rem)
self.hub.up2k.hash_file(
vfs.realpath,
vfs.flags,
rem,
fn,
"1.7.6.2",
time.time(),
)
def _rename(self, vp1: str, vp2: str) -> None:
if not self.args.smbw:
yeet("blocked rename (no --smbw): " + vp1)
vp1 = vp1.lstrip("/")
vp2 = vp2.lstrip("/")
vfs2, ap2 = self._v2a("rename", vp2, vp1)
if not vfs2.axs.uwrite:
yeet("blocked rename (no-write-acc): " + vp2)
vfs1, _ = self.asrv.vfs.get(vp1, LEELOO_DALLAS, True, True)
if not vfs1.axs.umove:
yeet("blocked rename (no-move-acc): " + vp1)
self.hub.up2k.handle_mv(LEELOO_DALLAS, vp1, vp2)
try:
bos.makedirs(ap2)
except:
pass
def _mkdir(self, vpath: str) -> None:
if not self.args.smbw:
yeet("blocked mkdir (no --smbw): " + vpath)
vfs, ap = self._v2a("mkdir", vpath)
if not vfs.axs.uwrite:
yeet("blocked mkdir (no-write-acc): " + vpath)
return bos.mkdir(ap)
def _stat(self, vpath: str, *a: Any, **ka: Any) -> os.stat_result:
return bos.stat(self._v2a("stat", vpath, *a)[1], *a, **ka)
def _unlink(self, vpath: str) -> None:
if not self.args.smbw:
yeet("blocked delete (no --smbw): " + vpath)
# return bos.unlink(self._v2a("stat", vpath, *a)[1])
vfs, ap = self._v2a("delete", vpath)
if not vfs.axs.udel:
yeet("blocked delete (no-del-acc): " + vpath)
vpath = vpath.replace("\\", "/").lstrip("/")
self.hub.up2k.handle_rm(LEELOO_DALLAS, "1.7.6.2", [vpath], [])
def _utime(self, vpath: str, times: tuple[float, float]) -> None:
if not self.args.smbw:
yeet("blocked utime (no --smbw): " + vpath)
vfs, ap = self._v2a("utime", vpath)
if not vfs.axs.uwrite:
yeet("blocked utime (no-write-acc): " + vpath)
return bos.utime(ap, times)
def _p_exists(self, vpath: str) -> bool:
try:
bos.stat(self._v2a("p.exists", vpath)[1])
return True
except:
return False
def _p_getsize(self, vpath: str) -> int:
st = bos.stat(self._v2a("p.getsize", vpath)[1])
return st.st_size
def _p_isdir(self, vpath: str) -> bool:
try:
st = bos.stat(self._v2a("p.isdir", vpath)[1])
return stat.S_ISDIR(st.st_mode)
except:
return False
def _p_join(self, *a) -> str:
# impacket.smbserver reads globs from queryDirectoryRequest['Buffer']
# where somehow `fds.*` becomes `fds"*` so lets fix that
ret = os.path.join(*a)
return ret.replace('"', ".") # type: ignore
def _hook(self, *a: Any, **ka: Any) -> None:
src = inspect.currentframe().f_back.f_code.co_name
error("\033[31m%s:hook(%s)\033[0m", src, a)
raise Exception("nope")
def _disarm(self) -> None:
from impacket import smbserver
smbserver.os.chmod = self._hook
smbserver.os.chown = self._hook
smbserver.os.ftruncate = self._hook
smbserver.os.lchown = self._hook
smbserver.os.link = self._hook
smbserver.os.lstat = self._hook
smbserver.os.replace = self._hook
smbserver.os.scandir = self._hook
smbserver.os.symlink = self._hook
smbserver.os.truncate = self._hook
smbserver.os.walk = self._hook
smbserver.os.path.abspath = self._hook
smbserver.os.path.expanduser = self._hook
smbserver.os.path.getatime = self._hook
smbserver.os.path.getctime = self._hook
smbserver.os.path.getmtime = self._hook
smbserver.os.path.isabs = self._hook
smbserver.os.path.isfile = self._hook
smbserver.os.path.islink = self._hook
smbserver.os.path.realpath = self._hook
def _is_in_file_jail(self, *a: Any) -> bool:
# handled by vfs
return True
def yeet(msg: str) -> None:
info(msg)
raise Exception(msg)

210
copyparty/ssdp.py Normal file
View File

@@ -0,0 +1,210 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import re
import select
import socket
from email.utils import formatdate
from .__init__ import TYPE_CHECKING
from .multicast import MC_Sck, MCast
from .util import CachedSet, html_escape, min_ex
if TYPE_CHECKING:
from .broker_util import BrokerCli
from .httpcli import HttpCli
from .svchub import SvcHub
if True: # pylint: disable=using-constant-test
from typing import Optional, Union
GRP = "239.255.255.250"
class SSDP_Sck(MC_Sck):
def __init__(self, *a):
super(SSDP_Sck, self).__init__(*a)
self.hport = 0
class SSDPr(object):
"""generates http responses for httpcli"""
def __init__(self, broker: "BrokerCli") -> None:
self.broker = broker
self.args = broker.args
def reply(self, hc: "HttpCli") -> bool:
if hc.vpath.endswith("device.xml"):
return self.tx_device(hc)
hc.reply(b"unknown request", 400)
return False
def tx_device(self, hc: "HttpCli") -> bool:
zs = """
<?xml version="1.0"?>
<root xmlns="urn:schemas-upnp-org:device-1-0">
<specVersion>
<major>1</major>
<minor>0</minor>
</specVersion>
<URLBase>{}</URLBase>
<device>
<presentationURL>{}</presentationURL>
<deviceType>urn:schemas-upnp-org:device:Basic:1</deviceType>
<friendlyName>{}</friendlyName>
<modelDescription>file server</modelDescription>
<manufacturer>ed</manufacturer>
<manufacturerURL>https://ocv.me/</manufacturerURL>
<modelName>copyparty</modelName>
<modelURL>https://github.com/9001/copyparty/</modelURL>
<UDN>{}</UDN>
<serviceList>
<service>
<serviceType>urn:schemas-upnp-org:device:Basic:1</serviceType>
<serviceId>urn:schemas-upnp-org:device:Basic</serviceId>
<controlURL>/.cpr/ssdp/services.xml</controlURL>
<eventSubURL>/.cpr/ssdp/services.xml</eventSubURL>
<SCPDURL>/.cpr/ssdp/services.xml</SCPDURL>
</service>
</serviceList>
</device>
</root>"""
c = html_escape
sip, sport = hc.s.getsockname()[:2]
sip = sip.replace("::ffff:", "")
proto = "https" if self.args.https_only else "http"
ubase = "{}://{}:{}".format(proto, sip, sport)
zsl = self.args.zsl
url = zsl if "://" in zsl else ubase + "/" + zsl.lstrip("/")
name = "{} @ {}".format(self.args.doctitle, self.args.name)
zs = zs.strip().format(c(ubase), c(url), c(name), c(self.args.zsid))
hc.reply(zs.encode("utf-8", "replace"))
return False # close connectino
class SSDPd(MCast):
"""communicates with ssdp clients over multicast"""
def __init__(self, hub: "SvcHub", ngen: int) -> None:
al = hub.args
vinit = al.zsv and not al.zmv
super(SSDPd, self).__init__(
hub, SSDP_Sck, al.zs_on, al.zs_off, GRP, "", 1900, vinit
)
self.srv: dict[socket.socket, SSDP_Sck] = {}
self.logsrc = "SSDP-{}".format(ngen)
self.ngen = ngen
self.rxc = CachedSet(0.7)
self.txc = CachedSet(5) # win10: every 3 sec
self.ptn_st = re.compile(b"\nst: *upnp:rootdevice", re.I)
def log(self, msg: str, c: Union[int, str] = 0) -> None:
self.log_func(self.logsrc, msg, c)
def run(self) -> None:
try:
bound = self.create_servers()
except:
t = "no server IP matches the ssdp config\n{}"
self.log(t.format(min_ex()), 1)
bound = []
if not bound:
self.log("failed to announce copyparty services on the network", 3)
return
# find http port for this listening ip
for srv in self.srv.values():
tcps = self.hub.tcpsrv.bound
hp = next((x[1] for x in tcps if x[0] in ("0.0.0.0", srv.ip)), 0)
hp = hp or next((x[1] for x in tcps if x[0] == "::"), 0)
if not hp:
hp = tcps[0][1]
self.log("assuming port {} for {}".format(hp, srv.ip), 3)
srv.hport = hp
self.log("listening")
while self.running:
rdy = select.select(self.srv, [], [], self.args.z_chk or 180)
rx: list[socket.socket] = rdy[0] # type: ignore
self.rxc.cln()
buf = b""
addr = ("0", 0)
for sck in rx:
try:
buf, addr = sck.recvfrom(4096)
self.eat(buf, addr)
except:
if not self.running:
break
t = "{} {} \033[33m|{}| {}\n{}".format(
self.srv[sck].name, addr, len(buf), repr(buf)[2:-1], min_ex()
)
self.log(t, 6)
self.log("stopped", 2)
def stop(self) -> None:
self.running = False
for srv in self.srv.values():
try:
srv.sck.close()
except:
pass
self.srv = {}
def eat(self, buf: bytes, addr: tuple[str, int]) -> None:
cip = addr[0]
if cip.startswith("169.254") and not self.ll_ok:
return
if buf in self.rxc.c:
return
srv: Optional[SSDP_Sck] = self.map_client(cip) # type: ignore
if not srv:
return
self.rxc.add(buf)
if not buf.startswith(b"M-SEARCH * HTTP/1."):
return
if not self.ptn_st.search(buf):
return
if self.args.zsv:
t = "{} [{}] \033[36m{} \033[0m|{}|"
self.log(t.format(srv.name, srv.ip, cip, len(buf)), "90")
zs = """
HTTP/1.1 200 OK
CACHE-CONTROL: max-age=1800
DATE: {0}
EXT:
LOCATION: http://{1}:{2}/.cpr/ssdp/device.xml
OPT: "http://schemas.upnp.org/upnp/1/0/"; ns=01
01-NLS: {3}
SERVER: UPnP/1.0
ST: upnp:rootdevice
USN: {3}::upnp:rootdevice
BOOTID.UPNP.ORG: 0
CONFIGID.UPNP.ORG: 1
"""
v4 = srv.ip.replace("::ffff:", "")
zs = zs.format(formatdate(usegmt=True), v4, srv.hport, self.args.zsid)
zb = zs[1:].replace("\n", "\r\n").encode("utf-8", "replace")
srv.sck.sendto(zb, addr[:2])
if cip not in self.txc.c:
self.log("{} [{}] --> {}".format(srv.name, srv.ip, cip), "6")
self.txc.add(cip)
self.txc.cln()

View File

@@ -1,21 +1,19 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import stat
import tarfile
import threading
from queue import Queue
from .bos import bos
from .sutil import StreamArc, errdesc
from .util import fsenc, min_ex
from .util import Daemon, fsenc, min_ex
try:
if True: # pylint: disable=using-constant-test
from typing import Any, Generator, Optional
from .util import NamedLogger
except:
pass
class QFile(object): # inherit io.StringIO for painful typing
@@ -60,9 +58,7 @@ class StreamTar(StreamArc):
fmt = tarfile.GNU_FORMAT
self.tar = tarfile.open(fileobj=self.qfile, mode="w|", format=fmt) # type: ignore
w = threading.Thread(target=self._gen, name="star-gen")
w.daemon = True
w.start()
Daemon(self._gen, "star-gen")
def gen(self) -> Generator[Optional[bytes], None, None]:
try:
@@ -84,6 +80,9 @@ class StreamTar(StreamArc):
src = f["ap"]
fsi = f["st"]
if stat.S_ISDIR(fsi.st_mode):
return
inf = tarfile.TarInfo(name=name)
inf.mode = fsi.st_mode
inf.size = fsi.st_size

View File

@@ -0,0 +1,5 @@
`dnslib` but heavily simplified/feature-stripped
L: MIT
Copyright (c) 2010 - 2017 Paul Chakravarti
https://github.com/paulc/dnslib/

View File

@@ -0,0 +1,11 @@
# coding: utf-8
"""
L: MIT
Copyright (c) 2010 - 2017 Paul Chakravarti
https://github.com/paulc/dnslib/tree/0.9.23
"""
from .dns import *
version = "0.9.23"

View File

@@ -0,0 +1,41 @@
# coding: utf-8
import types
class BimapError(Exception):
pass
class Bimap(object):
def __init__(self, name, forward, error=AttributeError):
self.name = name
self.error = error
self.forward = forward.copy()
self.reverse = dict([(v, k) for (k, v) in list(forward.items())])
def get(self, k, default=None):
try:
return self.forward[k]
except KeyError:
return default or str(k)
def __getitem__(self, k):
try:
return self.forward[k]
except KeyError:
if isinstance(self.error, types.FunctionType):
return self.error(self.name, k, True)
else:
raise self.error("%s: Invalid forward lookup: [%s]" % (self.name, k))
def __getattr__(self, k):
try:
if k == "__wrapped__":
raise AttributeError()
return self.reverse[k]
except KeyError:
if isinstance(self.error, types.FunctionType):
return self.error(self.name, k, False)
else:
raise self.error("%s: Invalid reverse lookup: [%s]" % (self.name, k))

View File

@@ -0,0 +1,15 @@
# coding: utf-8
from __future__ import print_function
def get_bits(data, offset, bits=1):
mask = ((1 << bits) - 1) << offset
return (data & mask) >> offset
def set_bits(data, value, offset, bits=1):
mask = ((1 << bits) - 1) << offset
clear = 0xFFFF ^ mask
data = (data & clear) | ((value << offset) & mask)
return data

View File

@@ -0,0 +1,56 @@
# coding: utf-8
import binascii
import struct
class BufferError(Exception):
pass
class Buffer(object):
def __init__(self, data=b""):
self.data = bytearray(data)
self.offset = 0
def remaining(self):
return len(self.data) - self.offset
def get(self, length):
if length > self.remaining():
raise BufferError(
"Not enough bytes [offset=%d,remaining=%d,requested=%d]"
% (self.offset, self.remaining(), length)
)
start = self.offset
end = self.offset + length
self.offset += length
return bytes(self.data[start:end])
def hex(self):
return binascii.hexlify(self.data)
def pack(self, fmt, *args):
self.offset += struct.calcsize(fmt)
self.data += struct.pack(fmt, *args)
def append(self, s):
self.offset += len(s)
self.data += s
def update(self, ptr, fmt, *args):
s = struct.pack(fmt, *args)
self.data[ptr : ptr + len(s)] = s
def unpack(self, fmt):
try:
data = self.get(struct.calcsize(fmt))
return struct.unpack(fmt, data)
except struct.error:
raise BufferError(
"Error unpacking struct '%s' <%s>"
% (fmt, binascii.hexlify(data).decode())
)
def __len__(self):
return len(self.data)

View File

@@ -0,0 +1,775 @@
# coding: utf-8
from __future__ import print_function
import binascii
from itertools import chain
from .bimap import Bimap, BimapError
from .bit import get_bits, set_bits
from .buffer import BufferError
from .label import DNSBuffer, DNSLabel
from .ranges import IP4, IP6, H, I, check_bytes
class DNSError(Exception):
pass
def unknown_qtype(name, key, forward):
if forward:
try:
return "TYPE%d" % (key,)
except:
raise DNSError("%s: Invalid forward lookup: [%s]" % (name, key))
else:
if key.startswith("TYPE"):
try:
return int(key[4:])
except:
pass
raise DNSError("%s: Invalid reverse lookup: [%s]" % (name, key))
QTYPE = Bimap(
"QTYPE",
{1: "A", 12: "PTR", 16: "TXT", 28: "AAAA", 33: "SRV", 47: "NSEC", 255: "ANY"},
unknown_qtype,
)
CLASS = Bimap("CLASS", {1: "IN", 254: "None", 255: "*", 0x8001: "F_IN"}, DNSError)
QR = Bimap("QR", {0: "QUERY", 1: "RESPONSE"}, DNSError)
RCODE = Bimap(
"RCODE",
{
0: "NOERROR",
1: "FORMERR",
2: "SERVFAIL",
3: "NXDOMAIN",
4: "NOTIMP",
5: "REFUSED",
6: "YXDOMAIN",
7: "YXRRSET",
8: "NXRRSET",
9: "NOTAUTH",
10: "NOTZONE",
},
DNSError,
)
OPCODE = Bimap(
"OPCODE", {0: "QUERY", 1: "IQUERY", 2: "STATUS", 4: "NOTIFY", 5: "UPDATE"}, DNSError
)
def label(label, origin=None):
if label.endswith("."):
return DNSLabel(label)
else:
return (origin if isinstance(origin, DNSLabel) else DNSLabel(origin)).add(label)
class DNSRecord(object):
@classmethod
def parse(cls, packet) -> "DNSRecord":
buffer = DNSBuffer(packet)
try:
header = DNSHeader.parse(buffer)
questions = []
rr = []
auth = []
ar = []
for i in range(header.q):
questions.append(DNSQuestion.parse(buffer))
for i in range(header.a):
rr.append(RR.parse(buffer))
for i in range(header.auth):
auth.append(RR.parse(buffer))
for i in range(header.ar):
ar.append(RR.parse(buffer))
return cls(header, questions, rr, auth=auth, ar=ar)
except (BufferError, BimapError) as e:
raise DNSError(
"Error unpacking DNSRecord [offset=%d]: %s" % (buffer.offset, e)
)
@classmethod
def question(cls, qname, qtype="A", qclass="IN"):
return DNSRecord(
q=DNSQuestion(qname, getattr(QTYPE, qtype), getattr(CLASS, qclass))
)
def __init__(
self, header=None, questions=None, rr=None, q=None, a=None, auth=None, ar=None
) -> None:
self.header = header or DNSHeader()
self.questions: list[DNSQuestion] = questions or []
self.rr: list[RR] = rr or []
self.auth: list[RR] = auth or []
self.ar: list[RR] = ar or []
if q:
self.questions.append(q)
if a:
self.rr.append(a)
self.set_header_qa()
def reply(self, ra=1, aa=1):
return DNSRecord(
DNSHeader(id=self.header.id, bitmap=self.header.bitmap, qr=1, ra=ra, aa=aa),
q=self.q,
)
def add_question(self, *q) -> None:
self.questions.extend(q)
self.set_header_qa()
def add_answer(self, *rr) -> None:
self.rr.extend(rr)
self.set_header_qa()
def add_auth(self, *auth) -> None:
self.auth.extend(auth)
self.set_header_qa()
def add_ar(self, *ar) -> None:
self.ar.extend(ar)
self.set_header_qa()
def set_header_qa(self) -> None:
self.header.q = len(self.questions)
self.header.a = len(self.rr)
self.header.auth = len(self.auth)
self.header.ar = len(self.ar)
def get_q(self):
return self.questions[0] if self.questions else DNSQuestion()
q = property(get_q)
def get_a(self):
return self.rr[0] if self.rr else RR()
a = property(get_a)
def pack(self) -> bytes:
self.set_header_qa()
buffer = DNSBuffer()
self.header.pack(buffer)
for q in self.questions:
q.pack(buffer)
for rr in self.rr:
rr.pack(buffer)
for auth in self.auth:
auth.pack(buffer)
for ar in self.ar:
ar.pack(buffer)
return buffer.data
def truncate(self):
return DNSRecord(DNSHeader(id=self.header.id, bitmap=self.header.bitmap, tc=1))
def format(self, prefix="", sort=False):
s = sorted if sort else lambda x: x
sections = [repr(self.header)]
sections.extend(s([repr(q) for q in self.questions]))
sections.extend(s([repr(rr) for rr in self.rr]))
sections.extend(s([repr(rr) for rr in self.auth]))
sections.extend(s([repr(rr) for rr in self.ar]))
return prefix + ("\n" + prefix).join(sections)
short = format
def __repr__(self):
return self.format()
__str__ = __repr__
class DNSHeader(object):
id = H("id")
bitmap = H("bitmap")
q = H("q")
a = H("a")
auth = H("auth")
ar = H("ar")
@classmethod
def parse(cls, buffer):
try:
(id, bitmap, q, a, auth, ar) = buffer.unpack("!HHHHHH")
return cls(id, bitmap, q, a, auth, ar)
except (BufferError, BimapError) as e:
raise DNSError(
"Error unpacking DNSHeader [offset=%d]: %s" % (buffer.offset, e)
)
def __init__(self, id=None, bitmap=None, q=0, a=0, auth=0, ar=0, **args) -> None:
self.id = id if id else 0
if bitmap is None:
self.bitmap = 0
else:
self.bitmap = bitmap
self.q = q
self.a = a
self.auth = auth
self.ar = ar
for k, v in args.items():
if k.lower() == "qr":
self.qr = v
elif k.lower() == "opcode":
self.opcode = v
elif k.lower() == "aa":
self.aa = v
elif k.lower() == "tc":
self.tc = v
elif k.lower() == "rd":
self.rd = v
elif k.lower() == "ra":
self.ra = v
elif k.lower() == "z":
self.z = v
elif k.lower() == "ad":
self.ad = v
elif k.lower() == "cd":
self.cd = v
elif k.lower() == "rcode":
self.rcode = v
def get_qr(self):
return get_bits(self.bitmap, 15)
def set_qr(self, val):
self.bitmap = set_bits(self.bitmap, val, 15)
qr = property(get_qr, set_qr)
def get_opcode(self):
return get_bits(self.bitmap, 11, 4)
def set_opcode(self, val):
self.bitmap = set_bits(self.bitmap, val, 11, 4)
opcode = property(get_opcode, set_opcode)
def get_aa(self):
return get_bits(self.bitmap, 10)
def set_aa(self, val):
self.bitmap = set_bits(self.bitmap, val, 10)
aa = property(get_aa, set_aa)
def get_tc(self):
return get_bits(self.bitmap, 9)
def set_tc(self, val):
self.bitmap = set_bits(self.bitmap, val, 9)
tc = property(get_tc, set_tc)
def get_rd(self):
return get_bits(self.bitmap, 8)
def set_rd(self, val):
self.bitmap = set_bits(self.bitmap, val, 8)
rd = property(get_rd, set_rd)
def get_ra(self):
return get_bits(self.bitmap, 7)
def set_ra(self, val):
self.bitmap = set_bits(self.bitmap, val, 7)
ra = property(get_ra, set_ra)
def get_z(self):
return get_bits(self.bitmap, 6)
def set_z(self, val):
self.bitmap = set_bits(self.bitmap, val, 6)
z = property(get_z, set_z)
def get_ad(self):
return get_bits(self.bitmap, 5)
def set_ad(self, val):
self.bitmap = set_bits(self.bitmap, val, 5)
ad = property(get_ad, set_ad)
def get_cd(self):
return get_bits(self.bitmap, 4)
def set_cd(self, val):
self.bitmap = set_bits(self.bitmap, val, 4)
cd = property(get_cd, set_cd)
def get_rcode(self):
return get_bits(self.bitmap, 0, 4)
def set_rcode(self, val):
self.bitmap = set_bits(self.bitmap, val, 0, 4)
rcode = property(get_rcode, set_rcode)
def pack(self, buffer):
buffer.pack("!HHHHHH", self.id, self.bitmap, self.q, self.a, self.auth, self.ar)
def __repr__(self):
f = [
self.aa and "AA",
self.tc and "TC",
self.rd and "RD",
self.ra and "RA",
self.z and "Z",
self.ad and "AD",
self.cd and "CD",
]
if OPCODE.get(self.opcode) == "UPDATE":
f1 = "zo"
f2 = "pr"
f3 = "up"
f4 = "ad"
else:
f1 = "q"
f2 = "a"
f3 = "ns"
f4 = "ar"
return (
"<DNS Header: id=0x%x type=%s opcode=%s flags=%s "
"rcode='%s' %s=%d %s=%d %s=%d %s=%d>"
% (
self.id,
QR.get(self.qr),
OPCODE.get(self.opcode),
",".join(filter(None, f)),
RCODE.get(self.rcode),
f1,
self.q,
f2,
self.a,
f3,
self.auth,
f4,
self.ar,
)
)
__str__ = __repr__
class DNSQuestion(object):
@classmethod
def parse(cls, buffer):
try:
qname = buffer.decode_name()
qtype, qclass = buffer.unpack("!HH")
return cls(qname, qtype, qclass)
except (BufferError, BimapError) as e:
raise DNSError(
"Error unpacking DNSQuestion [offset=%d]: %s" % (buffer.offset, e)
)
def __init__(self, qname=None, qtype=1, qclass=1) -> None:
self.qname = qname
self.qtype = qtype
self.qclass = qclass
def set_qname(self, qname):
if isinstance(qname, DNSLabel):
self._qname = qname
else:
self._qname = DNSLabel(qname)
def get_qname(self):
return self._qname
qname = property(get_qname, set_qname)
def pack(self, buffer):
buffer.encode_name(self.qname)
buffer.pack("!HH", self.qtype, self.qclass)
def __repr__(self):
return "<DNS Question: '%s' qtype=%s qclass=%s>" % (
self.qname,
QTYPE.get(self.qtype),
CLASS.get(self.qclass),
)
__str__ = __repr__
class RR(object):
rtype = H("rtype")
rclass = H("rclass")
ttl = I("ttl")
rdlength = H("rdlength")
@classmethod
def parse(cls, buffer):
try:
rname = buffer.decode_name()
rtype, rclass, ttl, rdlength = buffer.unpack("!HHIH")
if rdlength:
rdata = RDMAP.get(QTYPE.get(rtype), RD).parse(buffer, rdlength)
else:
rdata = ""
return cls(rname, rtype, rclass, ttl, rdata)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking RR [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, rname=None, rtype=1, rclass=1, ttl=0, rdata=None) -> None:
self.rname = rname
self.rtype = rtype
self.rclass = rclass
self.ttl = ttl
self.rdata = rdata
def set_rname(self, rname):
if isinstance(rname, DNSLabel):
self._rname = rname
else:
self._rname = DNSLabel(rname)
def get_rname(self):
return self._rname
rname = property(get_rname, set_rname)
def pack(self, buffer):
buffer.encode_name(self.rname)
buffer.pack("!HHI", self.rtype, self.rclass, self.ttl)
rdlength_ptr = buffer.offset
buffer.pack("!H", 0)
start = buffer.offset
self.rdata.pack(buffer)
end = buffer.offset
buffer.update(rdlength_ptr, "!H", end - start)
def __repr__(self):
return "<DNS RR: '%s' rtype=%s rclass=%s ttl=%d rdata='%s'>" % (
self.rname,
QTYPE.get(self.rtype),
CLASS.get(self.rclass),
self.ttl,
self.rdata,
)
__str__ = __repr__
class RD(object):
@classmethod
def parse(cls, buffer, length):
try:
data = buffer.get(length)
return cls(data)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking RD [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, data=b"") -> None:
check_bytes("data", data)
self.data = bytes(data)
def pack(self, buffer):
buffer.append(self.data)
def __repr__(self):
if len(self.data) > 0:
return "\\# %d %s" % (
len(self.data),
binascii.hexlify(self.data).decode().upper(),
)
else:
return "\\# 0"
attrs = ("data",)
def _force_bytes(x):
if isinstance(x, bytes):
return x
else:
return x.encode()
class TXT(RD):
@classmethod
def parse(cls, buffer, length):
try:
data = list()
start_bo = buffer.offset
now_length = 0
while buffer.offset < start_bo + length:
(txtlength,) = buffer.unpack("!B")
if now_length + txtlength < length:
now_length += txtlength
data.append(buffer.get(txtlength))
else:
raise DNSError(
"Invalid TXT record: len(%d) > RD len(%d)" % (txtlength, length)
)
return cls(data)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking TXT [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, data) -> None:
if type(data) in (tuple, list):
self.data = [_force_bytes(x) for x in data]
else:
self.data = [_force_bytes(data)]
if any([len(x) > 255 for x in self.data]):
raise DNSError("TXT record too long: %s" % self.data)
def pack(self, buffer):
for ditem in self.data:
if len(ditem) > 255:
raise DNSError("TXT record too long: %s" % ditem)
buffer.pack("!B", len(ditem))
buffer.append(ditem)
def __repr__(self):
return ",".join([repr(x) for x in self.data])
class A(RD):
data = IP4("data")
@classmethod
def parse(cls, buffer, length):
try:
data = buffer.unpack("!BBBB")
return cls(data)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking A [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, data) -> None:
if type(data) in (tuple, list):
self.data = tuple(data)
else:
self.data = tuple(map(int, data.rstrip(".").split(".")))
def pack(self, buffer):
buffer.pack("!BBBB", *self.data)
def __repr__(self):
return "%d.%d.%d.%d" % self.data
def _parse_ipv6(a):
l, _, r = a.partition("::")
l_groups = list(chain(*[divmod(int(x, 16), 256) for x in l.split(":") if x]))
r_groups = list(chain(*[divmod(int(x, 16), 256) for x in r.split(":") if x]))
zeros = [0] * (16 - len(l_groups) - len(r_groups))
return tuple(l_groups + zeros + r_groups)
def _format_ipv6(a):
left = []
right = []
current = "left"
for i in range(0, 16, 2):
group = (a[i] << 8) + a[i + 1]
if current == "left":
if group == 0 and i < 14:
if (a[i + 2] << 8) + a[i + 3] == 0:
current = "right"
else:
left.append("0")
else:
left.append("%x" % group)
else:
if group == 0 and len(right) == 0:
pass
else:
right.append("%x" % group)
if len(left) < 8:
return ":".join(left) + "::" + ":".join(right)
else:
return ":".join(left)
class AAAA(RD):
data = IP6("data")
@classmethod
def parse(cls, buffer, length):
try:
data = buffer.unpack("!16B")
return cls(data)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking AAAA [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, data) -> None:
if type(data) in (tuple, list):
self.data = tuple(data)
else:
self.data = _parse_ipv6(data)
def pack(self, buffer):
buffer.pack("!16B", *self.data)
def __repr__(self):
return _format_ipv6(self.data)
class CNAME(RD):
@classmethod
def parse(cls, buffer, length):
try:
label = buffer.decode_name()
return cls(label)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking CNAME [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, label=None) -> None:
self.label = label
def set_label(self, label):
if isinstance(label, DNSLabel):
self._label = label
else:
self._label = DNSLabel(label)
def get_label(self):
return self._label
label = property(get_label, set_label)
def pack(self, buffer):
buffer.encode_name(self.label)
def __repr__(self):
return "%s" % (self.label)
attrs = ("label",)
class PTR(CNAME):
pass
class SRV(RD):
priority = H("priority")
weight = H("weight")
port = H("port")
@classmethod
def parse(cls, buffer, length):
try:
priority, weight, port = buffer.unpack("!HHH")
target = buffer.decode_name()
return cls(priority, weight, port, target)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking SRV [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, priority=0, weight=0, port=0, target=None) -> None:
self.priority = priority
self.weight = weight
self.port = port
self.target = target
def set_target(self, target):
if isinstance(target, DNSLabel):
self._target = target
else:
self._target = DNSLabel(target)
def get_target(self):
return self._target
target = property(get_target, set_target)
def pack(self, buffer):
buffer.pack("!HHH", self.priority, self.weight, self.port)
buffer.encode_name(self.target)
def __repr__(self):
return "%d %d %d %s" % (self.priority, self.weight, self.port, self.target)
attrs = ("priority", "weight", "port", "target")
def decode_type_bitmap(type_bitmap):
rrlist = []
buf = DNSBuffer(type_bitmap)
while buf.remaining():
winnum, winlen = buf.unpack("BB")
bitmap = bytearray(buf.get(winlen))
for (pos, value) in enumerate(bitmap):
for i in range(8):
if (value << i) & 0x80:
bitpos = (256 * winnum) + (8 * pos) + i
rrlist.append(QTYPE[bitpos])
return rrlist
def encode_type_bitmap(rrlist):
rrlist = sorted([getattr(QTYPE, rr) for rr in rrlist])
buf = DNSBuffer()
curWindow = rrlist[0] // 256
bitmap = bytearray(32)
n = len(rrlist) - 1
for i, rr in enumerate(rrlist):
v = rr - curWindow * 256
bitmap[v // 8] |= 1 << (7 - v % 8)
if i == n or rrlist[i + 1] >= (curWindow + 1) * 256:
while bitmap[-1] == 0:
bitmap = bitmap[:-1]
buf.pack("BB", curWindow, len(bitmap))
buf.append(bitmap)
if i != n:
curWindow = rrlist[i + 1] // 256
bitmap = bytearray(32)
return buf.data
class NSEC(RD):
@classmethod
def parse(cls, buffer, length):
try:
end = buffer.offset + length
name = buffer.decode_name()
rrlist = decode_type_bitmap(buffer.get(end - buffer.offset))
return cls(name, rrlist)
except (BufferError, BimapError) as e:
raise DNSError("Error unpacking NSEC [offset=%d]: %s" % (buffer.offset, e))
def __init__(self, label, rrlist) -> None:
self.label = label
self.rrlist = rrlist
def set_label(self, label):
if isinstance(label, DNSLabel):
self._label = label
else:
self._label = DNSLabel(label)
def get_label(self):
return self._label
label = property(get_label, set_label)
def pack(self, buffer):
buffer.encode_name(self.label)
buffer.append(encode_type_bitmap(self.rrlist))
def __repr__(self):
return "%s %s" % (self.label, " ".join(self.rrlist))
attrs = ("label", "rrlist")
RDMAP = {"A": A, "AAAA": AAAA, "TXT": TXT, "PTR": PTR, "SRV": SRV, "NSEC": NSEC}

View File

@@ -0,0 +1,154 @@
# coding: utf-8
from __future__ import print_function
import re
from .bit import get_bits, set_bits
from .buffer import Buffer, BufferError
LDH = set(range(33, 127))
ESCAPE = re.compile(r"\\([0-9][0-9][0-9])")
class DNSLabelError(Exception):
pass
class DNSLabel(object):
def __init__(self, label):
if type(label) == DNSLabel:
self.label = label.label
elif type(label) in (list, tuple):
self.label = tuple(label)
else:
if not label or label in (b".", "."):
self.label = ()
elif type(label) is not bytes:
if type("") != type(b""):
label = ESCAPE.sub(lambda m: chr(int(m[1])), label)
self.label = tuple(label.encode("idna").rstrip(b".").split(b"."))
else:
if type("") == type(b""):
label = ESCAPE.sub(lambda m: chr(int(m.groups()[0])), label)
self.label = tuple(label.rstrip(b".").split(b"."))
def add(self, name):
new = DNSLabel(name)
if self.label:
new.label += self.label
return new
def idna(self):
return ".".join([s.decode("idna") for s in self.label]) + "."
def _decode(self, s):
if set(s).issubset(LDH):
return s.decode()
else:
return "".join([(chr(c) if (c in LDH) else "\\%03d" % c) for c in s])
def __str__(self):
return ".".join([self._decode(bytearray(s)) for s in self.label]) + "."
def __repr__(self):
return "<DNSLabel: '%s'>" % str(self)
def __hash__(self):
return hash(tuple(map(lambda x: x.lower(), self.label)))
def __ne__(self, other):
return not self == other
def __eq__(self, other):
if type(other) != DNSLabel:
return self.__eq__(DNSLabel(other))
else:
return [l.lower() for l in self.label] == [l.lower() for l in other.label]
def __len__(self):
return len(b".".join(self.label))
class DNSBuffer(Buffer):
def __init__(self, data=b""):
super(DNSBuffer, self).__init__(data)
self.names = {}
def decode_name(self, last=-1):
label = []
done = False
while not done:
(length,) = self.unpack("!B")
if get_bits(length, 6, 2) == 3:
self.offset -= 1
pointer = get_bits(self.unpack("!H")[0], 0, 14)
save = self.offset
if last == save:
raise BufferError(
"Recursive pointer in DNSLabel [offset=%d,pointer=%d,length=%d]"
% (self.offset, pointer, len(self.data))
)
if pointer < self.offset:
self.offset = pointer
else:
raise BufferError(
"Invalid pointer in DNSLabel [offset=%d,pointer=%d,length=%d]"
% (self.offset, pointer, len(self.data))
)
label.extend(self.decode_name(save).label)
self.offset = save
done = True
else:
if length > 0:
l = self.get(length)
try:
l.decode()
except UnicodeDecodeError:
raise BufferError("Invalid label <%s>" % l)
label.append(l)
else:
done = True
return DNSLabel(label)
def encode_name(self, name):
if not isinstance(name, DNSLabel):
name = DNSLabel(name)
if len(name) > 253:
raise DNSLabelError("Domain label too long: %r" % name)
name = list(name.label)
while name:
if tuple(name) in self.names:
pointer = self.names[tuple(name)]
pointer = set_bits(pointer, 3, 14, 2)
self.pack("!H", pointer)
return
else:
self.names[tuple(name)] = self.offset
element = name.pop(0)
if len(element) > 63:
raise DNSLabelError("Label component too long: %r" % element)
self.pack("!B", len(element))
self.append(element)
self.append(b"\x00")
def encode_name_nocompress(self, name):
if not isinstance(name, DNSLabel):
name = DNSLabel(name)
if len(name) > 253:
raise DNSLabelError("Domain label too long: %r" % name)
name = list(name.label)
while name:
element = name.pop(0)
if len(element) > 63:
raise DNSLabelError("Label component too long: %r" % element)
self.pack("!B", len(element))
self.append(element)
self.append(b"\x00")

View File

@@ -0,0 +1,105 @@
# coding: utf-8
from __future__ import print_function
import collections
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
class Lexer(object):
escape_chars = "\\"
escape = {"n": "\n", "t": "\t", "r": "\r"}
def __init__(self, f, debug=False):
if hasattr(f, "read"):
self.f = f
elif type(f) == str:
self.f = StringIO(f)
elif type(f) == bytes:
self.f = StringIO(f.decode())
else:
raise ValueError("Invalid input")
self.debug = debug
self.q = collections.deque()
self.state = self.lexStart
self.escaped = False
self.eof = False
def __iter__(self):
return self.parse()
def next_token(self):
if self.debug:
print("STATE", self.state)
(tok, self.state) = self.state()
return tok
def parse(self):
while self.state is not None and not self.eof:
tok = self.next_token()
if tok:
yield tok
def read(self, n=1):
s = ""
while self.q and n > 0:
s += self.q.popleft()
n -= 1
s += self.f.read(n)
if s == "":
self.eof = True
if self.debug:
print("Read: >%s<" % repr(s))
return s
def peek(self, n=1):
s = ""
i = 0
while len(self.q) > i and n > 0:
s += self.q[i]
i += 1
n -= 1
r = self.f.read(n)
if n > 0 and r == "":
self.eof = True
self.q.extend(r)
if self.debug:
print("Peek : >%s<" % repr(s + r))
return s + r
def pushback(self, s):
p = collections.deque(s)
p.extend(self.q)
self.q = p
def readescaped(self):
c = self.read(1)
if c in self.escape_chars:
self.escaped = True
n = self.peek(3)
if n.isdigit():
n = self.read(3)
if self.debug:
print("Escape: >%s<" % n)
return chr(int(n, 8))
elif n[0] in "x":
x = self.read(3)
if self.debug:
print("Escape: >%s<" % x)
return chr(int(x[1:], 16))
else:
c = self.read(1)
if self.debug:
print("Escape: >%s<" % c)
return self.escape.get(c, c)
else:
self.escaped = False
return c
def lexStart(self):
return (None, None)

View File

@@ -0,0 +1,81 @@
# coding: utf-8
import sys
if sys.version_info < (3,):
int_types = (
int,
long,
)
byte_types = (str, bytearray)
else:
int_types = (int,)
byte_types = (bytes, bytearray)
def check_instance(name, val, types):
if not isinstance(val, types):
raise ValueError(
"Attribute '%s' must be instance of %s [%s]" % (name, types, type(val))
)
def check_bytes(name, val):
return check_instance(name, val, byte_types)
def range_property(attr, min, max):
def getter(obj):
return getattr(obj, "_%s" % attr)
def setter(obj, val):
if isinstance(val, int_types) and min <= val <= max:
setattr(obj, "_%s" % attr, val)
else:
raise ValueError(
"Attribute '%s' must be between %d-%d [%s]" % (attr, min, max, val)
)
return property(getter, setter)
def B(attr):
return range_property(attr, 0, 255)
def H(attr):
return range_property(attr, 0, 65535)
def I(attr):
return range_property(attr, 0, 4294967295)
def ntuple_range(attr, n, min, max):
f = lambda x: isinstance(x, int_types) and min <= x <= max
def getter(obj):
return getattr(obj, "_%s" % attr)
def setter(obj, val):
if len(val) != n:
raise ValueError(
"Attribute '%s' must be tuple with %d elements [%s]" % (attr, n, val)
)
if all(map(f, val)):
setattr(obj, "_%s" % attr, val)
else:
raise ValueError(
"Attribute '%s' elements must be between %d-%d [%s]"
% (attr, min, max, val)
)
return property(getter, setter)
def IP4(attr):
return ntuple_range(attr, 4, 0, 255)
def IP6(attr):
return ntuple_range(attr, 16, 0, 255)

View File

@@ -0,0 +1,5 @@
`ifaddr` with py2.7 support enabled by make-sfx.sh which strips py3 hints using strip_hints and removes the `^if True:` blocks
L: BSD-2-Clause
Copyright (c) 2014 Stefan C. Mueller
https://github.com/pydron/ifaddr/

View File

@@ -0,0 +1,21 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
"""
L: BSD-2-Clause
Copyright (c) 2014 Stefan C. Mueller
https://github.com/pydron/ifaddr/tree/0.2.0
"""
import os
from ._shared import IP, Adapter
if os.name == "nt":
from ._win32 import get_adapters
elif os.name == "posix":
from ._posix import get_adapters
else:
raise RuntimeError("Unsupported Operating System: %s" % os.name)
__all__ = ["Adapter", "IP", "get_adapters"]

View File

@@ -0,0 +1,84 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import collections
import ctypes.util
import os
import socket
import ipaddress
if True: # pylint: disable=using-constant-test
from typing import Iterable, Optional
from . import _shared as shared
from ._shared import U
class ifaddrs(ctypes.Structure):
pass
ifaddrs._fields_ = [
("ifa_next", ctypes.POINTER(ifaddrs)),
("ifa_name", ctypes.c_char_p),
("ifa_flags", ctypes.c_uint),
("ifa_addr", ctypes.POINTER(shared.sockaddr)),
("ifa_netmask", ctypes.POINTER(shared.sockaddr)),
]
libc = ctypes.CDLL(ctypes.util.find_library("socket" if os.uname()[0] == "SunOS" else "c"), use_errno=True) # type: ignore
def get_adapters(include_unconfigured: bool = False) -> Iterable[shared.Adapter]:
addr0 = addr = ctypes.POINTER(ifaddrs)()
retval = libc.getifaddrs(ctypes.byref(addr))
if retval != 0:
eno = ctypes.get_errno()
raise OSError(eno, os.strerror(eno))
ips = collections.OrderedDict()
def add_ip(adapter_name: str, ip: Optional[shared.IP]) -> None:
if adapter_name not in ips:
index = None # type: Optional[int]
try:
# Mypy errors on this when the Windows CI runs:
# error: Module has no attribute "if_nametoindex"
index = socket.if_nametoindex(adapter_name) # type: ignore
except (OSError, AttributeError):
pass
ips[adapter_name] = shared.Adapter(
adapter_name, adapter_name, [], index=index
)
if ip is not None:
ips[adapter_name].ips.append(ip)
while addr:
name = addr[0].ifa_name.decode(encoding="UTF-8")
ip_addr = shared.sockaddr_to_ip(addr[0].ifa_addr)
if ip_addr:
if addr[0].ifa_netmask and not addr[0].ifa_netmask[0].sa_familiy:
addr[0].ifa_netmask[0].sa_familiy = addr[0].ifa_addr[0].sa_familiy
netmask = shared.sockaddr_to_ip(addr[0].ifa_netmask)
if isinstance(netmask, tuple):
netmaskStr = U(netmask[0])
prefixlen = shared.ipv6_prefixlength(ipaddress.IPv6Address(netmaskStr))
else:
if netmask is None:
t = "sockaddr_to_ip({}) returned None"
raise Exception(t.format(addr[0].ifa_netmask))
netmaskStr = U("0.0.0.0/" + netmask)
prefixlen = ipaddress.IPv4Network(netmaskStr).prefixlen
ip = shared.IP(ip_addr, prefixlen, name)
add_ip(name, ip)
else:
if include_unconfigured:
add_ip(name, None)
addr = addr[0].ifa_next
libc.freeifaddrs(addr0)
return ips.values()

View File

@@ -0,0 +1,203 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import ctypes
import platform
import socket
import sys
import ipaddress
if True: # pylint: disable=using-constant-test
from typing import Callable, List, Optional, Union
PY2 = sys.version_info < (3,)
if not PY2:
U: Callable[[str], str] = str
else:
U = unicode # noqa: F821 # pylint: disable=undefined-variable,self-assigning-variable
class Adapter(object):
"""
Represents a network interface device controller (NIC), such as a
network card. An adapter can have multiple IPs.
On Linux aliasing (multiple IPs per physical NIC) is implemented
by creating 'virtual' adapters, each represented by an instance
of this class. Each of those 'virtual' adapters can have both
a IPv4 and an IPv6 IP address.
"""
def __init__(
self, name: str, nice_name: str, ips: List["IP"], index: Optional[int] = None
) -> None:
#: Unique name that identifies the adapter in the system.
#: On Linux this is of the form of `eth0` or `eth0:1`, on
#: Windows it is a UUID in string representation, such as
#: `{846EE342-7039-11DE-9D20-806E6F6E6963}`.
self.name = name
#: Human readable name of the adpater. On Linux this
#: is currently the same as :attr:`name`. On Windows
#: this is the name of the device.
self.nice_name = nice_name
#: List of :class:`ifaddr.IP` instances in the order they were
#: reported by the system.
self.ips = ips
#: Adapter index as used by some API (e.g. IPv6 multicast group join).
self.index = index
def __repr__(self) -> str:
return "Adapter(name={name}, nice_name={nice_name}, ips={ips}, index={index})".format(
name=repr(self.name),
nice_name=repr(self.nice_name),
ips=repr(self.ips),
index=repr(self.index),
)
if True:
# Type of an IPv4 address (a string in "xxx.xxx.xxx.xxx" format)
_IPv4Address = str
# Type of an IPv6 address (a three-tuple `(ip, flowinfo, scope_id)`)
_IPv6Address = tuple[str, int, int]
class IP(object):
"""
Represents an IP address of an adapter.
"""
def __init__(
self, ip: Union[_IPv4Address, _IPv6Address], network_prefix: int, nice_name: str
) -> None:
#: IP address. For IPv4 addresses this is a string in
#: "xxx.xxx.xxx.xxx" format. For IPv6 addresses this
#: is a three-tuple `(ip, flowinfo, scope_id)`, where
#: `ip` is a string in the usual collon separated
#: hex format.
self.ip = ip
#: Number of bits of the IP that represent the
#: network. For a `255.255.255.0` netmask, this
#: number would be `24`.
self.network_prefix = network_prefix
#: Human readable name for this IP.
#: On Linux is this currently the same as the adapter name.
#: On Windows this is the name of the network connection
#: as configured in the system control panel.
self.nice_name = nice_name
@property
def is_IPv4(self) -> bool:
"""
Returns `True` if this IP is an IPv4 address and `False`
if it is an IPv6 address.
"""
return not isinstance(self.ip, tuple)
@property
def is_IPv6(self) -> bool:
"""
Returns `True` if this IP is an IPv6 address and `False`
if it is an IPv4 address.
"""
return isinstance(self.ip, tuple)
def __repr__(self) -> str:
return "IP(ip={ip}, network_prefix={network_prefix}, nice_name={nice_name})".format(
ip=repr(self.ip),
network_prefix=repr(self.network_prefix),
nice_name=repr(self.nice_name),
)
if platform.system() == "Darwin" or "BSD" in platform.system():
# BSD derived systems use marginally different structures
# than either Linux or Windows.
# I still keep it in `shared` since we can use
# both structures equally.
class sockaddr(ctypes.Structure):
_fields_ = [
("sa_len", ctypes.c_uint8),
("sa_familiy", ctypes.c_uint8),
("sa_data", ctypes.c_uint8 * 14),
]
class sockaddr_in(ctypes.Structure):
_fields_ = [
("sa_len", ctypes.c_uint8),
("sa_familiy", ctypes.c_uint8),
("sin_port", ctypes.c_uint16),
("sin_addr", ctypes.c_uint8 * 4),
("sin_zero", ctypes.c_uint8 * 8),
]
class sockaddr_in6(ctypes.Structure):
_fields_ = [
("sa_len", ctypes.c_uint8),
("sa_familiy", ctypes.c_uint8),
("sin6_port", ctypes.c_uint16),
("sin6_flowinfo", ctypes.c_uint32),
("sin6_addr", ctypes.c_uint8 * 16),
("sin6_scope_id", ctypes.c_uint32),
]
else:
class sockaddr(ctypes.Structure): # type: ignore
_fields_ = [("sa_familiy", ctypes.c_uint16), ("sa_data", ctypes.c_uint8 * 14)]
class sockaddr_in(ctypes.Structure): # type: ignore
_fields_ = [
("sin_familiy", ctypes.c_uint16),
("sin_port", ctypes.c_uint16),
("sin_addr", ctypes.c_uint8 * 4),
("sin_zero", ctypes.c_uint8 * 8),
]
class sockaddr_in6(ctypes.Structure): # type: ignore
_fields_ = [
("sin6_familiy", ctypes.c_uint16),
("sin6_port", ctypes.c_uint16),
("sin6_flowinfo", ctypes.c_uint32),
("sin6_addr", ctypes.c_uint8 * 16),
("sin6_scope_id", ctypes.c_uint32),
]
def sockaddr_to_ip(
sockaddr_ptr: "ctypes.pointer[sockaddr]",
) -> Optional[Union[_IPv4Address, _IPv6Address]]:
if sockaddr_ptr:
if sockaddr_ptr[0].sa_familiy == socket.AF_INET:
ipv4 = ctypes.cast(sockaddr_ptr, ctypes.POINTER(sockaddr_in))
ippacked = bytes(bytearray(ipv4[0].sin_addr))
ip = U(ipaddress.ip_address(ippacked))
return ip
elif sockaddr_ptr[0].sa_familiy == socket.AF_INET6:
ipv6 = ctypes.cast(sockaddr_ptr, ctypes.POINTER(sockaddr_in6))
flowinfo = ipv6[0].sin6_flowinfo
ippacked = bytes(bytearray(ipv6[0].sin6_addr))
ip = U(ipaddress.ip_address(ippacked))
scope_id = ipv6[0].sin6_scope_id
return (ip, flowinfo, scope_id)
return None
def ipv6_prefixlength(address: ipaddress.IPv6Address) -> int:
prefix_length = 0
for i in range(address.max_prefixlen):
if int(address) >> i & 1:
prefix_length = prefix_length + 1
return prefix_length

View File

@@ -0,0 +1,135 @@
# coding: utf-8
from __future__ import print_function, unicode_literals
import ctypes
from ctypes import wintypes
if True: # pylint: disable=using-constant-test
from typing import Iterable, List
from . import _shared as shared
NO_ERROR = 0
ERROR_BUFFER_OVERFLOW = 111
MAX_ADAPTER_NAME_LENGTH = 256
MAX_ADAPTER_DESCRIPTION_LENGTH = 128
MAX_ADAPTER_ADDRESS_LENGTH = 8
AF_UNSPEC = 0
class SOCKET_ADDRESS(ctypes.Structure):
_fields_ = [
("lpSockaddr", ctypes.POINTER(shared.sockaddr)),
("iSockaddrLength", wintypes.INT),
]
class IP_ADAPTER_UNICAST_ADDRESS(ctypes.Structure):
pass
IP_ADAPTER_UNICAST_ADDRESS._fields_ = [
("Length", wintypes.ULONG),
("Flags", wintypes.DWORD),
("Next", ctypes.POINTER(IP_ADAPTER_UNICAST_ADDRESS)),
("Address", SOCKET_ADDRESS),
("PrefixOrigin", ctypes.c_uint),
("SuffixOrigin", ctypes.c_uint),
("DadState", ctypes.c_uint),
("ValidLifetime", wintypes.ULONG),
("PreferredLifetime", wintypes.ULONG),
("LeaseLifetime", wintypes.ULONG),
("OnLinkPrefixLength", ctypes.c_uint8),
]
class IP_ADAPTER_ADDRESSES(ctypes.Structure):
pass
IP_ADAPTER_ADDRESSES._fields_ = [
("Length", wintypes.ULONG),
("IfIndex", wintypes.DWORD),
("Next", ctypes.POINTER(IP_ADAPTER_ADDRESSES)),
("AdapterName", ctypes.c_char_p),
("FirstUnicastAddress", ctypes.POINTER(IP_ADAPTER_UNICAST_ADDRESS)),
("FirstAnycastAddress", ctypes.c_void_p),
("FirstMulticastAddress", ctypes.c_void_p),
("FirstDnsServerAddress", ctypes.c_void_p),
("DnsSuffix", ctypes.c_wchar_p),
("Description", ctypes.c_wchar_p),
("FriendlyName", ctypes.c_wchar_p),
]
iphlpapi = ctypes.windll.LoadLibrary("Iphlpapi") # type: ignore
def enumerate_interfaces_of_adapter(
nice_name: str, address: IP_ADAPTER_UNICAST_ADDRESS
) -> Iterable[shared.IP]:
# Iterate through linked list and fill list
addresses = [] # type: List[IP_ADAPTER_UNICAST_ADDRESS]
while True:
addresses.append(address)
if not address.Next:
break
address = address.Next[0]
for address in addresses:
ip = shared.sockaddr_to_ip(address.Address.lpSockaddr)
if ip is None:
t = "sockaddr_to_ip({}) returned None"
raise Exception(t.format(address.Address.lpSockaddr))
network_prefix = address.OnLinkPrefixLength
yield shared.IP(ip, network_prefix, nice_name)
def get_adapters(include_unconfigured: bool = False) -> Iterable[shared.Adapter]:
# Call GetAdaptersAddresses() with error and buffer size handling
addressbuffersize = wintypes.ULONG(15 * 1024)
retval = ERROR_BUFFER_OVERFLOW
while retval == ERROR_BUFFER_OVERFLOW:
addressbuffer = ctypes.create_string_buffer(addressbuffersize.value)
retval = iphlpapi.GetAdaptersAddresses(
wintypes.ULONG(AF_UNSPEC),
wintypes.ULONG(0),
None,
ctypes.byref(addressbuffer),
ctypes.byref(addressbuffersize),
)
if retval != NO_ERROR:
raise ctypes.WinError() # type: ignore
# Iterate through adapters fill array
address_infos = [] # type: List[IP_ADAPTER_ADDRESSES]
address_info = IP_ADAPTER_ADDRESSES.from_buffer(addressbuffer)
while True:
address_infos.append(address_info)
if not address_info.Next:
break
address_info = address_info.Next[0]
# Iterate through unicast addresses
result = [] # type: List[shared.Adapter]
for adapter_info in address_infos:
# We don't expect non-ascii characters here, so encoding shouldn't matter
name = adapter_info.AdapterName.decode()
nice_name = adapter_info.Description
index = adapter_info.IfIndex
if adapter_info.FirstUnicastAddress:
ips = enumerate_interfaces_of_adapter(
adapter_info.FriendlyName, adapter_info.FirstUnicastAddress[0]
)
ips = list(ips)
result.append(shared.Adapter(name, nice_name, ips, index=index))
elif include_unconfigured:
result.append(shared.Adapter(name, nice_name, [], index=index))
return result

View File

@@ -0,0 +1,591 @@
# coding: utf-8
# modified copy of Project Nayuki's qrcodegen (MIT-licensed);
# https://github.com/nayuki/QR-Code-generator/blob/daa3114/python/qrcodegen.py
# the original ^ is extremely well commented so refer to that for explanations
# hacks: binary-only, auto-ecc, render, py2-compat
from __future__ import print_function, unicode_literals
import collections
import itertools
if True: # pylint: disable=using-constant-test
from collections.abc import Sequence
from typing import Callable, List, Optional, Tuple, Union
def num_char_count_bits(ver: int) -> int:
return 16 if (ver + 7) // 17 else 8
class Ecc(object):
ordinal: int
formatbits: int
def __init__(self, i: int, fb: int) -> None:
self.ordinal = i
self.formatbits = fb
LOW: "Ecc"
MEDIUM: "Ecc"
QUARTILE: "Ecc"
HIGH: "Ecc"
Ecc.LOW = Ecc(0, 1)
Ecc.MEDIUM = Ecc(1, 0)
Ecc.QUARTILE = Ecc(2, 3)
Ecc.HIGH = Ecc(3, 2)
class QrSegment(object):
@staticmethod
def make_seg(data: Union[bytes, Sequence[int]]) -> "QrSegment":
bb = _BitBuffer()
for b in data:
bb.append_bits(b, 8)
return QrSegment(len(data), bb)
numchars: int # num bytes, not the same as the data's bit length
bitdata: List[int] # The data bits of this segment
def __init__(self, numch: int, bitdata: Sequence[int]) -> None:
if numch < 0:
raise ValueError()
self.numchars = numch
self.bitdata = list(bitdata)
@staticmethod
def get_total_bits(segs: Sequence["QrSegment"], ver: int) -> Optional[int]:
result = 0
for seg in segs:
ccbits: int = num_char_count_bits(ver)
if seg.numchars >= (1 << ccbits):
return None # segment length doesn't fit the field's bit width
result += 4 + ccbits + len(seg.bitdata)
return result
class QrCode(object):
@staticmethod
def encode_binary(data: Union[bytes, Sequence[int]]) -> "QrCode":
return QrCode.encode_segments([QrSegment.make_seg(data)])
@staticmethod
def encode_segments(
segs: Sequence[QrSegment],
ecl: Ecc = Ecc.LOW,
minver: int = 2,
maxver: int = 40,
mask: int = -1,
) -> "QrCode":
for ver in range(minver, maxver + 1):
datacapacitybits: int = QrCode._get_num_data_codewords(ver, ecl) * 8
datausedbits: Optional[int] = QrSegment.get_total_bits(segs, ver)
if (datausedbits is not None) and (datausedbits <= datacapacitybits):
break
assert datausedbits
for newecl in (
Ecc.MEDIUM,
Ecc.QUARTILE,
Ecc.HIGH,
):
if datausedbits <= QrCode._get_num_data_codewords(ver, newecl) * 8:
ecl = newecl
# Concatenate all segments to create the data bit string
bb = _BitBuffer()
for seg in segs:
bb.append_bits(4, 4)
bb.append_bits(seg.numchars, num_char_count_bits(ver))
bb.extend(seg.bitdata)
assert len(bb) == datausedbits
# Add terminator and pad up to a byte if applicable
datacapacitybits = QrCode._get_num_data_codewords(ver, ecl) * 8
assert len(bb) <= datacapacitybits
bb.append_bits(0, min(4, datacapacitybits - len(bb)))
bb.append_bits(0, -len(bb) % 8)
assert len(bb) % 8 == 0
# Pad with alternating bytes until data capacity is reached
for padbyte in itertools.cycle((0xEC, 0x11)):
if len(bb) >= datacapacitybits:
break
bb.append_bits(padbyte, 8)
# Pack bits into bytes in big endian
datacodewords = bytearray([0] * (len(bb) // 8))
for (i, bit) in enumerate(bb):
datacodewords[i >> 3] |= bit << (7 - (i & 7))
return QrCode(ver, ecl, datacodewords, mask)
ver: int
size: int # w/h; 21..177 (ver * 4 + 17)
ecclvl: Ecc
mask: int # 0..7
modules: List[List[bool]]
unmaskable: List[List[bool]]
def __init__(
self,
ver: int,
ecclvl: Ecc,
datacodewords: Union[bytes, Sequence[int]],
msk: int,
) -> None:
self.ver = ver
self.size = ver * 4 + 17
self.ecclvl = ecclvl
self.modules = [[False] * self.size for _ in range(self.size)]
self.unmaskable = [[False] * self.size for _ in range(self.size)]
# Compute ECC, draw modules
self._draw_function_patterns()
allcodewords: bytes = self._add_ecc_and_interleave(bytearray(datacodewords))
self._draw_codewords(allcodewords)
if msk == -1: # automask
minpenalty: int = 1 << 32
for i in range(8):
self._apply_mask(i)
self._draw_format_bits(i)
penalty = self._get_penalty_score()
if penalty < minpenalty:
msk = i
minpenalty = penalty
self._apply_mask(i) # xor/undo
assert 0 <= msk <= 7
self.mask = msk
self._apply_mask(msk) # Apply the final choice of mask
self._draw_format_bits(msk) # Overwrite old format bits
def render(self, zoom=1, pad=4) -> str:
tab = self.modules
sz = self.size
if sz % 2 and zoom == 1:
tab.append([False] * sz)
tab = [[False] * sz] * pad + tab + [[False] * sz] * pad
tab = [[False] * pad + x + [False] * pad for x in tab]
rows: list[str] = []
if zoom == 1:
for y in range(0, len(tab), 2):
row = ""
for x in range(len(tab[y])):
v = 2 if tab[y][x] else 0
v += 1 if tab[y + 1][x] else 0
row += " ▄▀█"[v]
rows.append(row)
else:
for tr in tab:
row = ""
for zb in tr:
row += ""[int(zb)] * 2
rows.append(row)
return "\n".join(rows)
def _draw_function_patterns(self) -> None:
# Draw horizontal and vertical timing patterns
for i in range(self.size):
self._set_function_module(6, i, i % 2 == 0)
self._set_function_module(i, 6, i % 2 == 0)
# Draw 3 finder patterns (all corners except bottom right; overwrites some timing modules)
self._draw_finder_pattern(3, 3)
self._draw_finder_pattern(self.size - 4, 3)
self._draw_finder_pattern(3, self.size - 4)
# Draw numerous alignment patterns
alignpatpos: List[int] = self._get_alignment_pattern_positions()
numalign: int = len(alignpatpos)
skips: Sequence[Tuple[int, int]] = (
(0, 0),
(0, numalign - 1),
(numalign - 1, 0),
)
for i in range(numalign):
for j in range(numalign):
if (i, j) not in skips: # avoid finder corners
self._draw_alignment_pattern(alignpatpos[i], alignpatpos[j])
# draw config data with dummy mask value; ctor overwrites it
self._draw_format_bits(0)
self._draw_ver()
def _draw_format_bits(self, mask: int) -> None:
# Calculate error correction code and pack bits; ecclvl is uint2, mask is uint3
data: int = self.ecclvl.formatbits << 3 | mask
rem: int = data
for _ in range(10):
rem = (rem << 1) ^ ((rem >> 9) * 0x537)
bits: int = (data << 10 | rem) ^ 0x5412 # uint15
assert bits >> 15 == 0
# first copy
for i in range(0, 6):
self._set_function_module(8, i, _get_bit(bits, i))
self._set_function_module(8, 7, _get_bit(bits, 6))
self._set_function_module(8, 8, _get_bit(bits, 7))
self._set_function_module(7, 8, _get_bit(bits, 8))
for i in range(9, 15):
self._set_function_module(14 - i, 8, _get_bit(bits, i))
# second copy
for i in range(0, 8):
self._set_function_module(self.size - 1 - i, 8, _get_bit(bits, i))
for i in range(8, 15):
self._set_function_module(8, self.size - 15 + i, _get_bit(bits, i))
self._set_function_module(8, self.size - 8, True) # Always dark
def _draw_ver(self) -> None:
if self.ver < 7:
return
# Calculate error correction code and pack bits
rem: int = self.ver # ver is uint6, 7..40
for _ in range(12):
rem = (rem << 1) ^ ((rem >> 11) * 0x1F25)
bits: int = self.ver << 12 | rem # uint18
assert bits >> 18 == 0
# Draw two copies
for i in range(18):
bit: bool = _get_bit(bits, i)
a: int = self.size - 11 + i % 3
b: int = i // 3
self._set_function_module(a, b, bit)
self._set_function_module(b, a, bit)
def _draw_finder_pattern(self, x: int, y: int) -> None:
for dy in range(-4, 5):
for dx in range(-4, 5):
xx, yy = x + dx, y + dy
if (0 <= xx < self.size) and (0 <= yy < self.size):
# Chebyshev/infinity norm
self._set_function_module(
xx, yy, max(abs(dx), abs(dy)) not in (2, 4)
)
def _draw_alignment_pattern(self, x: int, y: int) -> None:
for dy in range(-2, 3):
for dx in range(-2, 3):
self._set_function_module(x + dx, y + dy, max(abs(dx), abs(dy)) != 1)
def _set_function_module(self, x: int, y: int, isdark: bool) -> None:
self.modules[y][x] = isdark
self.unmaskable[y][x] = True
def _add_ecc_and_interleave(self, data: bytearray) -> bytes:
ver: int = self.ver
assert len(data) == QrCode._get_num_data_codewords(ver, self.ecclvl)
# Calculate parameter numbers
numblocks: int = QrCode._NUM_ERROR_CORRECTION_BLOCKS[self.ecclvl.ordinal][ver]
blockecclen: int = QrCode._ECC_CODEWORDS_PER_BLOCK[self.ecclvl.ordinal][ver]
rawcodewords: int = QrCode._get_num_raw_data_modules(ver) // 8
numshortblocks: int = numblocks - rawcodewords % numblocks
shortblocklen: int = rawcodewords // numblocks
# Split data into blocks and append ECC to each block
blocks: List[bytes] = []
rsdiv: bytes = QrCode._reed_solomon_compute_divisor(blockecclen)
k: int = 0
for i in range(numblocks):
dat: bytearray = data[
k : k + shortblocklen - blockecclen + (0 if i < numshortblocks else 1)
]
k += len(dat)
ecc: bytes = QrCode._reed_solomon_compute_remainder(dat, rsdiv)
if i < numshortblocks:
dat.append(0)
blocks.append(dat + ecc)
assert k == len(data)
# Interleave (not concatenate) the bytes from every block into a single sequence
result = bytearray()
for i in range(len(blocks[0])):
for (j, blk) in enumerate(blocks):
# Skip the padding byte in short blocks
if (i != shortblocklen - blockecclen) or (j >= numshortblocks):
result.append(blk[i])
assert len(result) == rawcodewords
return result
def _draw_codewords(self, data: bytes) -> None:
assert len(data) == QrCode._get_num_raw_data_modules(self.ver) // 8
i: int = 0 # Bit index into the data
for right in range(self.size - 1, 0, -2):
# idx of right column in each column pair
if right <= 6:
right -= 1
for vert in range(self.size): # Vertical counter
for j in range(2):
x: int = right - j
upward: bool = (right + 1) & 2 == 0
y: int = (self.size - 1 - vert) if upward else vert
if (not self.unmaskable[y][x]) and (i < len(data) * 8):
self.modules[y][x] = _get_bit(data[i >> 3], 7 - (i & 7))
i += 1
# any remainder bits (0..7) were set 0/false/light by ctor
assert i == len(data) * 8
def _apply_mask(self, mask: int) -> None:
masker: Callable[[int, int], int] = QrCode._MASK_PATTERNS[mask]
for y in range(self.size):
for x in range(self.size):
self.modules[y][x] ^= (masker(x, y) == 0) and (
not self.unmaskable[y][x]
)
def _get_penalty_score(self) -> int:
result: int = 0
size: int = self.size
modules: List[List[bool]] = self.modules
# Adjacent modules in row having same color, and finder-like patterns
for y in range(size):
runcolor: bool = False
runx: int = 0
runhistory = collections.deque([0] * 7, 7)
for x in range(size):
if modules[y][x] == runcolor:
runx += 1
if runx == 5:
result += QrCode._PENALTY_N1
elif runx > 5:
result += 1
else:
self._finder_penalty_add_history(runx, runhistory)
if not runcolor:
result += (
self._finder_penalty_count_patterns(runhistory)
* QrCode._PENALTY_N3
)
runcolor = modules[y][x]
runx = 1
result += (
self._finder_penalty_terminate_and_count(runcolor, runx, runhistory)
* QrCode._PENALTY_N3
)
# Adjacent modules in column having same color, and finder-like patterns
for x in range(size):
runcolor = False
runy = 0
runhistory = collections.deque([0] * 7, 7)
for y in range(size):
if modules[y][x] == runcolor:
runy += 1
if runy == 5:
result += QrCode._PENALTY_N1
elif runy > 5:
result += 1
else:
self._finder_penalty_add_history(runy, runhistory)
if not runcolor:
result += (
self._finder_penalty_count_patterns(runhistory)
* QrCode._PENALTY_N3
)
runcolor = modules[y][x]
runy = 1
result += (
self._finder_penalty_terminate_and_count(runcolor, runy, runhistory)
* QrCode._PENALTY_N3
)
# 2*2 blocks of modules having same color
for y in range(size - 1):
for x in range(size - 1):
if (
modules[y][x]
== modules[y][x + 1]
== modules[y + 1][x]
== modules[y + 1][x + 1]
):
result += QrCode._PENALTY_N2
# Balance of dark and light modules
dark: int = sum((1 if cell else 0) for row in modules for cell in row)
total: int = size ** 2 # Note that size is odd, so dark/total != 1/2
# Compute the smallest integer k >= 0 such that (45-5k)% <= dark/total <= (55+5k)%
k: int = (abs(dark * 20 - total * 10) + total - 1) // total - 1
assert 0 <= k <= 9
result += k * QrCode._PENALTY_N4
assert 0 <= result <= 2568888
# ^ Non-tight upper bound based on default values of PENALTY_N1, ..., N4
return result
def _get_alignment_pattern_positions(self) -> List[int]:
ver: int = self.ver
if ver == 1:
return []
numalign: int = ver // 7 + 2
step: int = (
26
if (ver == 32)
else (ver * 4 + numalign * 2 + 1) // (numalign * 2 - 2) * 2
)
result: List[int] = [
(self.size - 7 - i * step) for i in range(numalign - 1)
] + [6]
return list(reversed(result))
@staticmethod
def _get_num_raw_data_modules(ver: int) -> int:
result: int = (16 * ver + 128) * ver + 64
if ver >= 2:
numalign: int = ver // 7 + 2
result -= (25 * numalign - 10) * numalign - 55
if ver >= 7:
result -= 36
assert 208 <= result <= 29648
return result
@staticmethod
def _get_num_data_codewords(ver: int, ecl: Ecc) -> int:
return (
QrCode._get_num_raw_data_modules(ver) // 8
- QrCode._ECC_CODEWORDS_PER_BLOCK[ecl.ordinal][ver]
* QrCode._NUM_ERROR_CORRECTION_BLOCKS[ecl.ordinal][ver]
)
@staticmethod
def _reed_solomon_compute_divisor(degree: int) -> bytes:
if not (1 <= degree <= 255):
raise ValueError("Degree out of range")
# Polynomial coefficients are stored from highest to lowest power, excluding the leading term which is always 1.
# For example the polynomial x^3 + 255x^2 + 8x + 93 is stored as the uint8 array [255, 8, 93].
result = bytearray([0] * (degree - 1) + [1]) # start with monomial x^0
# Compute the product polynomial (x - r^0) * (x - r^1) * (x - r^2) * ... * (x - r^{degree-1}),
# and drop the highest monomial term which is always 1x^degree.
# Note that r = 0x02, which is a generator element of this field GF(2^8/0x11D).
root: int = 1
for _ in range(degree):
# Multiply the current product by (x - r^i)
for j in range(degree):
result[j] = QrCode._reed_solomon_multiply(result[j], root)
if j + 1 < degree:
result[j] ^= result[j + 1]
root = QrCode._reed_solomon_multiply(root, 0x02)
return result
@staticmethod
def _reed_solomon_compute_remainder(data: bytes, divisor: bytes) -> bytes:
result = bytearray([0] * len(divisor))
for b in data: # Polynomial division
factor: int = b ^ result.pop(0)
result.append(0)
for (i, coef) in enumerate(divisor):
result[i] ^= QrCode._reed_solomon_multiply(coef, factor)
return result
@staticmethod
def _reed_solomon_multiply(x: int, y: int) -> int:
if (x >> 8 != 0) or (y >> 8 != 0):
raise ValueError("Byte out of range")
z: int = 0 # Russian peasant multiplication
for i in reversed(range(8)):
z = (z << 1) ^ ((z >> 7) * 0x11D)
z ^= ((y >> i) & 1) * x
assert z >> 8 == 0
return z
def _finder_penalty_count_patterns(self, runhistory: collections.deque[int]) -> int:
n: int = runhistory[1]
assert n <= self.size * 3
core: bool = (
n > 0
and (runhistory[2] == runhistory[4] == runhistory[5] == n)
and runhistory[3] == n * 3
)
return (
1 if (core and runhistory[0] >= n * 4 and runhistory[6] >= n) else 0
) + (1 if (core and runhistory[6] >= n * 4 and runhistory[0] >= n) else 0)
def _finder_penalty_terminate_and_count(
self,
currentruncolor: bool,
currentrunlength: int,
runhistory: collections.deque[int],
) -> int:
if currentruncolor: # Terminate dark run
self._finder_penalty_add_history(currentrunlength, runhistory)
currentrunlength = 0
currentrunlength += self.size # Add light border to final run
self._finder_penalty_add_history(currentrunlength, runhistory)
return self._finder_penalty_count_patterns(runhistory)
def _finder_penalty_add_history(
self, currentrunlength: int, runhistory: collections.deque[int]
) -> None:
if runhistory[0] == 0:
currentrunlength += self.size # Add light border to initial run
runhistory.appendleft(currentrunlength)
_PENALTY_N1: int = 3
_PENALTY_N2: int = 3
_PENALTY_N3: int = 40
_PENALTY_N4: int = 10
# fmt: off
_ECC_CODEWORDS_PER_BLOCK: Sequence[Sequence[int]] = (
(-1, 7, 10, 15, 20, 26, 18, 20, 24, 30, 18, 20, 24, 26, 30, 22, 24, 28, 30, 28, 28, 28, 28, 30, 30, 26, 28, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30), # noqa: E241 # L
(-1, 10, 16, 26, 18, 24, 16, 18, 22, 22, 26, 30, 22, 22, 24, 24, 28, 28, 26, 26, 26, 26, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28), # noqa: E241 # M
(-1, 13, 22, 18, 26, 18, 24, 18, 22, 20, 24, 28, 26, 24, 20, 30, 24, 28, 28, 26, 30, 28, 30, 30, 30, 30, 28, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30), # noqa: E241 # Q
(-1, 17, 28, 22, 16, 22, 28, 26, 26, 24, 28, 24, 28, 22, 24, 24, 30, 28, 28, 26, 28, 30, 24, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30)) # noqa: E241 # H
_NUM_ERROR_CORRECTION_BLOCKS: Sequence[Sequence[int]] = (
(-1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 4, 4, 4, 4, 4, 6, 6, 6, 6, 7, 8, 8, 9, 9, 10, 12, 12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 24, 25), # noqa: E241 # L
(-1, 1, 1, 1, 2, 2, 4, 4, 4, 5, 5, 5, 8, 9, 9, 10, 10, 11, 13, 14, 16, 17, 17, 18, 20, 21, 23, 25, 26, 28, 29, 31, 33, 35, 37, 38, 40, 43, 45, 47, 49), # noqa: E241 # M
(-1, 1, 1, 2, 2, 4, 4, 6, 6, 8, 8, 8, 10, 12, 16, 12, 17, 16, 18, 21, 20, 23, 23, 25, 27, 29, 34, 34, 35, 38, 40, 43, 45, 48, 51, 53, 56, 59, 62, 65, 68), # noqa: E241 # Q
(-1, 1, 1, 2, 4, 4, 4, 5, 6, 8, 8, 11, 11, 16, 16, 18, 16, 19, 21, 25, 25, 25, 34, 30, 32, 35, 37, 40, 42, 45, 48, 51, 54, 57, 60, 63, 66, 70, 74, 77, 81)) # noqa: E241 # H
# fmt: on
_MASK_PATTERNS: Sequence[Callable[[int, int], int]] = (
(lambda x, y: (x + y) % 2),
(lambda x, y: y % 2),
(lambda x, y: x % 3),
(lambda x, y: (x + y) % 3),
(lambda x, y: (x // 3 + y // 2) % 2),
(lambda x, y: x * y % 2 + x * y % 3),
(lambda x, y: (x * y % 2 + x * y % 3) % 2),
(lambda x, y: ((x + y) % 2 + x * y % 3) % 2),
)
class _BitBuffer(list): # type: ignore
def append_bits(self, val: int, n: int) -> None:
if (n < 0) or (val >> n != 0):
raise ValueError("Value out of range")
self.extend(((val >> i) & 1) for i in reversed(range(n)))
def _get_bit(x: int, i: int) -> bool:
return (x >> i) & 1 != 0
class DataTooLongError(ValueError):
pass

View File

@@ -16,28 +16,12 @@ import codecs
import platform
import sys
PY3 = sys.version_info[0] > 2
PY3 = sys.version_info > (3,)
WINDOWS = platform.system() == "Windows"
FS_ERRORS = "surrogateescape"
try:
if True: # pylint: disable=using-constant-test
from typing import Any
except:
pass
def u(text: Any) -> str:
if PY3:
return text
else:
return text.decode("unicode_escape")
def b(data: Any) -> bytes:
if PY3:
return data.encode("latin1")
else:
return data
if PY3:
@@ -171,9 +155,6 @@ def decodefilename(fn: bytes) -> str:
FS_ENCODING = sys.getfilesystemencoding()
# FS_ENCODING = "ascii"; fn = b("[abc\xff]"); encoded = u("[abc\udcff]")
# FS_ENCODING = 'cp932'; fn = b('[abc\x81\x00]'); encoded = u('[abc\udc81\x00]')
# FS_ENCODING = 'UTF-8'; fn = b('[abc\xff]'); encoded = u('[abc\udcff]')
if WINDOWS and not PY3:

View File

@@ -6,12 +6,10 @@ from datetime import datetime
from .bos import bos
try:
if True: # pylint: disable=using-constant-test
from typing import Any, Generator, Optional
from .util import NamedLogger
except:
pass
class StreamArc(object):
@@ -25,7 +23,7 @@ class StreamArc(object):
self.fgen = fgen
def gen(self) -> Generator[Optional[bytes], None, None]:
pass
raise Exception("override me")
def errdesc(errors: list[tuple[str, str]]) -> tuple[dict[str, Any], list[str]]:

View File

@@ -4,7 +4,9 @@ from __future__ import print_function, unicode_literals
import argparse
import base64
import calendar
import errno
import gzip
import logging
import os
import re
import shlex
@@ -16,31 +18,46 @@ import threading
import time
from datetime import datetime, timedelta
try:
# from inspect import currentframe
# print(currentframe().f_lineno)
if True: # pylint: disable=using-constant-test
from types import FrameType
import typing
from typing import Any, Optional, Union
except:
pass
from .__init__ import ANYWIN, MACOS, PY2, VT100, WINDOWS, EnvParams, unicode
from .__init__ import ANYWIN, MACOS, TYPE_CHECKING, VT100, EnvParams, unicode
from .authsrv import AuthSrv
from .mtag import HAVE_FFMPEG, HAVE_FFPROBE
from .tcpsrv import TcpSrv
from .th_srv import HAVE_PIL, HAVE_VIPS, HAVE_WEBP, ThumbSrv
from .up2k import Up2k
from .util import (
FFMPEG_URL,
VERSIONS,
Daemon,
Garda,
HLog,
HMaccas,
alltrace,
ansi_re,
is_exe,
min_ex,
mp,
pybin,
start_log_thrs,
start_stackmon,
)
if TYPE_CHECKING:
try:
from .mdns import MDNS
from .ssdp import SSDPd
except:
pass
class SvcHub(object):
"""
@@ -53,8 +70,15 @@ class SvcHub(object):
put() can return a queue (if want_reply=True) which has a blocking get() with the response.
"""
def __init__(self, args: argparse.Namespace, argv: list[str], printed: str) -> None:
def __init__(
self,
args: argparse.Namespace,
dargs: argparse.Namespace,
argv: list[str],
printed: str,
) -> None:
self.args = args
self.dargs = dargs
self.argv = argv
self.E: EnvParams = args.E
self.logf: Optional[typing.TextIO] = None
@@ -75,15 +99,21 @@ class SvcHub(object):
self.iphash = HMaccas(os.path.join(self.E.cfg, "iphash"), 8)
# for non-http clients (ftp)
self.bans: dict[str, int] = {}
self.gpwd = Garda(self.args.ban_pw)
self.g404 = Garda(self.args.ban_404)
if args.sss or args.s >= 3:
args.ss = True
args.no_dav = True
args.no_logues = True
args.no_readme = True
args.lo = args.lo or "cpp-%Y-%m%d-%H%M%S.txt.xz"
args.ls = args.ls or "**,*,ln,p,r"
if args.ss or args.s >= 2:
args.s = True
args.no_logues = True
args.no_readme = True
args.unpost = 0
args.no_del = True
args.no_mv = True
@@ -103,6 +133,11 @@ class SvcHub(object):
if args.lo:
self._setup_logfile(printed)
lg = logging.getLogger()
lh = HLog(self.log)
lg.handlers = [lh]
lg.setLevel(logging.DEBUG)
if args.stackmon:
start_stackmon(args.stackmon, 0)
@@ -126,17 +161,29 @@ class SvcHub(object):
ch = "abcdefghijklmnopqrstuvwx"[int(args.theme / 2)]
args.theme = "{0}{1} {0} {1}".format(ch, bri)
if not args.hardlink and args.never_symlink:
args.no_dedup = True
if args.log_fk:
args.log_fk = re.compile(args.log_fk)
# initiate all services to manage
self.asrv = AuthSrv(self.args, self.log)
self.asrv = AuthSrv(self.args, self.log, dargs=self.dargs)
if args.cgen:
self.asrv.cgen()
if args.exit == "cfg":
sys.exit(0)
if args.ls:
self.asrv.dbg_ls()
if not ANYWIN:
self._setlimits()
self.log("root", "max clients: {}".format(self.args.nc))
if not self._process_config():
raise Exception("bad config")
self.tcpsrv = TcpSrv(self)
self.up2k = Up2k(self)
@@ -150,6 +197,7 @@ class SvcHub(object):
self.args.th_dec = list(decs.keys())
self.thumbsrv = None
want_ff = False
if not args.no_thumb:
t = ", ".join(self.args.th_dec) or "(None available)"
self.log("thumb", "decoder preference: {}".format(t))
@@ -161,8 +209,12 @@ class SvcHub(object):
if self.args.th_dec:
self.thumbsrv = ThumbSrv(self)
else:
want_ff = True
msg = "need either Pillow, pyvips, or FFmpeg to create thumbnails; for example:\n{0}{1} -m pip install --user Pillow\n{0}{1} -m pip install --user pyvips\n{0}apt install ffmpeg"
msg = msg.format(" " * 37, os.path.basename(sys.executable))
msg = msg.format(" " * 37, os.path.basename(pybin))
if is_exe:
msg = "copyparty.exe cannot use Pillow or pyvips; need ffprobe.exe and ffmpeg.exe to create thumbnails"
self.log("thumb", msg, c=3)
if not args.no_acode and args.no_thumb:
@@ -174,13 +226,43 @@ class SvcHub(object):
msg = "setting --no-acode because either FFmpeg or FFprobe is not available"
self.log("thumb", msg, c=6)
args.no_acode = True
want_ff = True
if want_ff and ANYWIN:
self.log("thumb", "download FFmpeg to fix it:\033[0m " + FFMPEG_URL, 3)
args.th_poke = min(args.th_poke, args.th_maxage, args.ac_maxage)
zms = ""
if not args.https_only:
zms += "d"
if not args.http_only:
zms += "D"
if args.ftp or args.ftps:
from .ftpd import Ftpd
self.ftpd = Ftpd(self)
zms += "f" if args.ftp else "F"
if args.smb:
# impacket.dcerpc is noisy about listen timeouts
sto = socket.getdefaulttimeout()
socket.setdefaulttimeout(None)
from .smbd import SMB
self.smbd = SMB(self)
socket.setdefaulttimeout(sto)
self.smbd.start()
zms += "s"
if not args.zms:
args.zms = zms
self.zc_ngen = 0
self.mdns: Optional["MDNS"] = None
self.ssdp: Optional["SSDPd"] = None
# decide which worker impl to use
if self.check_mp_enable():
@@ -222,12 +304,92 @@ class SvcHub(object):
return
time.sleep(0.1) # purely cosmetic dw
self.log("root", "workers OK\n")
if self.tcpsrv.qr:
self.log("qr-code", self.tcpsrv.qr)
else:
self.log("root", "workers OK\n")
self.up2k.init_vols()
thr = threading.Thread(target=self.sd_notify, name="sd-notify")
thr.daemon = True
thr.start()
Daemon(self.sd_notify, "sd-notify")
def _process_config(self) -> bool:
al = self.args
al.zm_on = al.zm_on or al.z_on
al.zs_on = al.zs_on or al.z_on
al.zm_off = al.zm_off or al.z_off
al.zs_off = al.zs_off or al.z_off
ns = "zm_on zm_off zs_on zs_off acao acam"
for n in ns.split(" "):
vs = getattr(al, n).split(",")
vs = [x.strip() for x in vs]
vs = [x for x in vs if x]
setattr(al, n, vs)
ns = "acao acam"
for n in ns.split(" "):
vs = getattr(al, n)
vd = {zs: 1 for zs in vs}
setattr(al, n, vd)
ns = "acao"
for n in ns.split(" "):
vs = getattr(al, n)
vs = [x.lower() for x in vs]
setattr(al, n, vs)
R = al.rp_loc
if "//" in R or ":" in R:
t = "found URL in --rp-loc; it should be just the location, for example /foo/bar"
raise Exception(t)
al.R = R = R.strip("/")
al.SR = "/" + R if R else ""
al.RS = R + "/" if R else ""
al.SRS = "/" + R + "/" if R else "/"
return True
def _setlimits(self) -> None:
try:
import resource
soft, hard = [
x if x > 0 else 1024 * 1024
for x in list(resource.getrlimit(resource.RLIMIT_NOFILE))
]
except:
self.log("root", "failed to read rlimits from os", 6)
return
if not soft or not hard:
t = "got bogus rlimits from os ({}, {})"
self.log("root", t.format(soft, hard), 6)
return
want = self.args.nc * 4
new_soft = min(hard, want)
if new_soft < soft:
return
# t = "requesting rlimit_nofile({}), have {}"
# self.log("root", t.format(new_soft, soft), 6)
try:
import resource
resource.setrlimit(resource.RLIMIT_NOFILE, (new_soft, hard))
soft = new_soft
except:
t = "rlimit denied; max open files: {}"
self.log("root", t.format(soft), 3)
return
if soft < want:
t = "max open files: {} (wanted {} for -nc {})"
self.log("root", t.format(soft, want, self.args.nc), 3)
self.args.nc = min(self.args.nc, soft // 2)
def _logname(self) -> str:
dt = datetime.utcnow()
@@ -252,16 +414,18 @@ class SvcHub(object):
fn = sel_fn
try:
import lzma
lh = lzma.open(fn, "wt", encoding="utf-8", errors="replace", preset=0)
if fn.lower().endswith(".xz"):
import lzma
lh = lzma.open(fn, "wt", encoding="utf-8", errors="replace", preset=0)
else:
lh = open(fn, "wt", encoding="utf-8", errors="replace")
except:
import codecs
lh = codecs.open(fn, "w", encoding="utf-8", errors="replace")
argv = [sys.executable] + self.argv
argv = [pybin] + self.argv
if hasattr(shlex, "quote"):
argv = [shlex.quote(x) for x in argv]
else:
@@ -269,17 +433,20 @@ class SvcHub(object):
msg = "[+] opened logfile [{}]\n".format(fn)
printed += msg
lh.write("t0: {:.3f}\nargv: {}\n\n{}".format(self.E.t0, " ".join(argv), printed))
t = "t0: {:.3f}\nargv: {}\n\n{}"
lh.write(t.format(self.E.t0, " ".join(argv), printed))
self.logf = lh
self.logf_base_fn = base_fn
print(msg, end="")
def run(self) -> None:
self.tcpsrv.run()
if getattr(self.args, "z_chk", 0) and (
getattr(self.args, "zm", False) or getattr(self.args, "zs", False)
):
Daemon(self.tcpsrv.netmon, "netmon")
thr = threading.Thread(target=self.thr_httpsrv_up, name="sig-hsrv-up2")
thr.daemon = True
thr.start()
Daemon(self.thr_httpsrv_up, "sig-hsrv-up2")
sigs = [signal.SIGINT, signal.SIGTERM]
if not ANYWIN:
@@ -294,9 +461,7 @@ class SvcHub(object):
# never lucky
if ANYWIN:
# msys-python probably fine but >msys-python
thr = threading.Thread(target=self.stop_thr, name="svchub-sig")
thr.daemon = True
thr.start()
Daemon(self.stop_thr, "svchub-sig")
try:
while not self.stop_req:
@@ -311,14 +476,39 @@ class SvcHub(object):
else:
self.stop_thr()
def start_zeroconf(self) -> None:
self.zc_ngen += 1
if getattr(self.args, "zm", False):
try:
from .mdns import MDNS
if self.mdns:
self.mdns.stop(True)
self.mdns = MDNS(self, self.zc_ngen)
Daemon(self.mdns.run, "mdns")
except:
self.log("root", "mdns startup failed;\n" + min_ex(), 3)
if getattr(self.args, "zs", False):
try:
from .ssdp import SSDPd
if self.ssdp:
self.ssdp.stop()
self.ssdp = SSDPd(self, self.zc_ngen)
Daemon(self.ssdp.run, "ssdp")
except:
self.log("root", "ssdp startup failed;\n" + min_ex(), 3)
def reload(self) -> str:
if self.reloading:
return "cannot reload; already in progress"
self.reloading = True
t = threading.Thread(target=self._reload, name="reloading")
t.daemon = True
t.start()
Daemon(self._reload, "reloading")
return "reload initiated"
def _reload(self) -> None:
@@ -341,6 +531,17 @@ class SvcHub(object):
self.shutdown()
def kill9(self, delay: float = 0.0) -> None:
if delay > 0.01:
time.sleep(delay)
print("component stuck; issuing sigkill")
time.sleep(0.1)
if ANYWIN:
os.system("taskkill /f /pid {}".format(os.getpid()))
else:
os.kill(os.getpid(), signal.SIGKILL)
def signal_handler(self, sig: int, frame: Optional[FrameType]) -> None:
if self.stopping:
if self.nsigs <= 0:
@@ -350,10 +551,7 @@ class SvcHub(object):
except:
pass
if ANYWIN:
os.system("taskkill /f /pid {}".format(os.getpid()))
else:
os.kill(os.getpid(), signal.SIGKILL)
self.kill9()
else:
self.nsigs -= 1
return
@@ -380,8 +578,18 @@ class SvcHub(object):
ret = 1
try:
self.pr("OPYTHAT")
self.tcpsrv.shutdown()
slp = 0.0
if self.mdns:
Daemon(self.mdns.stop)
slp = time.time() + 0.5
if self.ssdp:
Daemon(self.ssdp.stop)
slp = time.time() + 0.5
self.broker.shutdown()
self.tcpsrv.shutdown()
self.up2k.shutdown()
if self.thumbsrv:
self.thumbsrv.shutdown()
@@ -394,6 +602,14 @@ class SvcHub(object):
if n == 3:
self.pr("waiting for thumbsrv (10sec)...")
if hasattr(self, "smbd"):
slp = max(slp, time.time() + 0.5)
Daemon(self.kill9, a=(1,))
Daemon(self.smbd.stop)
while time.time() < slp:
time.sleep(0.1)
self.pr("nailed it", end="")
ret = self.retcode
except:
@@ -417,7 +633,7 @@ class SvcHub(object):
with self.log_mutex:
ts = datetime.utcnow().strftime("%Y-%m%d-%H%M%S.%f")[:-3]
self.logf.write("@{} [{}] {}\n".format(ts, src, msg))
self.logf.write("@{} [{}\033[0m] {}\n".format(ts, src, msg))
now = time.time()
if now >= self.next_day:
@@ -471,26 +687,26 @@ class SvcHub(object):
print(msg.encode("utf-8", "replace").decode(), end="")
except:
print(msg.encode("ascii", "replace").decode(), end="")
except OSError as ex:
if ex.errno != errno.EPIPE:
raise
if self.logf:
self.logf.write(msg)
def pr(self, *a: Any, **ka: Any) -> None:
with self.log_mutex:
print(*a, **ka)
try:
with self.log_mutex:
print(*a, **ka)
except OSError as ex:
if ex.errno != errno.EPIPE:
raise
def check_mp_support(self) -> str:
vmin = sys.version_info[1]
if WINDOWS:
msg = "need python 3.3 or newer for multiprocessing;"
if PY2 or vmin < 3:
return msg
elif MACOS:
if MACOS:
return "multiprocessing is wonky on mac osx;"
else:
msg = "need python 3.3+ for multiprocessing;"
if PY2 or vmin < 3:
return msg
elif sys.version_info < (3, 3):
return "need python 3.3 or newer for multiprocessing;"
try:
x: mp.Queue[tuple[str, str]] = mp.Queue(1)

View File

@@ -2,6 +2,7 @@
from __future__ import print_function, unicode_literals
import calendar
import stat
import time
import zlib
@@ -9,12 +10,10 @@ from .bos import bos
from .sutil import StreamArc, errdesc
from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile
try:
if True: # pylint: disable=using-constant-test
from typing import Any, Generator, Optional
from .util import NamedLogger
except:
pass
def dostime2unix(buf: bytes) -> int:
@@ -240,6 +239,9 @@ class StreamZip(StreamArc):
src = f["ap"]
st = f["st"]
if stat.S_ISDIR(st.st_mode):
return
sz = st.st_size
ts = st.st_mtime
@@ -271,6 +273,7 @@ class StreamZip(StreamArc):
yield self._ct(buf)
def gen(self) -> Generator[bytes, None, None]:
errf: dict[str, Any] = {}
errors = []
try:
for f in self.fgen:
@@ -311,5 +314,5 @@ class StreamZip(StreamArc):
ecdr, _ = gen_ecdr(self.items, cdir_pos, cdir_end)
yield self._ct(ecdr)
finally:
if errors:
if errf:
bos.unlink(errf["ap"])

View File

@@ -5,13 +5,33 @@ import os
import re
import socket
import sys
import time
from .__init__ import ANYWIN, MACOS, TYPE_CHECKING, unicode
from .util import chkcmd
from .__init__ import ANYWIN, PY2, TYPE_CHECKING, VT100, unicode
from .stolen.qrcodegen import QrCode
from .util import (
E_ACCESS,
E_ADDR_IN_USE,
E_ADDR_NOT_AVAIL,
E_UNREACH,
Netdev,
min_ex,
sunpack,
termsize,
)
if True:
from typing import Generator
if TYPE_CHECKING:
from .svchub import SvcHub
if not hasattr(socket, "IPPROTO_IPV6"):
setattr(socket, "IPPROTO_IPV6", 41)
if not hasattr(socket, "IP_FREEBIND"):
setattr(socket, "IP_FREEBIND", 15)
class TcpSrv(object):
"""
@@ -29,44 +49,114 @@ class TcpSrv(object):
self.stopping = False
self.srv: list[socket.socket] = []
self.bound: list[tuple[str, int]] = []
self.netdevs: dict[str, Netdev] = {}
self.netlist = ""
self.nsrv = 0
self.qr = ""
pad = False
ok: dict[str, list[int]] = {}
for ip in self.args.i:
ok[ip] = []
if ip == "::":
if socket.has_ipv6:
ips = ["::", "0.0.0.0"]
dual = True
else:
ips = ["0.0.0.0"]
dual = False
else:
ips = [ip]
dual = False
for ipa in ips:
ok[ipa] = []
for port in self.args.p:
self.nsrv += 1
successful_binds = 0
try:
self._listen(ip, port)
ok[ip].append(port)
for ipa in ips:
try:
self._listen(ipa, port)
ok[ipa].append(port)
successful_binds += 1
except:
if dual and ":" in ipa:
t = "listen on IPv6 [{}] failed; trying IPv4 {}...\n{}"
self.log("tcpsrv", t.format(ipa, ips[1], min_ex()), 3)
pad = True
continue
# binding 0.0.0.0 after :: fails on dualstack
# but is necessary on non-dualstakc
if successful_binds:
continue
raise
except Exception as ex:
if self.args.ign_ebind or self.args.ign_ebind_all:
t = "could not listen on {}:{}: {}"
self.log("tcpsrv", t.format(ip, port, ex), c=3)
pad = True
else:
raise
if not self.srv and not self.args.ign_ebind_all:
raise Exception("could not listen on any of the given interfaces")
if self.nsrv != len(self.srv):
if pad:
self.log("tcpsrv", "")
ip = "127.0.0.1"
eps = {ip: "local only"}
nonlocals = [x for x in self.args.i if x != ip]
eps = {
"127.0.0.1": Netdev("127.0.0.1", 0, "", "local only"),
"::1": Netdev("::1", 0, "", "local only"),
}
nonlocals = [x for x in self.args.i if x not in [k.split("/")[0] for k in eps]]
if nonlocals:
eps = self.detect_interfaces(self.args.i)
try:
self.netdevs = self.detect_interfaces(self.args.i)
except:
t = "failed to discover server IP addresses\n"
self.log("tcpsrv", t + min_ex(), 3)
self.netdevs = {}
eps.update({k.split("/")[0]: v for k, v in self.netdevs.items()})
if not eps:
for x in nonlocals:
eps[x] = "external"
eps[x] = Netdev(x, 0, "", "external")
else:
self.netdevs = {}
# keep IPv6 LL-only nics
ll_ok: set[str] = set()
for ip, nd in self.netdevs.items():
if not ip.startswith("fe80"):
continue
just_ll = True
for ip2, nd2 in self.netdevs.items():
if nd == nd2 and ":" in ip2 and not ip2.startswith("fe80"):
just_ll = False
if just_ll or self.args.ll:
ll_ok.add(ip.split("/")[0])
qr1: dict[str, list[int]] = {}
qr2: dict[str, list[int]] = {}
msgs = []
title_tab: dict[str, dict[str, int]] = {}
title_vars = [x[1:] for x in self.args.wintitle.split(" ") if x.startswith("$")]
t = "available @ {}://{}:{}/ (\033[33m{}\033[0m)"
for ip, desc in sorted(eps.items(), key=lambda x: x[1]):
if ip.startswith("fe80") and ip not in ll_ok:
continue
for port in sorted(self.args.p):
if port not in ok.get(ip, ok.get("0.0.0.0", [])):
if (
port not in ok.get(ip, [])
and port not in ok.get("::", [])
and port not in ok.get("0.0.0.0", [])
):
continue
proto = " http"
@@ -75,7 +165,15 @@ class TcpSrv(object):
elif self.args.https_only or port == 443:
proto = "https"
msgs.append(t.format(proto, ip, port, desc))
hip = "[{}]".format(ip) if ":" in ip else ip
msgs.append(t.format(proto, hip, port, desc))
is_ext = "external" in unicode(desc)
qrt = qr1 if is_ext else qr2
try:
qrt[ip].append(port)
except:
qrt[ip] = [port]
if not self.args.wintitle:
continue
@@ -86,7 +184,7 @@ class TcpSrv(object):
ep = "{}:{}".format(ip, port)
hits = []
if "pub" in title_vars and "external" in unicode(desc):
if "pub" in title_vars and is_ext:
hits.append(("pub", ep))
if "pub" in title_vars or "all" in title_vars:
@@ -103,42 +201,101 @@ class TcpSrv(object):
title_tab[tk] = {tv: 1}
if msgs:
msgs[-1] += "\n"
for t in msgs:
self.log("tcpsrv", t)
if self.args.wintitle:
self._set_wintitle(title_tab)
else:
print("\n", end="")
if self.args.qr or self.args.qrs:
self.qr = self._qr(qr1, qr2)
def _listen(self, ip: str, port: int) -> None:
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
ipv = socket.AF_INET6 if ":" in ip else socket.AF_INET
srv = socket.socket(ipv, socket.SOCK_STREAM)
if not ANYWIN or self.args.reuseaddr:
srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
srv.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
srv.settimeout(None) # < does not inherit, ^ does
srv.settimeout(None) # < does not inherit, ^ opts above do
try:
srv.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, False)
except:
pass # will create another ipv4 socket instead
if not ANYWIN and self.args.freebind:
srv.setsockopt(socket.SOL_IP, socket.IP_FREEBIND, 1)
try:
srv.bind((ip, port))
sport = srv.getsockname()[1]
if port != sport:
# linux 6.0.16 lets you bind a port which is in use
# except it just gives you a random port instead
raise OSError(E_ADDR_IN_USE[0], "")
self.srv.append(srv)
except (OSError, socket.error) as ex:
if ex.errno in [98, 48]:
if ex.errno in E_ADDR_IN_USE:
e = "\033[1;31mport {} is busy on interface {}\033[0m".format(port, ip)
elif ex.errno in [99, 49]:
elif ex.errno in E_ADDR_NOT_AVAIL:
e = "\033[1;31minterface {} does not exist\033[0m".format(ip)
else:
raise
raise Exception(e)
def run(self) -> None:
all_eps = [x.getsockname()[:2] for x in self.srv]
bound: list[tuple[str, int]] = []
srvs: list[socket.socket] = []
for srv in self.srv:
srv.listen(self.args.nc)
ip, port = srv.getsockname()
ip, port = srv.getsockname()[:2]
try:
srv.listen(self.args.nc)
try:
ok = srv.getsockopt(socket.SOL_SOCKET, socket.SO_ACCEPTCONN)
except:
ok = 1 # macos
if not ok:
# some linux don't throw on listen(0.0.0.0) after listen(::)
raise Exception("failed to listen on {}".format(srv.getsockname()))
except:
if ip == "0.0.0.0" and ("::", port) in bound:
# dualstack
srv.close()
continue
if ip == "::" and ("0.0.0.0", port) in all_eps:
# no ipv6
srv.close()
continue
raise
bound.append((ip, port))
srvs.append(srv)
fno = srv.fileno()
msg = "listening @ {}:{} f{} p{}".format(ip, port, fno, os.getpid())
hip = "[{}]".format(ip) if ":" in ip else ip
msg = "listening @ {}:{} f{} p{}".format(hip, port, fno, os.getpid())
self.log("tcpsrv", msg)
if self.args.q:
print(msg)
self.hub.broker.say("listen", srv)
self.srv = srvs
self.bound = bound
self.nsrv = len(srvs)
self._distribute_netdevs()
def _distribute_netdevs(self):
self.hub.broker.say("set_netdevs", self.netdevs)
self.hub.start_zeroconf()
def shutdown(self) -> None:
self.stopping = True
try:
@@ -149,180 +306,111 @@ class TcpSrv(object):
self.log("tcpsrv", "ok bye")
def ips_linux_ifconfig(self) -> dict[str, str]:
# for termux
try:
txt, _ = chkcmd(["ifconfig"])
except:
return {}
eps: dict[str, str] = {}
dev = None
ip = None
up = None
for ln in (txt + "\n").split("\n"):
if not ln.strip() and dev and ip:
eps[ip] = dev + ("" if up else ", \033[31mLINK-DOWN")
dev = ip = up = None
def netmon(self):
while not self.stopping:
time.sleep(self.args.z_chk)
netdevs = self.detect_interfaces(self.args.i)
if not netdevs:
continue
if ln == ln.lstrip():
dev = re.split(r"[: ]", ln)[0]
added = "nothing"
removed = "nothing"
for k, v in netdevs.items():
if k not in self.netdevs:
added = "{} = {}".format(k, v)
for k, v in self.netdevs.items():
if k not in netdevs:
removed = "{} = {}".format(k, v)
if "UP" in re.split(r"[<>, \t]", ln):
up = True
t = "network change detected:\n added {}\nremoved {}"
self.log("tcpsrv", t.format(added, removed), 3)
self.netdevs = netdevs
self._distribute_netdevs()
m = re.match(r"^\s+inet\s+([^ ]+)", ln)
if m:
ip = m.group(1)
def detect_interfaces(self, listen_ips: list[str]) -> dict[str, Netdev]:
from .stolen.ifaddr import get_adapters
nics = get_adapters(True)
eps: dict[str, Netdev] = {}
for nic in nics:
for nip in nic.ips:
ipa = nip.ip[0] if ":" in str(nip.ip) else nip.ip
sip = "{}/{}".format(ipa, nip.network_prefix)
nd = Netdev(sip, nic.index or 0, nic.nice_name, "")
eps[sip] = nd
try:
idx = socket.if_nametoindex(nd.name)
if idx and idx != nd.idx:
t = "netdev idx mismatch; ifaddr={} cpython={}"
self.log("tcpsrv", t.format(nd.idx, idx), 3)
nd.idx = idx
except:
pass
netlist = str(sorted(eps.items()))
if netlist == self.netlist and self.netdevs:
return {}
self.netlist = netlist
if "0.0.0.0" not in listen_ips and "::" not in listen_ips:
eps = {k: v for k, v in eps.items() if k.split("/")[0] in listen_ips}
try:
ext_devs = list(self._extdevs_nix())
ext_ips = [k for k, v in eps.items() if v.name in ext_devs]
ext_ips = [x.split("/")[0] for x in ext_ips]
if not ext_ips:
raise Exception()
except:
rt = self._defroute()
ext_ips = [rt] if rt else []
for lip in listen_ips:
if not ext_ips or lip not in ["0.0.0.0", "::"] + ext_ips:
continue
desc = "\033[32mexternal"
ips = ext_ips if lip in ["0.0.0.0", "::"] else [lip]
for ip in ips:
ip = next((x for x in eps if x.startswith(ip + "/")), "")
if ip and "external" not in eps[ip].desc:
eps[ip].desc += ", " + desc
return eps
def ips_linux(self) -> dict[str, str]:
try:
txt, _ = chkcmd(["ip", "addr"])
except:
return self.ips_linux_ifconfig()
def _extdevs_nix(self) -> Generator[str, None, None]:
with open("/proc/net/route", "rb") as f:
next(f)
for ln in f:
r = ln.decode("utf-8").strip().split()
if r[1] == "0" * 8 and int(r[3], 16) & 2:
yield r[0]
r = re.compile(r"^\s+inet ([^ ]+)/.* (.*)")
ri = re.compile(r"^\s*[0-9]+\s*:.*")
up = False
eps: dict[str, str] = {}
for ln in txt.split("\n"):
if ri.match(ln):
up = "UP" in re.split("[>,< ]", ln)
try:
ip, dev = r.match(ln.rstrip()).groups() # type: ignore
eps[ip] = dev + ("" if up else ", \033[31mLINK-DOWN")
except:
pass
return eps
def ips_macos(self) -> dict[str, str]:
eps: dict[str, str] = {}
try:
txt, _ = chkcmd(["ifconfig"])
except:
return eps
rdev = re.compile(r"^([^ ]+):")
rip = re.compile(r"^\tinet ([0-9\.]+) ")
dev = "UNKNOWN"
for ln in txt.split("\n"):
m = rdev.match(ln)
if m:
dev = m.group(1)
m = rip.match(ln)
if m:
eps[m.group(1)] = dev
dev = "UNKNOWN"
return eps
def ips_windows_ipconfig(self) -> tuple[dict[str, str], set[str]]:
eps: dict[str, str] = {}
offs: set[str] = set()
try:
txt, _ = chkcmd(["ipconfig"])
except:
return eps, offs
rdev = re.compile(r"(^[^ ].*):$")
rip = re.compile(r"^ +IPv?4? [^:]+: *([0-9\.]{7,15})$")
roff = re.compile(r".*: Media disconnected$")
dev = None
for ln in txt.replace("\r", "").split("\n"):
m = rdev.match(ln)
if m:
if dev and dev not in eps.values():
offs.add(dev)
dev = m.group(1).split(" adapter ", 1)[-1]
if dev and roff.match(ln):
offs.add(dev)
dev = None
m = rip.match(ln)
if m and dev:
eps[m.group(1)] = dev
dev = None
if dev and dev not in eps.values():
offs.add(dev)
return eps, offs
def ips_windows_netsh(self) -> dict[str, str]:
eps: dict[str, str] = {}
try:
txt, _ = chkcmd("netsh interface ip show address".split())
except:
return eps
rdev = re.compile(r'.* "([^"]+)"$')
rip = re.compile(r".* IP\b.*: +([0-9\.]{7,15})$")
dev = None
for ln in txt.replace("\r", "").split("\n"):
m = rdev.match(ln)
if m:
dev = m.group(1)
m = rip.match(ln)
if m and dev:
eps[m.group(1)] = dev
return eps
def detect_interfaces(self, listen_ips: list[str]) -> dict[str, str]:
if MACOS:
eps = self.ips_macos()
elif ANYWIN:
eps, off = self.ips_windows_ipconfig() # sees more interfaces + link state
eps.update(self.ips_windows_netsh()) # has better names
for k, v in eps.items():
if v in off:
eps[k] += ", \033[31mLINK-DOWN"
else:
eps = self.ips_linux()
if "0.0.0.0" not in listen_ips:
eps = {k: v for k, v in eps.items() if k in listen_ips}
default_route = None
def _defroute(self) -> str:
ret = ""
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
for ip in [
"10.255.255.255",
"172.31.255.255",
"192.168.255.255",
"239.255.255.255",
"10.254.39.23",
"172.31.39.23",
"192.168.39.23",
"239.254.39.23",
"169.254.39.23",
# could add 1.1.1.1 as a final fallback
# but external connections is kinshi
]:
try:
s.connect((ip, 1))
default_route = s.getsockname()[0]
ret = s.getsockname()[0]
break
except (OSError, socket.error) as ex:
if ex.errno == 13:
if ex.errno in E_ACCESS:
self.log("tcpsrv", "eaccess {} (trying next)".format(ip))
elif ex.errno not in [101, 10065, 10051]:
elif ex.errno not in E_UNREACH:
self.log("tcpsrv", "route lookup failed; err {}".format(ex.errno))
s.close()
for lip in listen_ips:
if default_route and lip in ["0.0.0.0", default_route]:
desc = "\033[32mexternal"
try:
eps[default_route] += ", " + desc
except:
eps[default_route] = desc
return eps
return ret
def _set_wintitle(self, vs: dict[str, dict[str, int]]) -> None:
vs["all"] = vs.get("all", {"Local-Only": 1})
@@ -330,19 +418,108 @@ class TcpSrv(object):
vs2 = {}
for k, eps in vs.items():
vs2[k] = {
ep: 1
for ep in eps.keys()
if ":" not in ep or ep.split(":")[0] not in eps
}
filt = {ep: 1 for ep in eps if ":" not in ep}
have = set(filt)
for ep in sorted(eps):
ip = ep.split(":")[0]
if ip not in have:
have.add(ip)
filt[ep] = 1
lo = [x for x in filt if x.startswith("127.")]
if len(filt) > 3 and lo:
for ip in lo:
filt.pop(ip)
vs2[k] = filt
title = ""
vs = vs2
for p in self.args.wintitle.split(" "):
if p.startswith("$"):
p = " and ".join(sorted(vs.get(p[1:], {"(None)": 1}).keys()))
seps = list(sorted(vs.get(p[1:], {"(None)": 1}).keys()))
p = ", ".join(seps[:3])
if len(seps) > 3:
p += ", ..."
title += "{} ".format(p)
print("\033]0;{}\033\\".format(title), file=sys.stderr, end="")
print("\033]0;{}\033\\\n".format(title), file=sys.stderr, end="")
sys.stderr.flush()
def _qr(self, t1: dict[str, list[int]], t2: dict[str, list[int]]) -> str:
ip = None
ips = list(t1) + list(t2)
qri = self.args.qri
if self.args.zm and not qri:
name = self.args.name + ".local"
t1[name] = next(v for v in (t1 or t2).values())
ips = [name] + ips
for ip in ips:
if ip.startswith(qri) or qri == ".":
break
ip = ""
if not ip:
# maybe /bin/ip is missing or smth
ip = qri
if not ip:
return ""
if ":" in ip:
ip = "[{}]".format(ip)
if self.args.http_only:
https = ""
elif self.args.https_only:
https = "s"
else:
https = "s" if self.args.qrs else ""
ports = t1.get(ip, t2.get(ip, []))
dport = 443 if https else 80
port = "" if dport in ports or not ports else ":{}".format(ports[0])
txt = "http{}://{}{}/{}".format(https, ip, port, self.args.qrl)
btxt = txt.encode("utf-8")
if PY2:
btxt = sunpack(b"B" * len(btxt), btxt)
fg = self.args.qr_fg
bg = self.args.qr_bg
pad = self.args.qrp
zoom = self.args.qrz
qrc = QrCode.encode_binary(btxt)
if zoom == 0:
try:
tw, th = termsize()
tsz = min(tw // 2, th)
zoom = 1 if qrc.size + pad * 2 >= tsz else 2
except:
zoom = 1
qr = qrc.render(zoom, pad)
if not VT100:
return "{}\n{}".format(txt, qr)
halfc = "\033[40;48;5;{0}m{1}\033[47;48;5;{2}m"
if not fg:
halfc = "\033[0;40m{1}\033[0;47m"
def ansify(m: re.Match) -> str:
return halfc.format(fg, " " * len(m.group(1)), bg)
if zoom > 1:
qr = re.sub("(█+)", ansify, qr)
qr = qr.replace("\n", "\033[K\n") + "\033[K" # win10do
cc = " \033[0;38;5;{0};47;48;5;{1}m" if fg else " \033[0;30;47m"
t = cc + "\n{2}\033[999G\033[0m\033[J"
t = t.format(fg, bg, qr)
if ANYWIN:
# prevent color loss on terminal resize
t = t.replace("\n", "`\n`")
return txt + t

View File

@@ -9,10 +9,8 @@ from .bos import bos
from .th_srv import HAVE_WEBP, thumb_path
from .util import Cooldown
try:
if True: # pylint: disable=using-constant-test
from typing import Optional, Union
except:
pass
if TYPE_CHECKING:
from .httpsrv import HttpSrv
@@ -30,6 +28,8 @@ class ThumbCli(object):
try:
c = hsrv.th_cfg
if not c:
raise Exception()
except:
c = {k: {} for k in ["thumbable", "pil", "vips", "ffi", "ffv", "ffa"]}

View File

@@ -3,6 +3,7 @@ from __future__ import print_function, unicode_literals
import base64
import hashlib
import logging
import os
import shutil
import subprocess as sp
@@ -11,15 +12,25 @@ import time
from queue import Queue
from .__init__ import TYPE_CHECKING
from .__init__ import ANYWIN, TYPE_CHECKING
from .bos import bos
from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
from .util import BytesIO, Cooldown, Pebkac, fsenc, min_ex, runcmd, statdir, vsplit
from .util import (
BytesIO,
Cooldown,
Daemon,
FFMPEG_URL,
Pebkac,
afsenc,
fsenc,
min_ex,
runcmd,
statdir,
vsplit,
)
try:
if True: # pylint: disable=using-constant-test
from typing import Optional, Union
except:
pass
if TYPE_CHECKING:
from .svchub import SvcHub
@@ -53,12 +64,16 @@ try:
HAVE_AVIF = True
except:
pass
logging.getLogger("PIL").setLevel(logging.WARNING)
except:
pass
try:
HAVE_VIPS = True
import pyvips
logging.getLogger("pyvips").setLevel(logging.WARNING)
except:
HAVE_VIPS = False
@@ -69,14 +84,14 @@ def thumb_path(histpath: str, rem: str, mtime: float, fmt: str) -> str:
# base64 = 64 = 4096
rd, fn = vsplit(rem)
if rd:
h = hashlib.sha512(fsenc(rd)).digest()
h = hashlib.sha512(afsenc(rd)).digest()
b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24]
rd = "{}/{}/".format(b64[:2], b64[2:4]).lower() + b64
else:
rd = "top"
# could keep original filenames but this is safer re pathlen
h = hashlib.sha512(fsenc(fn)).digest()
h = hashlib.sha512(afsenc(fn)).digest()
fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
if fmt in ("opus", "caf"):
@@ -106,11 +121,7 @@ class ThumbSrv(object):
self.q: Queue[Optional[tuple[str, str]]] = Queue(self.nthr * 4)
for n in range(self.nthr):
thr = threading.Thread(
target=self.worker, name="thumb-{}-{}".format(n, self.nthr)
)
thr.daemon = True
thr.start()
Daemon(self.worker, "thumb-{}-{}".format(n, self.nthr))
want_ff = not self.args.no_vthumb or not self.args.no_athumb
if want_ff and (not HAVE_FFMPEG or not HAVE_FFPROBE):
@@ -124,11 +135,11 @@ class ThumbSrv(object):
msg = "cannot create audio/video thumbnails because some of the required programs are not available: "
msg += ", ".join(missing)
self.log(msg, c=3)
if ANYWIN and not self.args.no_acode:
self.log("download FFmpeg to fix it:\033[0m " + FFMPEG_URL, 3)
if self.args.th_clean:
t = threading.Thread(target=self.cleaner, name="thumb.cln")
t.daemon = True
t.start()
Daemon(self.cleaner, "thumb.cln")
self.fmt_pil, self.fmt_vips, self.fmt_ffi, self.fmt_ffv, self.fmt_ffa = [
set(y.split(","))
@@ -189,12 +200,12 @@ class ThumbSrv(object):
self.log("wait {}".format(tpath))
except:
thdir = os.path.dirname(tpath)
bos.makedirs(thdir)
bos.makedirs(os.path.join(thdir, "w"))
inf_path = os.path.join(thdir, "dir.txt")
if not bos.path.exists(inf_path):
with open(inf_path, "wb") as f:
f.write(fsenc(os.path.dirname(abspath)))
f.write(afsenc(os.path.dirname(abspath)))
self.busy[tpath] = [cond]
do_conv = True
@@ -240,47 +251,55 @@ class ThumbSrv(object):
abspath, tpath = task
ext = abspath.split(".")[-1].lower()
png_ok = False
fun = None
funs = []
if not bos.path.exists(tpath):
for lib in self.args.th_dec:
if fun:
break
elif lib == "pil" and ext in self.fmt_pil:
fun = self.conv_pil
if lib == "pil" and ext in self.fmt_pil:
funs.append(self.conv_pil)
elif lib == "vips" and ext in self.fmt_vips:
fun = self.conv_vips
funs.append(self.conv_vips)
elif lib == "ff" and ext in self.fmt_ffi or ext in self.fmt_ffv:
fun = self.conv_ffmpeg
funs.append(self.conv_ffmpeg)
elif lib == "ff" and ext in self.fmt_ffa:
if tpath.endswith(".opus") or tpath.endswith(".caf"):
fun = self.conv_opus
funs.append(self.conv_opus)
elif tpath.endswith(".png"):
fun = self.conv_waves
funs.append(self.conv_waves)
png_ok = True
else:
fun = self.conv_spec
funs.append(self.conv_spec)
if not png_ok and tpath.endswith(".png"):
raise Pebkac(400, "png only allowed for waveforms")
if fun:
tdir, tfn = os.path.split(tpath)
ttpath = os.path.join(tdir, "w", tfn)
for fun in funs:
try:
fun(abspath, tpath)
fun(abspath, ttpath)
break
except Exception as ex:
msg = "{} could not create thumbnail of {}\n{}"
msg = msg.format(fun.__name__, abspath, min_ex())
c: Union[str, int] = 1 if "<Signals.SIG" in msg else "1;30"
c: Union[str, int] = 1 if "<Signals.SIG" in msg else "90"
self.log(msg, c)
if getattr(ex, "returncode", 0) != 321:
with open(tpath, "wb") as _:
pass
if fun == funs[-1]:
with open(ttpath, "wb") as _:
pass
else:
# ffmpeg may spawn empty files on windows
try:
os.unlink(tpath)
os.unlink(ttpath)
except:
pass
try:
bos.rename(ttpath, tpath)
except:
pass
with self.mutex:
subs = self.busy[tpath]
del self.busy[tpath]
@@ -323,7 +342,7 @@ class ThumbSrv(object):
try:
im = self.fancy_pillow(im)
except Exception as ex:
self.log("fancy_pillow {}".format(ex), "1;30")
self.log("fancy_pillow {}".format(ex), "90")
im.thumbnail(self.res)
fmts = ["RGB", "L"]
@@ -361,7 +380,8 @@ class ThumbSrv(object):
img = pyvips.Image.thumbnail(abspath, w, **kw)
break
except:
pass
if c == crops[-1]:
raise
img.write_to_file(tpath, Q=40)
@@ -423,7 +443,7 @@ class ThumbSrv(object):
if not ret:
return
c: Union[str, int] = "1;30"
c: Union[str, int] = "90"
t = "FFmpeg failed (probably a corrupt video file):\n"
if (
(not self.args.th_ff_jpg or time.time() - int(self.args.th_ff_jpg) < 60)
@@ -621,7 +641,7 @@ class ThumbSrv(object):
def _clean(self, cat: str, thumbpath: str) -> int:
# self.log("cln {}".format(thumbpath))
exts = ["jpg", "webp"] if cat == "th" else ["opus", "caf"]
exts = ["jpg", "webp", "png"] if cat == "th" else ["opus", "caf"]
maxage = getattr(self.args, cat + "_maxage")
now = time.time()
prev_b64 = None

View File

@@ -11,7 +11,16 @@ from operator import itemgetter
from .__init__ import ANYWIN, TYPE_CHECKING, unicode
from .bos import bos
from .up2k import up2k_wark_from_hashlist
from .util import HAVE_SQLITE3, Pebkac, absreal, gen_filekey, min_ex, quotep, s3dec
from .util import (
HAVE_SQLITE3,
Daemon,
Pebkac,
absreal,
gen_filekey,
min_ex,
quotep,
s3dec,
)
if HAVE_SQLITE3:
import sqlite3
@@ -21,10 +30,8 @@ try:
except:
pass
try:
if True: # pylint: disable=using-constant-test
from typing import Any, Optional, Union
except:
pass
if TYPE_CHECKING:
from .httpconn import HttpConn
@@ -90,14 +97,17 @@ class U2idx(object):
return None
cur = None
if ANYWIN:
if ANYWIN and not bos.path.exists(db_path + "-wal"):
uri = ""
try:
uri = "{}?mode=ro&nolock=1".format(Path(db_path).as_uri())
cur = sqlite3.connect(uri, 2, uri=True).cursor()
cur.execute('pragma table_info("up")').fetchone()
self.log("ro: {}".format(db_path))
except:
self.log("could not open read-only: {}\n{}".format(uri, min_ex()))
# may not fail until the pragma so unset it
cur = None
if not cur:
# on windows, this steals the write-lock from up2k.deferred_init --
@@ -190,7 +200,7 @@ class U2idx(object):
v = "exists(select 1 from mt where mt.w = mtw and " + vq
else:
raise Pebkac(400, "invalid key [" + v + "]")
raise Pebkac(400, "invalid key [{}]".format(v))
q += v + " "
continue
@@ -270,16 +280,7 @@ class U2idx(object):
self.active_id = "{:.6f}_{}".format(
time.time(), threading.current_thread().ident
)
thr = threading.Thread(
target=self.terminator,
args=(
self.active_id,
done_flag,
),
name="u2idx-terminator",
)
thr.daemon = True
thr.start()
Daemon(self.terminator, "u2idx-terminator", (self.active_id, done_flag))
if not uq or not uv:
uq = "select * from up"
@@ -310,6 +311,7 @@ class U2idx(object):
sret = []
fk = flags.get("fk")
dots = flags.get("dotsrch")
c = cur.execute(uq, tuple(vuv))
for hit in c:
w, ts, sz, rd, fn, ip, at = hit[:7]
@@ -320,6 +322,10 @@ class U2idx(object):
if rd.startswith("//") or fn.startswith("//"):
rd, fn = s3dec(rd, fn)
rp = quotep("/".join([x for x in [vtop, rd, fn] if x]))
if not dots and "/." in ("/" + rp):
continue
if not fk:
suf = ""
else:
@@ -336,8 +342,7 @@ class U2idx(object):
)[:fk]
)
rp = quotep("/".join([x for x in [vtop, rd, fn] if x])) + suf
sret.append({"ts": int(ts), "sz": sz, "rp": rp, "w": w[:16]})
sret.append({"ts": int(ts), "sz": sz, "rp": rp + suf, "w": w[:16]})
for hit in sret:
w = hit["w"]

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

View File

@@ -0,0 +1 @@
../../../bin/partyfuse.py

1
copyparty/web/a/up2k.py Symbolic link
View File

@@ -0,0 +1 @@
../../../bin/up2k.py

View File

@@ -0,0 +1 @@
../../../contrib/webdav-cfg.bat

View File

@@ -27,8 +27,8 @@ window.baguetteBox = (function () {
isOverlayVisible = false,
touch = {}, // start-pos
touchFlag = false, // busy
re_i = /.+\.(gif|jpe?g|png|webp)(\?|$)/i,
re_v = /.+\.(webm|mp4)(\?|$)/i,
re_i = /.+\.(a?png|avif|bmp|gif|heif|jpe?g|jfif|svg|webp)(\?|$)/i,
re_v = /.+\.(webm|mkv|mp4)(\?|$)/i,
anims = ['slideIn', 'fadeIn', 'none'],
data = {}, // all galleries
imagesElements = [],
@@ -246,12 +246,24 @@ window.baguetteBox = (function () {
}
function keyDownHandler(e) {
if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing || modal.busy)
if (modal.busy)
return;
if (e.key == '?')
return halp();
if (anymod(e, true))
return;
var k = e.code + '', v = vid(), pos = -1;
if (k == "ArrowLeft" || k == "KeyJ")
if (k == "BracketLeft")
setloop(1);
else if (k == "BracketRight")
setloop(2);
else if (e.shiftKey)
return;
else if (k == "ArrowLeft" || k == "KeyJ")
showPreviousImage();
else if (k == "ArrowRight" || k == "KeyL")
showNextImage();
@@ -265,8 +277,8 @@ window.baguetteBox = (function () {
playpause();
else if (k == "KeyU" || k == "KeyO")
relseek(k == "KeyU" ? -10 : 10);
else if (k.indexOf('Digit') === 0)
vid().currentTime = vid().duration * parseInt(k.slice(-1)) * 0.1;
else if (k.indexOf('Digit') === 0 && v)
v.currentTime = v.duration * parseInt(k.slice(-1)) * 0.1;
else if (k == "KeyM" && v) {
v.muted = vmute = !vmute;
mp_ctl();
@@ -289,10 +301,6 @@ window.baguetteBox = (function () {
rotn(e.shiftKey ? -1 : 1);
else if (k == "KeyY")
dlpic();
else if (k == "BracketLeft")
setloop(1);
else if (k == "BracketRight")
setloop(2);
}
function anim() {
@@ -406,7 +414,7 @@ window.baguetteBox = (function () {
}
function keyUpHandler(e) {
if (e.ctrlKey || e.altKey || e.metaKey || e.isComposing)
if (anymod(e))
return;
var k = e.code + '';

View File

@@ -572,6 +572,11 @@ html.dy {
* {
line-height: 1.2em;
}
::selection {
color: var(--bg-d1);
background: var(--fg);
text-shadow: none;
}
html,body,tr,th,td,#files,a {
color: inherit;
background: none;
@@ -754,8 +759,9 @@ html.y #files thead th {
display: inline;
}
#path a {
margin: 0 0 0 -.2em;
padding: 0 0 0 .4em;
padding: 0 .35em;
position: relative;
z-index: 1;
/* ie: */
border-bottom: .1em solid #777\9;
margin-right: 1em\9;
@@ -763,18 +769,17 @@ html.y #files thead th {
#path a:first-child {
padding-left: .8em;
}
#path a:not(:last-child):after {
content: '';
#path i {
width: 1.05em;
height: 1.05em;
margin: -.2em .3em -.2em -.4em;
margin: -.5em .15em -.15em -.7em;
display: inline-block;
border: 1px solid rgba(255,224,192,0.3);
border-width: .05em .05em 0 0;
transform: rotate(45deg);
background: linear-gradient(45deg, rgba(0,0,0,0) 40%, rgba(0,0,0,0.25) 75%, rgba(0,0,0,0.35));
}
html.y #path a:not(:last-child)::after {
html.y #path i {
background: none;
border-color: rgba(0,0,0,0.2);
border-width: .1em .1em 0 0;
@@ -793,6 +798,24 @@ html.y #path a:hover {
.logue:empty {
display: none;
}
#doc>iframe,
.logue>iframe {
background: var(--bgg);
border: 1px solid var(--bgg);
border-width: 0 .3em 0 .3em;
border-radius: .5em;
visibility: hidden;
margin: 0 -.3em;
width: 100%;
height: 0;
}
#doc>iframe.focus,
.logue>iframe.focus {
box-shadow: 0 0 .1em .1em var(--a);
}
#pro.logue>iframe {
height: 100vh;
}
#pro.logue {
margin-bottom: .8em;
}
@@ -817,6 +840,10 @@ html.y #path a:hover {
.mdo {
max-width: 52em;
}
.mdo.sb,
#epi.logue.mdo>iframe {
max-width: 54em;
}
.mdo,
.mdo * {
line-height: 1.4em;
@@ -857,6 +884,12 @@ html.y #path a:hover {
color: var(--srv-3);
border-bottom: 1px solid var(--srv-3b);
}
#goh+span {
color: var(--bg-u5);
padding-left: .5em;
margin-left: .5em;
border-left: .2em solid var(--bg-u5);
}
#repl {
padding: .33em;
}
@@ -1069,18 +1102,18 @@ html.y #widget.open {
top: -.12em;
}
#wtico {
cursor: url(/.cpr/dd/4.png), pointer;
cursor: url(dd/4.png), pointer;
animation: cursor 500ms;
}
#wtico:hover {
animation: cursor 500ms infinite;
}
@keyframes cursor {
0% {cursor: url(/.cpr/dd/2.png), pointer}
30% {cursor: url(/.cpr/dd/3.png), pointer}
50% {cursor: url(/.cpr/dd/4.png), pointer}
75% {cursor: url(/.cpr/dd/5.png), pointer}
85% {cursor: url(/.cpr/dd/4.png), pointer}
0% {cursor: url(dd/2.png), pointer}
30% {cursor: url(dd/3.png), pointer}
50% {cursor: url(dd/4.png), pointer}
75% {cursor: url(dd/5.png), pointer}
85% {cursor: url(dd/4.png), pointer}
}
@keyframes spin {
100% {transform: rotate(360deg)}
@@ -1095,7 +1128,6 @@ html.y #widget.open {
#wtoggle {
position: absolute;
white-space: nowrap;
font-size: .8em;
top: -1em;
right: 0;
height: 1em;
@@ -1177,7 +1209,7 @@ html.y #widget.open {
font-size: .4em;
margin: -.3em .1em;
}
#wtoggle.sel #wzip #selzip {
#wtoggle.sel .l1 {
top: -.6em;
padding: .4em .3em;
}
@@ -1221,6 +1253,40 @@ html.y #widget.open {
width: calc(100% - 10.5em);
background: rgba(0,0,0,0.2);
}
#widget.cmp {
height: 1.6em;
bottom: -1.6em;
}
#widget.cmp.open {
bottom: 0;
}
#widget.cmp #wtoggle {
font-size: 1.2em;
}
#widget.cmp #wtgrid {
display: none;
}
#widget.cmp #pctl {
top: 0;
left: 0;
font-size: .75em;
}
#widget.cmp #pctl a {
margin: 0;
}
#widget.cmp #barpos,
#widget.cmp #barbuf {
width: calc(100% - 11em);
border-radius: 0;
left: 5em;
top: 0;
}
#widget.cmp #pvol {
top: 0;
right: 0;
max-width: 5.8em;
border-radius: 0;
}
.opview {
display: none;
}
@@ -1344,8 +1410,12 @@ input.eq_gain {
padding-right: .2em;
text-align: right;
}
#srch_form:not(.tags) #tsrch_tags,
#srch_form:not(.tags) #tsrch_adv {
display: none;
}
#op_search input {
margin: 0;
margin: .1em 0 0 0;
}
#srch_q {
white-space: pre;
@@ -1805,6 +1875,36 @@ a.btn,
-ms-user-select: none;
user-select: none;
}
#hkhelp {
background: var(--bg);
}
#hkhelp table {
margin: 2em 2em 0 2em;
float: left;
}
#hkhelp th {
border-bottom: 1px solid var(--bg-u5);
background: var(--bg-u1);
font-weight: bold;
text-align: right;
}
#hkhelp tr+tr th {
border-top: 1.5em solid var(--bg);
}
#hkhelp td {
padding: .2em .3em;
}
#hkhelp td:first-child {
font-family: 'scp', monospace, monospace;
}
html.noscroll,
html.noscroll .sbar {
scrollbar-width: none;
}
html.noscroll::-webkit-scrollbar,
html.noscroll .sbar::-webkit-scrollbar {
display: none;
}
@@ -2484,7 +2584,6 @@ html.b #u2conf a.b:hover {
#u2conf input[type="checkbox"]:checked+label:hover {
background: var(--u2-o-1h-bg);
}
#op_up2k.srch #u2conf td:nth-child(1)>*,
#op_up2k.srch #u2conf td:nth-child(2)>*,
#op_up2k.srch #u2conf td:nth-child(3)>* {
background: #777;

View File

@@ -5,10 +5,11 @@
<meta charset="utf-8">
<title>{{ title }}</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=0.8">
<meta name="viewport" content="width=device-width, initial-scale=0.8, minimum-scale=0.6">
<meta name="theme-color" content="#333">
{{ html_head }}
<link rel="stylesheet" media="screen" href="/.cpr/ui.css?_={{ ts }}">
<link rel="stylesheet" media="screen" href="/.cpr/browser.css?_={{ ts }}">
<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/ui.css?_={{ ts }}">
<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/browser.css?_={{ ts }}">
{%- if css %}
<link rel="stylesheet" media="screen" href="{{ css }}?_={{ ts }}">
{%- endif %}
@@ -35,13 +36,13 @@
<input type="file" name="f" multiple /><br />
<input type="submit" value="start upload">
</form>
<a id="bbsw" href="?b=u"><br />switch to basic browser</a>
<a id="bbsw" href="?b=u" rel="nofollow"><br />switch to basic browser</a>
</div>
<div id="op_mkdir" class="opview opbox act">
<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
<input type="hidden" name="act" value="mkdir" />
📂<input type="text" name="name" class="i">
📂<input type="text" name="name" class="i" placeholder="awesome mix vol.1">
<input type="submit" value="make directory">
</form>
</div>
@@ -49,14 +50,14 @@
<div id="op_new_md" class="opview opbox">
<form method="post" enctype="multipart/form-data" accept-charset="utf-8" action="{{ url_suf }}">
<input type="hidden" name="act" value="new_md" />
📝<input type="text" name="name" class="i">
📝<input type="text" name="name" class="i" placeholder="weekend-plans">
<input type="submit" value="new markdown doc">
</form>
</div>
<div id="op_msg" class="opview opbox act">
<form method="post" enctype="application/x-www-form-urlencoded" accept-charset="utf-8" action="{{ url_suf }}">
📟<input type="text" name="msg" class="i">
📟<input type="text" name="msg" class="i" placeholder="lorem ipsum dolor sit amet">
<input type="submit" value="send msg to srv log">
</form>
</div>
@@ -70,7 +71,7 @@
<h1 id="path">
<a href="#" id="entree">🌲</a>
{%- for n in vpnodes %}
<a href="/{{ n[0] }}">{{ n[1] }}</a>
<a href="{{ r }}/{{ n[0] }}">{{ n[1] }}</a>
{%- endfor %}
</h1>
@@ -84,7 +85,7 @@
<div id="bdoc"></div>
{%- endif %}
<div id="pro" class="logue">{{ logues[0] }}</div>
<div id="pro" class="logue">{{ "" if sb_lg else logues[0] }}</div>
<table id="files">
<thead>
@@ -118,9 +119,9 @@
</tbody>
</table>
<div id="epi" class="logue">{{ logues[1] }}</div>
<div id="epi" class="logue">{{ "" if sb_lg else logues[1] }}</div>
<h2><a href="/?h" id="goh">control-panel</a></h2>
<h2 id="wfp"><a href="{{ r }}/?h" id="goh">control-panel</a></h2>
<a href="#" id="repl">π</a>
@@ -133,7 +134,9 @@
<div id="widget"></div>
<script>
var acct = "{{ acct }}",
var SR = {{ r|tojson }},
TS = "{{ ts }}",
acct = "{{ acct }}",
perms = {{ perms }},
themes = {{ themes }},
dtheme = "{{ dtheme }}",
@@ -148,21 +151,24 @@
have_del = {{ have_del|tojson }},
have_unpost = {{ have_unpost }},
have_zip = {{ have_zip|tojson }},
sb_md = "{{ sb_md }}",
sb_lg = "{{ sb_lg }}",
lifetime = {{ lifetime }},
turbolvl = {{ turbolvl }},
frand = {{ frand|tojson }},
u2sort = "{{ u2sort }}",
have_emp = {{ have_emp|tojson }},
txt_ext = "{{ txt_ext }}",
{% if no_prism %}no_prism = 1,{% endif %}
logues = {{ logues|tojson if sb_lg else "[]" }},
readme = {{ readme|tojson }},
ls0 = {{ ls0|tojson }};
document.documentElement.className = localStorage.theme || dtheme;
</script>
<script src="/.cpr/util.js?_={{ ts }}"></script>
<script src="/.cpr/baguettebox.js?_={{ ts }}"></script>
<script src="/.cpr/browser.js?_={{ ts }}"></script>
<script src="/.cpr/up2k.js?_={{ ts }}"></script>
<script src="{{ r }}/.cpr/util.js?_={{ ts }}"></script>
<script src="{{ r }}/.cpr/baguettebox.js?_={{ ts }}"></script>
<script src="{{ r }}/.cpr/browser.js?_={{ ts }}"></script>
<script src="{{ r }}/.cpr/up2k.js?_={{ ts }}"></script>
{%- if js %}
<script src="{{ js }}?_={{ ts }}"></script>
{%- endif %}

File diff suppressed because it is too large Load Diff

View File

@@ -57,7 +57,7 @@
<div>{{ logues[1] }}</div><br />
{%- endif %}
<h2><a href="/{{ url_suf }}{{ url_suf and '&amp;' or '?' }}h">control-panel</a></h2>
<h2><a href="{{ r }}/{{ url_suf }}{{ url_suf and '&amp;' or '?' }}h">control-panel</a></h2>
</body>
</html>

View File

View File

View File

@@ -4,6 +4,12 @@ html, body {
font-family: sans-serif;
line-height: 1.5em;
}
html.y #helpbox a {
color: #079;
}
html.z #helpbox a {
color: #fc5;
}
#repl {
position: absolute;
top: 0;

View File

@@ -3,11 +3,12 @@
<title>📝 {{ title }}</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=0.7">
<meta name="theme-color" content="#333">
{{ html_head }}
<link rel="stylesheet" href="/.cpr/ui.css?_={{ ts }}">
<link rel="stylesheet" href="/.cpr/md.css?_={{ ts }}">
<link rel="stylesheet" href="{{ r }}/.cpr/ui.css?_={{ ts }}">
<link rel="stylesheet" href="{{ r }}/.cpr/md.css?_={{ ts }}">
{%- if edit %}
<link rel="stylesheet" href="/.cpr/md2.css?_={{ ts }}">
<link rel="stylesheet" href="{{ r }}/.cpr/md2.css?_={{ ts }}">
{%- endif %}
</head>
<body>
@@ -31,7 +32,7 @@
{%- else %}
<a href="{{ arg_base }}edit" tt="good: higher performance$Ngood: same document width as viewer$Nbad: assumes you know markdown">edit (basic)</a>
<a href="{{ arg_base }}edit2" tt="not in-house so probably less buggy">edit (fancy)</a>
<a href="{{ arg_base }}raw">view raw</a>
<a href="{{ arg_base }}">view raw</a>
{%- endif %}
</div>
<div id="toc"></div>
@@ -127,7 +128,8 @@ write markdown (most html is 🙆 too)
<script>
var last_modified = {{ lastmod }},
var SR = {{ r|tojson }},
last_modified = {{ lastmod }},
have_emp = {{ have_emp|tojson }},
dfavico = "{{ favico }}";
@@ -152,10 +154,10 @@ l.light = drk? 0:1;
})();
</script>
<script src="/.cpr/util.js?_={{ ts }}"></script>
<script src="/.cpr/deps/marked.js?_={{ ts }}"></script>
<script src="/.cpr/md.js?_={{ ts }}"></script>
<script src="{{ r }}/.cpr/util.js?_={{ ts }}"></script>
<script src="{{ r }}/.cpr/deps/marked.js?_={{ ts }}"></script>
<script src="{{ r }}/.cpr/md.js?_={{ ts }}"></script>
{%- if edit %}
<script src="/.cpr/md2.js?_={{ ts }}"></script>
<script src="{{ r }}/.cpr/md2.js?_={{ ts }}"></script>
{%- endif %}
</body></html>

View File

@@ -1,12 +1,13 @@
"use strict";
var dom_toc = ebi('toc');
var dom_wrap = ebi('mw');
var dom_hbar = ebi('mh');
var dom_nav = ebi('mn');
var dom_pre = ebi('mp');
var dom_src = ebi('mt');
var dom_navtgl = ebi('navtoggle');
var dom_toc = ebi('toc'),
dom_wrap = ebi('mw'),
dom_hbar = ebi('mh'),
dom_nav = ebi('mn'),
dom_pre = ebi('mp'),
dom_src = ebi('mt'),
dom_navtgl = ebi('navtoggle'),
hash0 = location.hash;
// chrome 49 needs this
@@ -35,12 +36,12 @@ var dbg = function () { };
// add navbar
(function () {
var parts = get_evpath().split('/'), link = '', o;
for (var a = 0, aa = parts.length - 2; a <= aa; a++) {
var parts = (get_evpath().slice(0, -1).split('?')[0] + '?v').split('/'), link = '', o;
for (var a = 0, aa = parts.length - 1; a <= aa; a++) {
link += parts[a] + (a < aa ? '/' : '');
o = mknod('a');
o.setAttribute('href', link);
o.textContent = uricom_dec(parts[a]) || 'top';
o.textContent = uricom_dec(parts[a].split('?')[0]) || 'top';
dom_nav.appendChild(o);
}
})();
@@ -256,7 +257,7 @@ function convert_markdown(md_text, dest_dom) {
var html = dom_li.innerHTML;
dom_li.innerHTML =
'<span class="todo_' + clas + '">' + char + '</span>' +
html.substr(html.indexOf('>') + 1);
html.slice(html.indexOf('>') + 1);
}
// separate <code> for each line in <pre>
@@ -328,6 +329,15 @@ function convert_markdown(md_text, dest_dom) {
catch (ex) {
md_plug_err(ex, ext[1]);
}
if (hash0)
setTimeout(function () {
try {
QS(hash0).scrollIntoView();
hash0 = '';
}
catch (ex) { }
}, 1);
}
@@ -498,5 +508,5 @@ dom_navtgl.onclick = function () {
if (sread('hidenav') == 1)
dom_navtgl.onclick();
if (window['tt'])
if (window.tt && tt.init)
tt.init();

View File

@@ -107,7 +107,8 @@ var draw_md = (function () {
map_src = genmap(dom_ref, map_src);
map_pre = genmap(dom_pre, map_pre);
clmod(ebi('save'), 'disabled', src == server_md);
clmod(ebi('save'), 'disabled',
src.replace(/\r/g, "") == server_md.replace(/\r/g, ""));
var t1 = Date.now();
delay = t1 - t0 > 100 ? 25 : 1;
@@ -230,7 +231,8 @@ redraw = (function () {
// modification checker
function Modpoll() {
var r = {
skip_one: true,
initial: true,
skip_one: false,
disabled: false
};
@@ -253,7 +255,7 @@ function Modpoll() {
}
console.log('modpoll...');
var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
var url = (document.location + '').split('?')[0] + '?_=' + Date.now();
var xhr = new XHR();
xhr.open('GET', url, true);
xhr.responseType = 'text';
@@ -275,8 +277,18 @@ function Modpoll() {
if (!this.responseText)
return;
var server_ref = server_md.replace(/\r/g, '');
var server_now = this.responseText.replace(/\r/g, '');
var new_md = this.responseText,
server_ref = server_md.replace(/\r/g, ''),
server_now = new_md.replace(/\r/g, '');
// firefox bug: sometimes get stale text even if copyparty sent a 200
if (r.initial && server_ref != server_now)
return modal.confirm('Your browser decided to show an outdated copy of the document!\n\nDo you want to load the latest version from the server instead?', function () {
dom_src.value = server_md = new_md;
draw_md();
}, null);
r.initial = false;
if (server_ref != server_now) {
console.log("modpoll diff |" + server_ref.length + "|, |" + server_now.length + "|");
@@ -296,6 +308,7 @@ function Modpoll() {
console.log('modpoll eq');
};
setTimeout(r.periodic, 300);
if (md_opt.modpoll_freq > 0)
setInterval(r.periodic, 1000 * md_opt.modpoll_freq);
@@ -389,7 +402,7 @@ function save_cb() {
function run_savechk(lastmod, txt, btn, ntry) {
// download the saved doc from the server and compare
var url = (document.location + '').split('?')[0] + '?raw&_=' + Date.now();
var url = (document.location + '').split('?')[0] + '?_=' + Date.now();
var xhr = new XHR();
xhr.open('GET', url, true);
xhr.responseType = 'text';
@@ -917,7 +930,9 @@ var set_lno = (function () {
(function () {
function keydown(ev) {
ev = ev || window.event;
var kc = ev.code || ev.keyCode || ev.which;
var kc = ev.code || ev.keyCode || ev.which,
editing = document.activeElement == dom_src;
//console.log(ev.key, ev.code, ev.keyCode, ev.which);
if (ctrl(ev) && (ev.code == "KeyS" || kc == 83)) {
save();
@@ -928,12 +943,17 @@ var set_lno = (function () {
if (d)
d.click();
}
if (document.activeElement != dom_src)
return true;
set_lno();
if (editing)
set_lno();
if (ctrl(ev)) {
if (ev.code == "KeyE") {
dom_nsbs.click();
return false;
}
if (!editing)
return true;
if (ev.code == "KeyH" || kc == 72) {
md_header(ev.shiftKey);
return false;
@@ -958,10 +978,6 @@ var set_lno = (function () {
iter_uni();
return false;
}
if (ev.code == "KeyE") {
dom_nsbs.click();
return false;
}
var up = ev.code == "ArrowUp" || kc == 38;
var dn = ev.code == "ArrowDown" || kc == 40;
if (up || dn) {
@@ -974,6 +990,9 @@ var set_lno = (function () {
}
}
else {
if (!editing)
return true;
if (ev.code == "Tab" || kc == 9) {
md_indent(ev.shiftKey);
return false;

View File

@@ -3,11 +3,12 @@
<title>📝 {{ title }}</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=0.7">
<meta name="theme-color" content="#333">
{{ html_head }}
<link rel="stylesheet" href="/.cpr/ui.css?_={{ ts }}">
<link rel="stylesheet" href="/.cpr/mde.css?_={{ ts }}">
<link rel="stylesheet" href="/.cpr/deps/mini-fa.css?_={{ ts }}">
<link rel="stylesheet" href="/.cpr/deps/easymde.css?_={{ ts }}">
<link rel="stylesheet" href="{{ r }}/.cpr/ui.css?_={{ ts }}">
<link rel="stylesheet" href="{{ r }}/.cpr/mde.css?_={{ ts }}">
<link rel="stylesheet" href="{{ r }}/.cpr/deps/mini-fa.css?_={{ ts }}">
<link rel="stylesheet" href="{{ r }}/.cpr/deps/easymde.css?_={{ ts }}">
</head>
<body>
<div id="mw">
@@ -25,7 +26,8 @@
<a href="#" id="repl">π</a>
<script>
var last_modified = {{ lastmod }},
var SR = {{ r|tojson }},
last_modified = {{ lastmod }},
have_emp = {{ have_emp|tojson }},
dfavico = "{{ favico }}";
@@ -47,8 +49,8 @@ l.light = drk? 0:1;
})();
</script>
<script src="/.cpr/util.js?_={{ ts }}"></script>
<script src="/.cpr/deps/marked.js?_={{ ts }}"></script>
<script src="/.cpr/deps/easymde.js?_={{ ts }}"></script>
<script src="/.cpr/mde.js?_={{ ts }}"></script>
<script src="{{ r }}/.cpr/util.js?_={{ ts }}"></script>
<script src="{{ r }}/.cpr/deps/marked.js?_={{ ts }}"></script>
<script src="{{ r }}/.cpr/deps/easymde.js?_={{ ts }}"></script>
<script src="{{ r }}/.cpr/mde.js?_={{ ts }}"></script>
</body></html>

View File

@@ -7,7 +7,7 @@ var dom_md = ebi('mt');
(function () {
var n = document.location + '';
n = n.substr(n.indexOf('//') + 2).split('?')[0].split('/');
n = (n.slice(n.indexOf('//') + 2).split('?')[0] + '?v').split('/');
n[0] = 'top';
var loc = [];
var nav = [];
@@ -15,7 +15,7 @@ var dom_md = ebi('mt');
if (a > 0)
loc.push(n[a]);
var dec = uricom_dec(n[a]).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
var dec = uricom_dec(n[a].split('?')[0]).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
nav.push('<a href="/' + loc.join('/') + '">' + dec + '</a>');
}
@@ -166,7 +166,7 @@ function save_cb() {
//alert('save OK -- wrote ' + r.size + ' bytes.\n\nsha512: ' + r.sha512);
// download the saved doc from the server and compare
var url = (document.location + '').split('?')[0] + '?raw';
var url = (document.location + '').split('?')[0] + '?_=' + Date.now();
var xhr = new XHR();
xhr.open('GET', url, true);
xhr.responseType = 'text';

View File

@@ -6,8 +6,9 @@
<title>{{ svcname }}</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=0.8">
{{ html_head }}
<link rel="stylesheet" media="screen" href="/.cpr/msg.css?_={{ ts }}">
<meta name="theme-color" content="#333">
{{ html_head }}
<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/msg.css?_={{ ts }}">
</head>
<body>

Some files were not shown because too many files have changed in this diff Show More