From fade907c6ac4e806ca7d6a101358aa7dd2ee99ba Mon Sep 17 00:00:00 2001 From: yarikoptic Date: Tue, 9 Jul 2024 02:44:51 +0000 Subject: [PATCH 01/30] initial report from boox installation --- ...id_boox__58___xargs_Permission_denied.mdwn | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 doc/bugs/install_on_android_boox__58___xargs_Permission_denied.mdwn diff --git a/doc/bugs/install_on_android_boox__58___xargs_Permission_denied.mdwn b/doc/bugs/install_on_android_boox__58___xargs_Permission_denied.mdwn new file mode 100644 index 0000000000..420d3235c6 --- /dev/null +++ b/doc/bugs/install_on_android_boox__58___xargs_Permission_denied.mdwn @@ -0,0 +1,143 @@ +### Please describe the problem. + +A new kind of an Android device -- a new attempt to make use of git-annex on it ;) This time BOOX Tablet Tab Ultra C Pro ePaper Tablet PC 10.3 with Termux installed from Google Play store and reporting + +``` +~ $ pwd +/data/data/com.termux/files/home + +~ $ df . -h +Filesystem Size Used Avail Use% Mounted on +/dev/block/dm-10 108G 3.9G 104G 4% /data/data/com.google.android.gms + +~ $ mount | grep data/data/com +/dev/block/dm-10 on /data/data/com.termux type f2fs (rw,lazytime,seclabel,nosuid,nodev,noatime,background_gc=on,discard,no_h +eap,user_xattr,inline_xattr,acl,inline_data,inline_dentry,flush_merge,extent_cache,mode=adaptive,active_logs=6,reserve_root= +32768,resuid=0,resgid=1065,inlinecrypt,alloc_mode=default,fsync_mode=nobarrier) +/dev/block/dm-10 on /data/data/com.google.android.gms type f2fs (rw,lazytime,seclabel,nosuid,nodev,noatime,background_gc=on, +discard,no_heap,user_xattr,inline_xattr,acl,inline_data,inline_dentry,flush_merge,extent_cache,mode=adaptive,active_logs=6,r +eserve_root=32768,resuid=0,resgid=1065,inlinecrypt,alloc_mode=default,fsync_mode=nobarrier) + +~ $ uname -a +Linux localhost 4.14.190-perf-gb4109db-dirty #439 SMP PREEMPT Wed Apr 10 19:23:59 CST 2024 aarch64 Android +``` + +but installation failed with message similar to the one I get now if I try to install extracted git-annex: + +``` +~ $ git-annex.linux/runshell git-annex +Running on Android.. Tuning for optimal behavior. +/data/data/com.termux/files/home/git-annex.linux/bin/xargs: 2: exec: /data/data/com.termux/files/home/git-annex.linux/exe/xa +rgs: Permission denied +``` + +
+and here is via bash -x + +```shell +~ $ bash -x git-annex.linux/runshell git-annex ++ GIT_ANNEX_PACKAGE_INSTALL= ++ set -e ++ orig_IFS=' +' ++ unset IFS +++ uname -o ++ os=Android +++ dirname git-annex.linux/runshell ++ base=git-annex.linux ++ '[' '!' -d git-annex.linux ']' ++ '[' '!' -e git-annex.linux/bin/git-annex ']' +++ pwd ++ orig=/data/data/com.termux/files/home ++ cd git-annex.linux +++ pwd ++ base=/data/data/com.termux/files/home/git-annex.linux ++ cd /data/data/com.termux/files/home ++ echo /data/data/com.termux/files/home/git-annex.linux ++ grep -q '[:;]' ++ tbase= ++ '[' -z '' ']' ++ '[' '!' -e /data/data/com.termux/files/home/.ssh/git-annex-shell ']' ++ '[' '!' -e /data/data/com.termux/files/home/.ssh/git-annex-wrapper ']' ++ GIT_ANNEX_APP_BASE=/data/data/com.termux/files/home/git-annex.linux ++ export GIT_ANNEX_APP_BASE ++ ORIG_PATH=/data/data/com.termux/files/usr/bin:/product/bin:/apex/com.android.runtime/bin:/apex/com.android.art/bin:/system +_ext/bin:/system/bin:/system/xbin:/odm/bin:/vendor/bin:/vendor/xbin ++ export ORIG_PATH ++ PATH=/data/data/com.termux/files/home/git-annex.linux/bin:/data/data/com.termux/files/usr/bin:/product/bin:/apex/com.andro +id.runtime/bin:/apex/com.android.art/bin:/system_ext/bin:/system/bin:/system/xbin:/odm/bin:/vendor/bin:/vendor/xbin:/data/da +ta/com.termux/files/home/git-annex.linux/extra ++ export PATH +++ cat /data/data/com.termux/files/home/git-annex.linux/libdirs ++ for lib in $(cat "$base/libdirs") ++ GIT_ANNEX_LD_LIBRARY_PATH=/data/data/com.termux/files/home/git-annex.linux//lib/aarch64-linux-gnu: ++ export GIT_ANNEX_LD_LIBRARY_PATH ++ GIT_ANNEX_DIR=/data/data/com.termux/files/home/git-annex.linux ++ export GIT_ANNEX_DIR ++ ORIG_GCONV_PATH= ++ export ORIG_GCONV_PATH +++ cat /data/data/com.termux/files/home/git-annex.linux/gconvdir ++ GCONV_PATH=/data/data/com.termux/files/home/git-annex.linux//usr/lib/aarch64-linux-gnu/gconv ++ export GCONV_PATH ++ ORIG_GIT_EXEC_PATH= ++ export ORIG_GIT_EXEC_PATH ++ GIT_EXEC_PATH=/data/data/com.termux/files/home/git-annex.linux/git-core ++ export GIT_EXEC_PATH ++ ORIG_GIT_TEMPLATE_DIR= ++ export ORIG_GIT_TEMPLATE_DIR ++ GIT_TEMPLATE_DIR=/data/data/com.termux/files/home/git-annex.linux/templates ++ export GIT_TEMPLATE_DIR ++ ORIG_MANPATH= ++ export ORIG_MANPATH ++ MANPATH=/data/data/com.termux/files/home/git-annex.linux/usr/share/man: ++ export MANPATH ++ unset LD_PRELOAD ++ ORIG_LOCPATH= ++ export ORIG_LOCPATH ++ '[' -z '' ']' ++ '[' -z '' ']' +++ cat /data/data/com.termux/files/home/git-annex.linux/buildid +++ echo /data/data/com.termux/files/home/git-annex.linux +++ tr / _ ++ locpathbase=fd647a438576b806711ffed585497e8d__data_data_com.termux_files_home_git-annex.linux +++ echo fd647a438576b806711ffed585497e8d__data_data_com.termux_files_home_git-annex.linux +++ md5sum +++ cut -d ' ' -f 1 ++ locpathmd5=01ac43587c3dc29b384e1f7525ef0eb8 ++ '[' -n 01ac43587c3dc29b384e1f7525ef0eb8 ']' ++ locpathbase=01ac43587c3dc29b384e1f7525ef0eb8 ++ LOCPATH=/data/data/com.termux/files/home/.cache/git-annex/locales/01ac43587c3dc29b384e1f7525ef0eb8 ++ export LOCPATH ++ for localecache in $HOME/.cache/git-annex/locales/* +++ cat /data/data/com.termux/files/home/.cache/git-annex/locales/01ac43587c3dc29b384e1f7525ef0eb8/base ++ cachebase=/data/data/com.termux/files/home/git-annex.linux ++ '[' '!' -d /data/data/com.termux/files/home/git-annex.linux ']' +++ cat /data/data/com.termux/files/home/.cache/git-annex/locales/01ac43587c3dc29b384e1f7525ef0eb8/buildid +++ cat /data/data/com.termux/files/home/git-annex.linux/buildid ++ '[' fd647a438576b806711ffed585497e8d '!=' fd647a438576b806711ffed585497e8d ']' ++ '[' '!' -d /data/data/com.termux/files/home/.cache/git-annex/locales/01ac43587c3dc29b384e1f7525ef0eb8 ']' ++ useproot= ++ case "$os" in ++ '[' -e /data/data/com.termux/files/home/git-annex.linux/git ']' ++ echo 'Running on Android.. Tuning for optimal behavior.' +Running on Android.. Tuning for optimal behavior. ++ cd /data/data/com.termux/files/home/git-annex.linux ++ find . ++ grep git ++ grep -v git-annex ++ xargs rm -rf +/data/data/com.termux/files/home/git-annex.linux/bin/xargs: 2: exec: /data/data/com.termux/files/home/git-annex.linux/exe/xa +rgs: Permission denied ++ grep -v git-remote-gcrypt ++ grep -v git-remote-tor-annex +``` + +
+ +and my foo here is not strong enough to deduce what it is really not happy about + +### What steps will reproduce the problem? + +- get boox +- try to install git-annex from termux + From 9ce207532e06a9b844f0d066629c81ff627bceb3 Mon Sep 17 00:00:00 2001 From: "benjamin.poldrack@d09ccff6d42dd20277610b59867cf7462927b8e3" Date: Thu, 11 Jul 2024 07:23:30 +0000 Subject: [PATCH 02/30] --- ...copy__95__file__95__range_for_get_and_copy.mdwn | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/todo/use_copy__95__file__95__range_for_get_and_copy.mdwn diff --git a/doc/todo/use_copy__95__file__95__range_for_get_and_copy.mdwn b/doc/todo/use_copy__95__file__95__range_for_get_and_copy.mdwn new file mode 100644 index 0000000000..1e056135df --- /dev/null +++ b/doc/todo/use_copy__95__file__95__range_for_get_and_copy.mdwn @@ -0,0 +1,14 @@ +I was looking into why `annex get` and `annex copy` between local clones on NFS mounts don't utilize NFS4.2's server-side copy feature, +which would be pretty relevant for a setup like ours (institutional compute cluster; big datalad datasets). +This seems to boil down to not calling `copy_file_range`. However, `cp` generally does call `copy_file_range`, so that seemed confusing. +Turns out the culprit is `--reflink=always` which does not work as expected on ZFS. `--reflink=auto` does, though. +A summary of how that is, can be found here: https://github.com/openzfs/zfs/pull/13392#issuecomment-1742172842 + +I am not sure why annex insists on `always` rather than `auto`, so not sure whether the solution actually would be to change that. +Reading old issues it seems the reason was to let annex handle the fallback, which kinda is the problem in case of ZFS. +Changing (back?) to `auto` may be an issue in other cases - I don't know. If annex' fallback when `cp --reflink=always` fails would end up calling `copy_file_range`, that would still solve the issue, though, as NFS would then end up performing a server-side copy rather than transferring big files back and forth. + +Just to be clear: It's specifically ZFS via NFS on linux that's the issue here. + + +P.S.: Didn't want to call this a bug, mostly b/c the "real bug" isn't in annex exactly (see link above), so putting it here. From a82a573f75d596cc6bb318a874d1b06d5df78362 Mon Sep 17 00:00:00 2001 From: "benjamin.poldrack@d09ccff6d42dd20277610b59867cf7462927b8e3" Date: Thu, 11 Jul 2024 07:47:27 +0000 Subject: [PATCH 03/30] --- doc/todo/use_copy__95__file__95__range_for_get_and_copy.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/use_copy__95__file__95__range_for_get_and_copy.mdwn b/doc/todo/use_copy__95__file__95__range_for_get_and_copy.mdwn index 1e056135df..46dcf4d65d 100644 --- a/doc/todo/use_copy__95__file__95__range_for_get_and_copy.mdwn +++ b/doc/todo/use_copy__95__file__95__range_for_get_and_copy.mdwn @@ -12,3 +12,5 @@ Just to be clear: It's specifically ZFS via NFS on linux that's the issue here. P.S.: Didn't want to call this a bug, mostly b/c the "real bug" isn't in annex exactly (see link above), so putting it here. + +[[!meta author=ben]] From ed11ce6fcb6da651be44f5e6f328df586d150f1b Mon Sep 17 00:00:00 2001 From: "ashton@37fa3fec6d2eef022a3491c85362a34141fbf0db" Date: Fri, 12 Jul 2024 08:08:22 +0000 Subject: [PATCH 04/30] --- ...indows_host_to_a_Windows_remote__63__.mdwn | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn diff --git a/doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn b/doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn new file mode 100644 index 0000000000..f4e69bba46 --- /dev/null +++ b/doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn @@ -0,0 +1,26 @@ +I have a (hopefully) fairly common situation but I can't find anything concrete about what's recommended. + +I have two Windows machines (both running Windows 11, with git-annex installed). They can talk to each other over SMB. + +On each machine, I have a mapped network share pointing to the other machine's annex directory: + +Machine 1: + D:\ -> \\Machine2\annex +Machine 2: + E:\ -> \\Machine1\annex + +Machine 1 is the "primary" in that it has all of the data available. Machine 2 is transient and may not necessarily have a copy of all the data. + +On machine 2, I set it up like this (cloning on machine 1): + +git clone E:\ + +On machine 1, I added D:\ (pointing to machine 2) as a remote. + +This works, but I wonder if there's a better way given the constraints I have: + +- I tried to get ssh working, it was a mess. One machine is AzureAD enrolled, while the other is not which means authentication did not work well at all. I tried to get a third-party SSH server running (Bitvise) with a local account or a "virtual" account, but never quite got there. + +- I cannot run WSL. Well, I can. But I'd prefer not to because it's an obstruction/extra hurdle for some things I'm doing. + +How do others accomplish this? From 60eae008d8e858f2783e32452d27d09f24445662 Mon Sep 17 00:00:00 2001 From: "ashton@37fa3fec6d2eef022a3491c85362a34141fbf0db" Date: Fri, 12 Jul 2024 08:11:30 +0000 Subject: [PATCH 05/30] --- doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn b/doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn index f4e69bba46..2dff3ec2fb 100644 --- a/doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn +++ b/doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn @@ -6,6 +6,8 @@ On each machine, I have a mapped network share pointing to the other machine's a Machine 1: D:\ -> \\Machine2\annex + + Machine 2: E:\ -> \\Machine1\annex From af4d90eea86cac5fd5fc0702d65b33bd096f7634 Mon Sep 17 00:00:00 2001 From: "ashton@37fa3fec6d2eef022a3491c85362a34141fbf0db" Date: Fri, 12 Jul 2024 08:11:56 +0000 Subject: [PATCH 06/30] --- doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn b/doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn index 2dff3ec2fb..1c7664c92f 100644 --- a/doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn +++ b/doc/forum/Windows_host_to_a_Windows_remote__63__.mdwn @@ -13,7 +13,7 @@ Machine 2: Machine 1 is the "primary" in that it has all of the data available. Machine 2 is transient and may not necessarily have a copy of all the data. -On machine 2, I set it up like this (cloning on machine 1): +On machine 2, I set it up like this (cloning from machine 1): git clone E:\ From df97c82c44484993cf4062bb17b475016a48097f Mon Sep 17 00:00:00 2001 From: xentac Date: Fri, 12 Jul 2024 23:37:36 +0000 Subject: [PATCH 07/30] --- doc/bugs/S3_bucket_not_configured.mdwn | 45 ++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 doc/bugs/S3_bucket_not_configured.mdwn diff --git a/doc/bugs/S3_bucket_not_configured.mdwn b/doc/bugs/S3_bucket_not_configured.mdwn new file mode 100644 index 0000000000..25b04fd09b --- /dev/null +++ b/doc/bugs/S3_bucket_not_configured.mdwn @@ -0,0 +1,45 @@ +### Please describe the problem. +I had a working git-annex repo (simple, one remote full clone on a separate machine) that worked fine with assistant and everything. I added a backblaze S3 special remote and now the assistant gives me an Internal Server Error with the message "S3 bucket not configured". The git annex command line seems to not care about this situation (I can push to the remote, it's enabled, etc etc). + +### What steps will reproduce the problem? +I used the command + +AWS_ACCESS_KEY_ID=SECRET AWS_SECRET_ACCESS_KEY=SECRET git annex initremote backblaze type=S3 signature=v4 host=s3.us-west-002.backblazeb2.com encryption=shared bucket=annex-calibre protocol=https + +to create the special remote. I was pretty proud of myself that I got it first try without messing up. + + +### What version of git-annex are you using? On what operating system? +xentac@baxter:~/calibre$ git annex version +git-annex version: 10.20240430-1~ndall+1 +build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV +dependency versions: aws-0.22.1 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.1 ghc-9.0.2 http-client-0.7.13.1 persistent-sqlite-2.13.1.0 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.2.1 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL VURL X* +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg rclone hook external +operating system: linux x86_64 +supported repository versions: 8 9 10 +upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10 +local repository version: 10 + +Running on Ubuntu 24.04 LTS + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + +The only relevant daemon.log lines look like this: + +ConfigMonitor crashed: S3 bucket not configured + +12/Jul/2024:12:47:05 -1000 [Error#yesod-core] S3 bucket not configured @(yesod-core-1.6.24.0-BAaAxHVEp0K8WBRS1ADQQK:Yesod.Core.Class.Yesod src/Yesod/Core/Class/Yesod.hs:705:6) + + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +I've successfully used git-annex (on and off) since you ran the kickstarter to develop the assistant :) I really like the project. + From 8102bf5fe2b256d968f072ad6fc6249bab986971 Mon Sep 17 00:00:00 2001 From: xentac Date: Fri, 12 Jul 2024 23:49:33 +0000 Subject: [PATCH 08/30] Added a comment --- .../comment_1_a9296f75b7392f4075ba0e39b6774262._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/S3_bucket_not_configured/comment_1_a9296f75b7392f4075ba0e39b6774262._comment diff --git a/doc/bugs/S3_bucket_not_configured/comment_1_a9296f75b7392f4075ba0e39b6774262._comment b/doc/bugs/S3_bucket_not_configured/comment_1_a9296f75b7392f4075ba0e39b6774262._comment new file mode 100644 index 0000000000..215434cc85 --- /dev/null +++ b/doc/bugs/S3_bucket_not_configured/comment_1_a9296f75b7392f4075ba0e39b6774262._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="xentac" + avatar="http://cdn.libravatar.org/avatar/773b6c7b0dc34f10b66aa46d2730a5b3" + subject="comment 1" + date="2024-07-12T23:49:33Z" + content=""" +I may have just fixed it by restarting the daemon from the webapp... Still weird that the webapp got into that stuck state. +"""]] From 8ec34ea2a1aeecb4b0b57e4da3b3443be2bdbc8f Mon Sep 17 00:00:00 2001 From: "m.risse@77eac2c22d673d5f10305c0bade738ad74055f92" Date: Mon, 15 Jul 2024 14:57:26 +0000 Subject: [PATCH 09/30] --- ..._message_when_cloning_shared_git_repo.mdwn | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 doc/bugs/Wrong_error_message_when_cloning_shared_git_repo.mdwn diff --git a/doc/bugs/Wrong_error_message_when_cloning_shared_git_repo.mdwn b/doc/bugs/Wrong_error_message_when_cloning_shared_git_repo.mdwn new file mode 100644 index 0000000000..905b528ad1 --- /dev/null +++ b/doc/bugs/Wrong_error_message_when_cloning_shared_git_repo.mdwn @@ -0,0 +1,62 @@ +### Please describe the problem. + +When cloning a repository via SSH that is shared on the remote system (e.g. belongs to a different account), git-annex produces a wrong/misleading error message. + + +### What steps will reproduce the problem? + +1. Create a shared git repository with one account +2. With another account, ensure that git refuses to do anything with the repository: `git annex info` should lead to a "git-annex: Git refuses to operate in this repository" error +3. Clone the repository via ssh with that other account +4. Run `git annex init` in that clone +5. Notice a misleading error message: + +``` +$ git annex init +init + Unable to parse git config from origin + + Remote origin does not have git-annex installed; setting annex-ignore + + This could be a problem with the git-annex installation on the remote. Please make sure that git-annex-shell is available in PATH when you ssh into the remote. Once you have fixed the git-annex installation, run: git annex enableremote origin +ok +(recording state in git...) +``` + +The error will go away once the safe.directory config is set for the repository: + +``` +git config --global --add safe.directory +``` + +This command is correctly suggested in the output of `git annex info` from the above step 2, but since one doesn't always run that first and instead just tries to clone, the misleading error message can cause a bit of wasted time trying to figure out why git-annex wouldn't be available in the PATH when in reality it is. + +### What version of git-annex are you using? On what operating system? + +``` +git-annex version: 10.20240430 +build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite S3 WebDAV +dependency versions: aws-0.24.1 bloomfilter-2.0.1.2 crypton-0.34 DAV-1.3.4 feed-1.3.2.1 ghc-9.6.5 http-client-0.7.17 persistent-sqlite-2.13.3.0 torrent-10000.1.3 uuid-1.3.15 yesod-1.6.2.1 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL VURL X* +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg rclone hook external +operating system: linux x86_64 +supported repository versions: 8 9 10 +upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10 +local repository version: 10 +``` +on Ubuntu installed from nixpkgs. + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + From cce86415b10d169821d4bde5fd48d32eed4e939d Mon Sep 17 00:00:00 2001 From: "m.risse@77eac2c22d673d5f10305c0bade738ad74055f92" Date: Mon, 15 Jul 2024 15:18:28 +0000 Subject: [PATCH 10/30] Added a comment --- ...nt_1_882cff9febf99518aeaa6a11b86dacf9._comment | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 doc/bugs/git_annex_copy_just_does_not_copy_sometimes/comment_1_882cff9febf99518aeaa6a11b86dacf9._comment diff --git a/doc/bugs/git_annex_copy_just_does_not_copy_sometimes/comment_1_882cff9febf99518aeaa6a11b86dacf9._comment b/doc/bugs/git_annex_copy_just_does_not_copy_sometimes/comment_1_882cff9febf99518aeaa6a11b86dacf9._comment new file mode 100644 index 0000000000..c10b8c4c28 --- /dev/null +++ b/doc/bugs/git_annex_copy_just_does_not_copy_sometimes/comment_1_882cff9febf99518aeaa6a11b86dacf9._comment @@ -0,0 +1,15 @@ +[[!comment format=mdwn + username="m.risse@77eac2c22d673d5f10305c0bade738ad74055f92" + nickname="m.risse" + avatar="http://cdn.libravatar.org/avatar/59541f50d845e5f81aff06e88a38b9de" + subject="comment 1" + date="2024-07-15T15:18:28Z" + content=""" +Notice that `git annex findkeys --not --in origin-storage` will list all keys that are locally available, but not in `origin-storage`, while `git annex copy --not --in origin-storage --to origin-storage` will copy over all locally available files not in `origin-storage` _that are part of the currently checked out worktree_. I.e. one works on keys, while the other works on paths. + +This means your `findkeys` into `copy` pipe is not equivalent to the plain copy command. + +Instead, what the copy command does copy is what `git annex find --not --in origin-storage` would return. + +More concretely, if no file in the current worktree points to `MD5E-s7265--9885654f68b8e72de9b681c8783b3bf8.yaml` then what you observe is expected. If such a file does exist, then this is indeed a bug, I think. +"""]] From a79176341db46dfc80fceb47d6a7665cda450348 Mon Sep 17 00:00:00 2001 From: nobodyinperson Date: Mon, 15 Jul 2024 18:32:36 +0000 Subject: [PATCH 11/30] Added a comment --- ...ment_1_0105cb95838b2ba6ce1bc3780ae649b6._comment | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/bugs/Wrong_error_message_when_cloning_shared_git_repo/comment_1_0105cb95838b2ba6ce1bc3780ae649b6._comment diff --git a/doc/bugs/Wrong_error_message_when_cloning_shared_git_repo/comment_1_0105cb95838b2ba6ce1bc3780ae649b6._comment b/doc/bugs/Wrong_error_message_when_cloning_shared_git_repo/comment_1_0105cb95838b2ba6ce1bc3780ae649b6._comment new file mode 100644 index 0000000000..e1bd3b4283 --- /dev/null +++ b/doc/bugs/Wrong_error_message_when_cloning_shared_git_repo/comment_1_0105cb95838b2ba6ce1bc3780ae649b6._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="nobodyinperson" + avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5" + subject="comment 1" + date="2024-07-15T18:32:36Z" + content=""" +possibly related: + +- https://git-annex.branchable.com/forum/When_--git-dir_is_not_in_--work-tree/ +- https://git-annex.branchable.com/bugs/enableremote_fails_with___34__wrong_reason__34___stated/ + +As a sidenote, I have been having nothing but pain and suffering with sharing git repos between users. The only viable way for me was to always access a git repo with the same user, otherwise it's been permission hell. +"""]] From 3590a17f9e18243d345ba99beab25205c6aad1ab Mon Sep 17 00:00:00 2001 From: "m.risse@77eac2c22d673d5f10305c0bade738ad74055f92" Date: Tue, 16 Jul 2024 09:21:54 +0000 Subject: [PATCH 12/30] Added a comment --- .../comment_1_1f06e694a186d8801730f1f6cc48a995._comment | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/design/p2p_protocol_over_http/comment_1_1f06e694a186d8801730f1f6cc48a995._comment diff --git a/doc/design/p2p_protocol_over_http/comment_1_1f06e694a186d8801730f1f6cc48a995._comment b/doc/design/p2p_protocol_over_http/comment_1_1f06e694a186d8801730f1f6cc48a995._comment new file mode 100644 index 0000000000..543e359ad0 --- /dev/null +++ b/doc/design/p2p_protocol_over_http/comment_1_1f06e694a186d8801730f1f6cc48a995._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="m.risse@77eac2c22d673d5f10305c0bade738ad74055f92" + nickname="m.risse" + avatar="http://cdn.libravatar.org/avatar/59541f50d845e5f81aff06e88a38b9de" + subject="comment 1" + date="2024-07-16T09:21:54Z" + content=""" +I just want to say that this would be nice to have for https://codeberg.org/matrss/forgejo-aneksajo as well, since only being able to use ssh is a pain point in some places (e.g. outgoing ssh connections are forbidden on the HPC systems at FZ Jülich, which means an intermediary is necessary to copy data from the HPC systems to a forgejo-aneksajo instance, currently). Being able to read and write via https (possibly reusing the existing access tokens from forgejo? That would be on me to see if it is doable) would alleviate this problem entirely. +"""]] From 5bc00a55dd413b1e9c9a8646e5d5de6cd3ff5e28 Mon Sep 17 00:00:00 2001 From: mih Date: Tue, 16 Jul 2024 15:02:46 +0000 Subject: [PATCH 13/30] --- doc/todo/Read-only_support_for_webdav.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/todo/Read-only_support_for_webdav.mdwn diff --git a/doc/todo/Read-only_support_for_webdav.mdwn b/doc/todo/Read-only_support_for_webdav.mdwn new file mode 100644 index 0000000000..c09271775f --- /dev/null +++ b/doc/todo/Read-only_support_for_webdav.mdwn @@ -0,0 +1,7 @@ +This is in response to https://git-annex.branchable.com/special_remotes/webdav/#comment-cd53cf0276427cc924ae66553985ec5c where `httpalso` is recommended as an approach to get read-only access to a `webdav` remote. + +A use case not possible with that approach is *authenticated* read-only access. According to http://git-annex.branchable.com/special_remotes/httpalso/#comment-4f41f401d4b0133d2ef12912b9217e44 this is not supported right now, but could be added. + +Weighing the two approaches (read-only `webdav` vs authenticated `httpalso`), it appears that only the read-only `webdav` is compatible with [git-remote-annex](https://git-annex.branchable.com/git-remote-annex/), because a user would need to declare *one* special remote (configuration) to use for cloning. + +It would be great to have authenticated, read-only access to webdav shares. Thanks in advance for considering! From 1287e4590f2a948ce54df708d1ef443b6303ae1a Mon Sep 17 00:00:00 2001 From: "m.risse@77eac2c22d673d5f10305c0bade738ad74055f92" Date: Tue, 16 Jul 2024 15:42:54 +0000 Subject: [PATCH 14/30] --- doc/related_software.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/related_software.mdwn b/doc/related_software.mdwn index 6dda61df3f..44580bbf30 100644 --- a/doc/related_software.mdwn +++ b/doc/related_software.mdwn @@ -69,4 +69,6 @@ designed to interoperate with it. converts a sequence of borgbackup archives into a git-annex repository, storing nested Git repositories as subtrees or bundles. +* [forgejo-aneksajo](https://codeberg.org/matrss/forgejo-aneksajo) is a soft-fork of Forgejo (a git forge) that integrates support for git-annex. + See also [[not]] for software that is *not* related to git-annex, but similar. From ba4d54577646f006e1ec9581b8e783088c86543d Mon Sep 17 00:00:00 2001 From: yarikoptic Date: Tue, 16 Jul 2024 15:58:50 +0000 Subject: [PATCH 15/30] reporting FTBFS on windows --- ...ws_build_has_been_failing_for_a_while.mdwn | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 doc/bugs/Windows_build_has_been_failing_for_a_while.mdwn diff --git a/doc/bugs/Windows_build_has_been_failing_for_a_while.mdwn b/doc/bugs/Windows_build_has_been_failing_for_a_while.mdwn new file mode 100644 index 0000000000..195f0daa4a --- /dev/null +++ b/doc/bugs/Windows_build_has_been_failing_for_a_while.mdwn @@ -0,0 +1,42 @@ +### Please describe the problem. + +As pinged via email, our daily builds has been failing for a while (about two weeks) on Windows. Citing from the [most recent build](https://github.com/datalad/git-annex/actions/runs/9933338199/job/27435937101) + +``` +[426 of 720] Compiling Annex.Content +D:\a\git-annex\git-annex\Annex\Content.hs:186:17: error: parse error on input `exlocker' +[468 of 720] Compiling Assistant.Gpg + | +186 | exlocker >>= \case +[469 of 720] Compiling Annex.Environment + | ^^^^^^^^ +[628 of 720] Compiling Utility.Verifiable +[630 of 720] Compiling Assistant.Pairing +[631 of 720] Compiling Assistant.Types.DaemonStatus +[655 of 720] Compiling Utility.WebApp +[656 of 720] Compiling Utility.Yesod +runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\bin (Win32 error 3): The system cannot find the path specified. +ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\lib (Win32 error 3): The system cannot find the path specified. +ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\bin (Win32 error 3): The system cannot find the path specified. +ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\lib (Win32 error 3): The system cannot find the path specified. +ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\bin (Win32 error 3): The system cannot find the path specified. +ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\lib (Win32 error 3): The system cannot find the path specified. +ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\bin (Win32 error 3): The system cannot find the path specified. +ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\lib (Win32 error 3): The system cannot find the path specified. +ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\bin (Win32 error 3): The system cannot find the path specified. +ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\lib (Win32 error 3): The system cannot find the path specified. +ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\bin (Win32 error 3): The system cannot find the path specified. + +Error: [S-7282] + Stack failed to execute the build plan. + + While executing the build plan, Stack encountered the error: + + [S-7011] + While building package git-annex-10.20240701 (scroll up to its section to see the error) + using: + D:\a\git-annex\git-annex\.stack-work\dist\274b403a\setup\setup --verbose=1 --builddir=.stack-work\dist\274b403a build exe:git-annex --ghc-options "" + Process exited with code: ExitFailure 1 +``` + + From b1db5115e0ea8b8d8e2ba94710478c74678efaed Mon Sep 17 00:00:00 2001 From: "m.risse@77eac2c22d673d5f10305c0bade738ad74055f92" Date: Wed, 17 Jul 2024 14:07:32 +0000 Subject: [PATCH 16/30] Added a comment --- ...omment_2_9e1d3ebf2d52e8ea7b177f121ef6c70c._comment | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/Wrong_error_message_when_cloning_shared_git_repo/comment_2_9e1d3ebf2d52e8ea7b177f121ef6c70c._comment diff --git a/doc/bugs/Wrong_error_message_when_cloning_shared_git_repo/comment_2_9e1d3ebf2d52e8ea7b177f121ef6c70c._comment b/doc/bugs/Wrong_error_message_when_cloning_shared_git_repo/comment_2_9e1d3ebf2d52e8ea7b177f121ef6c70c._comment new file mode 100644 index 0000000000..fa4855f08c --- /dev/null +++ b/doc/bugs/Wrong_error_message_when_cloning_shared_git_repo/comment_2_9e1d3ebf2d52e8ea7b177f121ef6c70c._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="m.risse@77eac2c22d673d5f10305c0bade738ad74055f92" + nickname="m.risse" + avatar="http://cdn.libravatar.org/avatar/59541f50d845e5f81aff06e88a38b9de" + subject="comment 2" + date="2024-07-17T14:07:32Z" + content=""" +Thanks for the hints. Seeing that is marked as fixed prompted me to check again and it turns out that I reported the wrong git-annex version above: while that is the local version, the remote system has 10.20230626-g8594d49 (the latest one available from conda-forge, unfortunately that is rather old now), and it makes sense that the relevant version in this case is the one running on the remote system. + +So I guess this is fixed already, my git-annex is just too old (although I haven't double checked, I don't see a convenient way right now to get a more recent git-annex version onto that remote system -- HPC, no root). +"""]] From a2ab2f70ea5616525182541ef0725337343c529b Mon Sep 17 00:00:00 2001 From: "m.risse@77eac2c22d673d5f10305c0bade738ad74055f92" Date: Fri, 19 Jul 2024 08:26:31 +0000 Subject: [PATCH 17/30] Added a comment --- ..._a5620a692a4e2f5cef4c2272e2285f41._comment | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/forum/import_and_export_treeish_for_rsync_and_webdav/comment_2_a5620a692a4e2f5cef4c2272e2285f41._comment diff --git a/doc/forum/import_and_export_treeish_for_rsync_and_webdav/comment_2_a5620a692a4e2f5cef4c2272e2285f41._comment b/doc/forum/import_and_export_treeish_for_rsync_and_webdav/comment_2_a5620a692a4e2f5cef4c2272e2285f41._comment new file mode 100644 index 0000000000..6ae2e8a26f --- /dev/null +++ b/doc/forum/import_and_export_treeish_for_rsync_and_webdav/comment_2_a5620a692a4e2f5cef4c2272e2285f41._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="m.risse@77eac2c22d673d5f10305c0bade738ad74055f92" + nickname="m.risse" + avatar="http://cdn.libravatar.org/avatar/59541f50d845e5f81aff06e88a38b9de" + subject="comment 2" + date="2024-07-19T08:26:30Z" + content=""" +> If you're importing and exporting to the same remote, what happens when there's a conflict? Eg, whatever else is writing files to the remote writes to a file, but you locally modify the same file, and export a tree, without importing the new version first. That would overwrite the modified file on the remote, losing data. + +The man page for git-annex-export states that this shouldn't be a problem, or at least shouldn't lead to data loss (might still require manual intervention): + +> When a file on a special remote has been modified by software other than git-annex, exporting to it will not overwrite the modified file, and the export will not succeed. You can resolve this conflict by using git annex import. + +Maybe this was improved after this forum post happened? + +There is probably some potential for issues when exporting and writing with another program _at the same time_, although that might be mitigated with webdav locks, for the webdav case. Also, you state that this is already a problem with the remotes that support export and import now. + +Is this concern outdated? I would love to be able to import and export to webdav, so that I could use a Nextcloud as a \"frontend\" to a git-annex repository, getting the \"best of both worlds\" so to speak. +"""]] From 287834335405cd2d4d414ff5676a88fecd189160 Mon Sep 17 00:00:00 2001 From: "m.risse@77eac2c22d673d5f10305c0bade738ad74055f92" Date: Fri, 19 Jul 2024 12:12:56 +0000 Subject: [PATCH 18/30] --- ...Peer_to_peer_connection_purely_over_magic-wormhole.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/todo/Peer_to_peer_connection_purely_over_magic-wormhole.mdwn diff --git a/doc/todo/Peer_to_peer_connection_purely_over_magic-wormhole.mdwn b/doc/todo/Peer_to_peer_connection_purely_over_magic-wormhole.mdwn new file mode 100644 index 0000000000..2113d3ffa0 --- /dev/null +++ b/doc/todo/Peer_to_peer_connection_purely_over_magic-wormhole.mdwn @@ -0,0 +1,7 @@ +Currently peer to peer communication seems to be possible only over tor (requiring root privileges to setup). It would be great to have an alternative connection method that can easily be used as an unprivileged user as well. + +Magic-wormhole has an experimental feature called "dilation" () which can be used to open a direct bidirectional TCP connection between two systems only using the usual magic-wormhole codes (which can be generated once and re-used, so essentially like a pre-shared key stored on each side). + +There is a project called fowl () that uses this feature to port-forward over such a tunnel, which could be used for this purpose or serve as a reference for how to use this feature in git-annex. This implementation has some issues, but I think the approach has potential. + +It would be great if `git annex remotedaemon` (I suppose? I am not too well-versed on the internals) could optionally be configured to establish such a tunnel to remotes and use it for communication. Or maybe this is already possible to implement from outside of git-annex and I just need a hint on how to do that? From 8a7fc275cb6941af48c782df88dc71effe4713de Mon Sep 17 00:00:00 2001 From: kdm9 Date: Fri, 19 Jul 2024 13:11:05 +0000 Subject: [PATCH 19/30] Added a comment --- ..._c3d33e26fbefbef400f219a542cff7fb._comment | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/bugs/__34__Missing_location__34___with_partsize__39__d_uploads/comment_2_c3d33e26fbefbef400f219a542cff7fb._comment diff --git a/doc/bugs/__34__Missing_location__34___with_partsize__39__d_uploads/comment_2_c3d33e26fbefbef400f219a542cff7fb._comment b/doc/bugs/__34__Missing_location__34___with_partsize__39__d_uploads/comment_2_c3d33e26fbefbef400f219a542cff7fb._comment new file mode 100644 index 0000000000..5964e72df4 --- /dev/null +++ b/doc/bugs/__34__Missing_location__34___with_partsize__39__d_uploads/comment_2_c3d33e26fbefbef400f219a542cff7fb._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="kdm9" + avatar="http://cdn.libravatar.org/avatar/b7b736335a0e9944a8169a582eb4c43d" + subject="comment 2" + date="2024-07-19T13:11:05Z" + content=""" +For future google searchers: + +When interfacing with Ceph storage via the S3 backend, I get errors like the following on larger files + +`XmlException {xmlErrorMessage = \"Missing ETag\"}` + + +Like above, these 'errors' are actually successes with a non-compliant S3 backend that is missing either the Location or Etag file. + + +I confirm that setting partsize > chunk works around this issue, in my case `chunk=4GiB partsize=5GiB`. + +Best, +Kevin +"""]] From b920655acda7c35521397bd2068894919d16b492 Mon Sep 17 00:00:00 2001 From: nobodyinperson Date: Fri, 19 Jul 2024 15:21:19 +0000 Subject: [PATCH 20/30] Added a comment: Also Serveo.net --- .../comment_1_c1e4564192cb1887ff17443a327556d7._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/todo/Peer_to_peer_connection_purely_over_magic-wormhole/comment_1_c1e4564192cb1887ff17443a327556d7._comment diff --git a/doc/todo/Peer_to_peer_connection_purely_over_magic-wormhole/comment_1_c1e4564192cb1887ff17443a327556d7._comment b/doc/todo/Peer_to_peer_connection_purely_over_magic-wormhole/comment_1_c1e4564192cb1887ff17443a327556d7._comment new file mode 100644 index 0000000000..274b2ce52a --- /dev/null +++ b/doc/todo/Peer_to_peer_connection_purely_over_magic-wormhole/comment_1_c1e4564192cb1887ff17443a327556d7._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="nobodyinperson" + avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5" + subject="Also Serveo.net" + date="2024-07-19T15:21:18Z" + content=""" +Don't know your use case excatly, but just today I stumbled upon [Serveo.net](https://serveo.net), which facilitates no-login, no-install reverse SSH tunnels, with with you can effectively break out of NAT'ed networks without root privileges. Together with `autossh` for persistence, maybe that can help you? Although I guess for forwarding to a privileged port 22 you would still need root privileges, meh... +"""]] From b143cb686d4886589aa414a3b1dae748a5bf16a4 Mon Sep 17 00:00:00 2001 From: adehnert Date: Sun, 21 Jul 2024 01:04:44 +0000 Subject: [PATCH 21/30] Added a comment: `git annex sync --ff-only` --- ...omment_32_7a4146e6cc4ae8546cc6db84d4987bf1._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/sync/comment_32_7a4146e6cc4ae8546cc6db84d4987bf1._comment diff --git a/doc/sync/comment_32_7a4146e6cc4ae8546cc6db84d4987bf1._comment b/doc/sync/comment_32_7a4146e6cc4ae8546cc6db84d4987bf1._comment new file mode 100644 index 0000000000..751dc656bd --- /dev/null +++ b/doc/sync/comment_32_7a4146e6cc4ae8546cc6db84d4987bf1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="adehnert" + avatar="http://cdn.libravatar.org/avatar/a4fc9cc6278919b6f40df8e3cc84355b" + subject="`git annex sync --ff-only`" + date="2024-07-21T01:04:44Z" + content=""" +It would be useful to have a `git annex sync --ff-only` option. I have an alias for `git pull --ff-only` that I use most of the time, and it seems like a `git annex` counterpart would be reasonable. If only one of my local repo and the remote repo have changed, I'm happy to resolve things automatically. If both have changed, then I'm going to want to think about what to do -- maybe rebase locally, maybe something else. Of course, I can manually check before running `git annex sync` or use `git pull --ff-only` myself, but especially with several remotes, that could take some effort, and this is what we have computers for. :) + +I guess there's a question of what to do when some remotes can be fast-forwarded to and others would need a merge. I think *think* my ideal behavior is that if some updates can't be done without merge commits, it doesn't update any branches. But it'd also be fine to do as many updates as it can without any merges. Or do some prefix of the fast-forward updates, and then error out when it gets to the first merge. Whichever of these apply, of course it should display something if it can't handle things with fast-forwards exclusively. +"""]] From 1ee30b29ee400f6ef02d976e077a209e8b7897e2 Mon Sep 17 00:00:00 2001 From: "m.risse@77eac2c22d673d5f10305c0bade738ad74055f92" Date: Sun, 21 Jul 2024 12:38:12 +0000 Subject: [PATCH 22/30] Added a comment --- ..._2_a457473353e2b8acf2afa86cec6ef9be._comment | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/todo/Peer_to_peer_connection_purely_over_magic-wormhole/comment_2_a457473353e2b8acf2afa86cec6ef9be._comment diff --git a/doc/todo/Peer_to_peer_connection_purely_over_magic-wormhole/comment_2_a457473353e2b8acf2afa86cec6ef9be._comment b/doc/todo/Peer_to_peer_connection_purely_over_magic-wormhole/comment_2_a457473353e2b8acf2afa86cec6ef9be._comment new file mode 100644 index 0000000000..5adc9546ee --- /dev/null +++ b/doc/todo/Peer_to_peer_connection_purely_over_magic-wormhole/comment_2_a457473353e2b8acf2afa86cec6ef9be._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="m.risse@77eac2c22d673d5f10305c0bade738ad74055f92" + nickname="m.risse" + avatar="http://cdn.libravatar.org/avatar/59541f50d845e5f81aff06e88a38b9de" + subject="comment 2" + date="2024-07-21T12:38:12Z" + content=""" +Forwarding to port 22 shouldn't require root, e.g. the mentioned fowl tunnel could also be used to tunnel a local port to a remote ssh server on port 22. You just cannot listen on a local privileged port, but that shouldn't be a problem. + +There are a bunch of \"tunnelers\" like serveo, e.g. ngrok and zrok, but the disadvantage of that is that it still requires a running ssh server. + +My imagined use-case would be something like two phones or laptops behind NAT without tor or a ssh daemon. I think with magic-wormhole's dilation feature it would be possible to make it so that you could run `git annex remotedaemon` or `git annex assistant` on one or both devices (after pairing) and have them communicate without any further setup required. + +Since magic-wormhole is already used for pairing it wouldn't even be a new dependency. + +Maybe this is already implementable from outside git-annex as a custom git-remote though, I'd have to take a look at what git-remote-tor-annex is really doing... +"""]] From 40c930a381c447cccbc25dc4d45f1d25f7ad4c65 Mon Sep 17 00:00:00 2001 From: adehnert Date: Sun, 21 Jul 2024 18:14:03 +0000 Subject: [PATCH 23/30] --- ...nex-adjust_cares_about_argument_order.mdwn | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn diff --git a/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn b/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn new file mode 100644 index 0000000000..b93517b05e --- /dev/null +++ b/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn @@ -0,0 +1,41 @@ +### Please describe the problem. + +If I run `git annex adjust --hide-missing --unlock`, that works. If I run `git annex adjust --unlock --hide-missing`, that tells me "Invalid option `--hide-missing'" and prints the whole list of git-annex commands (which itself makes it hard to see the more useful error). I would expect that the order of the options doesn't matter, and if it does, I'd get a `git annex adjust` specific usage message. + +### What steps will reproduce the problem? + +``` +$ git annex adjust --unlock --hide-missing +Invalid option `--hide-missing' + +Usage: git-annex COMMAND +[...] +$ git annex adjust --hide-missing --unlock +git-annex: Not in a git repository. +``` + +(Yes, I've also tried this in a git repo. That doesn't seem to change the error the first (IMO buggy) command does, though obviously the second command gives different results.) + +### What version of git-annex are you using? On what operating system? + +``` +git-annex version: 10.20240430 +build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite S3 WebDAV +dependency versions: aws-0.24.1 bloomfilter-2.0.1.2 crypton-0.34 DAV-1.3.4 feed-1.3.2.1 ghc-9.6.5 http-client-0.7.17 persistent-sqlite-2.13.3.0 torrent-10000.1.3 uuid-1.3.15 yesod-1.6.2.1 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL VURL X* +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg rclone hook external +operating system: linux x86_64 +supported repository versions: 8 9 10 +upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10 +``` + +Ubuntu. + +Same behavior occurs in `git annex` 10.20230926 with Nix-On-Droid (https://git-annex.branchable.com/Android/#index2h2) (which is where I actually want to use this, but is harder to file a bug from). + +### Please provide any additional information below. + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Yes, I've been really happy using it to manage a bunch of videos, where I only need some on my laptop at any given time. Way better than my previous "manually scp things around" strategy. From 024b331a4ba40bccab20775c43c52cfad76cabf5 Mon Sep 17 00:00:00 2001 From: adehnert Date: Sun, 21 Jul 2024 18:14:28 +0000 Subject: [PATCH 24/30] --- doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn b/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn index b93517b05e..ab13636124 100644 --- a/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn +++ b/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn @@ -14,7 +14,7 @@ $ git annex adjust --hide-missing --unlock git-annex: Not in a git repository. ``` -(Yes, I've also tried this in a git repo. That doesn't seem to change the error the first (IMO buggy) command does, though obviously the second command gives different results.) +(Yes, I've also tried this in a git repo. That doesn't seem to change the error the first (IMO buggy) command gives, though obviously the second command gives different results.) ### What version of git-annex are you using? On what operating system? From 264366f45d0f91d28f89323bf851f18377490a41 Mon Sep 17 00:00:00 2001 From: adehnert Date: Sun, 21 Jul 2024 18:15:11 +0000 Subject: [PATCH 25/30] --- doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn b/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn index ab13636124..4cbeb2bd06 100644 --- a/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn +++ b/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn @@ -1,6 +1,6 @@ ### Please describe the problem. -If I run `git annex adjust --hide-missing --unlock`, that works. If I run `git annex adjust --unlock --hide-missing`, that tells me "Invalid option `--hide-missing'" and prints the whole list of git-annex commands (which itself makes it hard to see the more useful error). I would expect that the order of the options doesn't matter, and if it does, I'd get a `git annex adjust` specific usage message. +If I run `git annex adjust --hide-missing --unlock`, that works. If I run `git annex adjust --unlock --hide-missing`, that tells me "Invalid option '--hide-missing'" and prints the whole list of git-annex commands (which itself makes it hard to see the more useful error). I would expect that the order of the options doesn't matter, and if it does, I'd get a `git annex adjust` specific usage message. ### What steps will reproduce the problem? From 2b96f62adac962d694d9127d5004e36486c91fff Mon Sep 17 00:00:00 2001 From: adehnert Date: Sun, 21 Jul 2024 18:17:11 +0000 Subject: [PATCH 26/30] --- doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn b/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn index 4cbeb2bd06..d568f5e027 100644 --- a/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn +++ b/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn @@ -35,6 +35,7 @@ Same behavior occurs in `git annex` 10.20230926 with Nix-On-Droid (https://git-a ### Please provide any additional information below. +This is admittedly arguable consistent with https://git-annex.branchable.com/git-annex-adjust/, which suggests `git annex adjust --unlock|--lock|--fix|--hide-missing [--unlock|--lock|--fix]|--unlock-present` as the syntax (notably with `--unlock` after `--hide-missing`), but it's pretty unusual for a Linux command, and even if the decision is made to have one order be an error, the error message should be easier to understand. ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) From 12bc3ca2a71fd2e6c4188f0d52661b39bc7fabbf Mon Sep 17 00:00:00 2001 From: adehnert Date: Sun, 21 Jul 2024 18:17:25 +0000 Subject: [PATCH 27/30] --- doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn b/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn index d568f5e027..ba563d816c 100644 --- a/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn +++ b/doc/bugs/git-annex-adjust_cares_about_argument_order.mdwn @@ -35,7 +35,7 @@ Same behavior occurs in `git annex` 10.20230926 with Nix-On-Droid (https://git-a ### Please provide any additional information below. -This is admittedly arguable consistent with https://git-annex.branchable.com/git-annex-adjust/, which suggests `git annex adjust --unlock|--lock|--fix|--hide-missing [--unlock|--lock|--fix]|--unlock-present` as the syntax (notably with `--unlock` after `--hide-missing`), but it's pretty unusual for a Linux command, and even if the decision is made to have one order be an error, the error message should be easier to understand. +This is admittedly arguably consistent with https://git-annex.branchable.com/git-annex-adjust/, which suggests `git annex adjust --unlock|--lock|--fix|--hide-missing [--unlock|--lock|--fix]|--unlock-present` as the syntax (notably with `--unlock` after `--hide-missing`), but it's pretty unusual for a Linux command, and even if the decision is made to have one order be an error, the error message should be easier to understand. ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) From 8eadd02b52cad7a82baf6046dc1c7523bef9bbdf Mon Sep 17 00:00:00 2001 From: adehnert Date: Sun, 21 Jul 2024 19:08:45 +0000 Subject: [PATCH 28/30] Added a comment: git-annex for managing music --- ...comment_7_b4e36d0cf1f8c9e505386c1dee3f6cbe._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/Android/comment_7_b4e36d0cf1f8c9e505386c1dee3f6cbe._comment diff --git a/doc/Android/comment_7_b4e36d0cf1f8c9e505386c1dee3f6cbe._comment b/doc/Android/comment_7_b4e36d0cf1f8c9e505386c1dee3f6cbe._comment new file mode 100644 index 0000000000..53b83e6568 --- /dev/null +++ b/doc/Android/comment_7_b4e36d0cf1f8c9e505386c1dee3f6cbe._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="adehnert" + avatar="http://cdn.libravatar.org/avatar/a4fc9cc6278919b6f40df8e3cc84355b" + subject="git-annex for managing music" + date="2024-07-21T19:08:45Z" + content=""" +I'm trying to use git-annex to manage music on my phone (surely a common-ish use case?), so I have a git-annex checkout under `/sdcard/Music/`. It sort of works? I did a clone under there (I think, it was a few weeks ago), it seemed to check out my files okay and set e.g. `core.symlinks = false` and `annex.sshcaching = true`. However, I keep getting warnings that I should run `git annex restage` or that some git hook didn't run because it wasn't executable. I'm also now using `git annex adjust --hide-missing --unlock` since I think locked files just don't work on exfat(?). Also, there's various characters that aren't supported that caused a lot of confusion when I was first setting it up... I think the Nix-on-Droid app-specific directory has a better filesystem, but I don't think my music player would be able to access that. + +Do folks have tips on using git-annex on Android? My suspicion is that because of the filesystem and maybe other things, there's a fair amount of details that don't apply to git-annex usage on a normal Linux machine, and I don't really understand Android, exfat, or git well enough to confidently compensate -- more of a howto or rundown of features/config that's likely to come up on Android would be really helpful for this page. +"""]] From 7d4045277a7a39c588c8f6d04f07b2d870e3e408 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 23 Jul 2024 21:02:31 -0400 Subject: [PATCH 29/30] bug --- ...orce_option_not_propagated_to_git-annex-transferrer.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/bugs/force_option_not_propagated_to_git-annex-transferrer.mdwn diff --git a/doc/bugs/force_option_not_propagated_to_git-annex-transferrer.mdwn b/doc/bugs/force_option_not_propagated_to_git-annex-transferrer.mdwn new file mode 100644 index 0000000000..e5473e68ca --- /dev/null +++ b/doc/bugs/force_option_not_propagated_to_git-annex-transferrer.mdwn @@ -0,0 +1,6 @@ +When downloading with annex.stalldetection configured, `git-annex +transferrer` is used. If a download is prevented by annex.diskreserve +setting, the message displayed says that --force will override the check. +But that doesn't work in this case. + +The --force option should be propagated to this command. --[[Joey]] From f7404a64c0ae1a7224f87c7dd3e1459801d7fdce Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 23 Jul 2024 21:16:56 -0400 Subject: [PATCH 30/30] Propagate --force to git-annex transferrer And other child processes. --- Annex/Path.hs | 6 +++++- CHANGELOG | 1 + ...orce_option_not_propagated_to_git-annex-transferrer.mdwn | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Annex/Path.hs b/Annex/Path.hs index aa51da1b58..c131ddba0f 100644 --- a/Annex/Path.hs +++ b/Annex/Path.hs @@ -85,7 +85,11 @@ gitAnnexChildProcess subcmd ps f a = do gitAnnexChildProcessParams :: String -> [CommandParam] -> Annex [CommandParam] gitAnnexChildProcessParams subcmd ps = do cps <- gitAnnexGitConfigOverrides - return (Param subcmd : cps ++ ps) + force <- Annex.getRead Annex.force + let cps' = if force + then Param "--force" : cps + else cps + return (Param subcmd : cps' ++ ps) gitAnnexGitConfigOverrides :: Annex [CommandParam] gitAnnexGitConfigOverrides = concatMap (\c -> [Param "-c", Param c]) diff --git a/CHANGELOG b/CHANGELOG index c14e78ed55..cf71820055 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,6 +6,7 @@ git-annex (10.20240702) UNRELEASED; urgency=medium git-annex remotedaemon is killed while locking a key to prevent its removal. * Added a dependency on clock. + * Propagate --force to git-annex transferrer. -- Joey Hess Tue, 02 Jul 2024 12:14:53 -0400 diff --git a/doc/bugs/force_option_not_propagated_to_git-annex-transferrer.mdwn b/doc/bugs/force_option_not_propagated_to_git-annex-transferrer.mdwn index e5473e68ca..ceb757e1a2 100644 --- a/doc/bugs/force_option_not_propagated_to_git-annex-transferrer.mdwn +++ b/doc/bugs/force_option_not_propagated_to_git-annex-transferrer.mdwn @@ -4,3 +4,5 @@ setting, the message displayed says that --force will override the check. But that doesn't work in this case. The --force option should be propagated to this command. --[[Joey]] + +> [[fixed|done]] --[[Joey]]