From 56c88653a736caa7b273e91e9e53cf9d514f2ee1 Mon Sep 17 00:00:00 2001 From: "65.254.22.34" <65.254.22.34@web> Date: Tue, 28 Jan 2025 17:09:30 +0000 Subject: [PATCH 01/14] poll vote (My phone (or MP3 player)) --- doc/design/assistant/polls/prioritizing_special_remotes.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn index 1307c892cd..5210056cfe 100644 --- a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn +++ b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn @@ -6,7 +6,7 @@ locally paired systems, and remote servers with rsync. Help me prioritize my work: What special remote would you most like to use with the git-annex assistant? -[[!poll open=yes 18 "Amazon S3 (done)" 13 "Amazon Glacier (done)" 10 "Box.com (done)" 79 "My phone (or MP3 player)" 29 "Tahoe-LAFS" 19 "OpenStack SWIFT" 37 "Google Drive"]] +[[!poll open=yes 18 "Amazon S3 (done)" 13 "Amazon Glacier (done)" 10 "Box.com (done)" 80 "My phone (or MP3 player)" 29 "Tahoe-LAFS" 19 "OpenStack SWIFT" 37 "Google Drive"]] This poll is ordered with the options I consider easiest to build listed first. Mostly because git-annex already supports them and they From 1e5c354720c750f9775c6a2d61c51d795c851247 Mon Sep 17 00:00:00 2001 From: "65.254.22.34" <65.254.22.34@web> Date: Tue, 28 Jan 2025 17:09:37 +0000 Subject: [PATCH 02/14] poll vote (My phone (or MP3 player)) --- doc/design/assistant/polls/prioritizing_special_remotes.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn index 5210056cfe..2c6d62e9c2 100644 --- a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn +++ b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn @@ -6,7 +6,7 @@ locally paired systems, and remote servers with rsync. Help me prioritize my work: What special remote would you most like to use with the git-annex assistant? -[[!poll open=yes 18 "Amazon S3 (done)" 13 "Amazon Glacier (done)" 10 "Box.com (done)" 80 "My phone (or MP3 player)" 29 "Tahoe-LAFS" 19 "OpenStack SWIFT" 37 "Google Drive"]] +[[!poll open=yes 18 "Amazon S3 (done)" 13 "Amazon Glacier (done)" 10 "Box.com (done)" 81 "My phone (or MP3 player)" 29 "Tahoe-LAFS" 19 "OpenStack SWIFT" 37 "Google Drive"]] This poll is ordered with the options I consider easiest to build listed first. Mostly because git-annex already supports them and they From 4a980d11cee64891c904167dacdbbe9342c6e53e Mon Sep 17 00:00:00 2001 From: andrew Date: Tue, 28 Jan 2025 18:07:59 +0000 Subject: [PATCH 03/14] --- ...t-remote-annex_for_directory_on_macOS.mdwn | 362 ++++++++++++++++++ 1 file changed, 362 insertions(+) create mode 100644 doc/bugs/git-remote-annex_for_directory_on_macOS.mdwn diff --git a/doc/bugs/git-remote-annex_for_directory_on_macOS.mdwn b/doc/bugs/git-remote-annex_for_directory_on_macOS.mdwn new file mode 100644 index 0000000000..435fce443d --- /dev/null +++ b/doc/bugs/git-remote-annex_for_directory_on_macOS.mdwn @@ -0,0 +1,362 @@ +### Please describe the problem. +Running `git-annex enableremote local1 --with-url` for directory special remote `local` fails on macOS. + +Trying to sync to directory special remote configured manually as git-remote-annex fails on macOS. + +### What steps will reproduce the problem? +[[!format sh """ +% pwd +/tmp/git-annex-testing +% mkdir repo +% cd repo +% git init +% git annex init +% echo "file a" > a.txt +% echo "file b" > b.txt +% git annex add a.txt +% git annex add b.txt +% git commit -m "added files" +% git annex info +... +local annex keys: 2 +local annex size: 14 bytes +annexed files in working tree: 2 +.. +% mkdir ../specialdirectoryremote +% git annex initremote local1 type=directory directory=/tmp/git-annex-testing/specialdirectoryremote encryption=none +initremote local1 ok +(recording state in git...) +% git annex copy --to=local1 +% git annex info local1 +uuid: 7d7560cd-2e80-489d-af4a-ec23b9b9abb7 +description: [local1] +trust: semitrusted +remote: local1 +cost: 100.0 +type: directory +available: true +directory: /tmp/git-annex-testing/specialdirectoryremote +encryption: none +chunking: none +remote annex keys: 2 +remote annex size: 14 bytes +% git-annex enableremote local1 --with-url +enableremote local1 +git-annex: Specify directory= +failed +enableremote: 1 failed +"""]] + + +### What version of git-annex are you using? On what operating system? +[[!format sh """ +% git --version +git version 2.48.0 +% git-annex version +git-annex version: 10.20250103-gbd5d7e936d15abc1376f64ea9a124bba0b6e2d11 +build flags: Assistant Webapp Pairing FsEvents TorrentParser MagicMime Servant Benchmark Feeds Testsuite S3 WebDAV +dependency versions: aws-0.24.3 bloomfilter-2.0.1.2 crypton-1.0.1 DAV-1.3.4 feed-1.3.2.1 ghc-9.8.4 http-client-0.7.18 persistent-sqlite-2.13.3.0 torrent-10000.1.3 uuid-1.3.16 yesod-1.6.2.1 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL GITBUNDLE GITMANIFEST VURL X* +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg rclone hook external +operating system: darwin x86_64 +supported repository versions: 8 9 10 +upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10 +local repository version: 10 +% sw_vers +ProductName: macOS +ProductVersion: 12.6.3 +BuildVersion: 21G419 +% uname -a +Darwin bumblebee.local 21.6.0 Darwin Kernel Version 21.6.0: Mon Dec 19 20:46:01 PST 2022; root:xnu-8020.240.18~2/RELEASE_ARM64_T8101 arm64 +% +"""]] + +### Please provide any additional information below. + +[[!format sh """ +% git --version +git version 2.48.0 +% git-annex version +git-annex version: 10.20250103-gbd5d7e936d15abc1376f64ea9a124bba0b6e2d11 +build flags: Assistant Webapp Pairing FsEvents TorrentParser MagicMime Servant Benchmark Feeds Testsuite S3 WebDAV +dependency versions: aws-0.24.3 bloomfilter-2.0.1.2 crypton-1.0.1 DAV-1.3.4 feed-1.3.2.1 ghc-9.8.4 http-client-0.7.18 persistent-sqlite-2.13.3.0 torrent-10000.1.3 uuid-1.3.16 yesod-1.6.2.1 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL GITBUNDLE GITMANIFEST VURL X* +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg rclone hook external +operating system: darwin x86_64 +supported repository versions: 8 9 10 +upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10 +local repository version: 10 +% sw_vers +ProductName: macOS +ProductVersion: 12.6.3 +BuildVersion: 21G419 +% uname -a +Darwin bumblebee.local 21.6.0 Darwin Kernel Version 21.6.0: Mon Dec 19 20:46:01 PST 2022; root:xnu-8020.240.18~2/RELEASE_ARM64_T8101 arm64 +% + + +% pwd +/tmp/git-annex-testing +% mkdir repo +% cd repo +% git init +% git annex init +% echo "file a" > a.txt +% echo "file b" > b.txt +% git annex add a.txt +% git annex add b.txt +% git commit -m "added files" +% git annex info +... +local annex keys: 2 +local annex size: 14 bytes +annexed files in working tree: 2 +.. +% mkdir ../specialdirectoryremote +% git annex initremote local1 type=directory directory=/tmp/git-annex-testing/specialdirectoryremote encryption=none +initremote local1 ok +(recording state in git...) +% git annex copy --to=local1 +% git annex info local1 +uuid: 7d7560cd-2e80-489d-af4a-ec23b9b9abb7 +description: [local1] +trust: semitrusted +remote: local1 +cost: 100.0 +type: directory +available: true +directory: /tmp/git-annex-testing/specialdirectoryremote +encryption: none +chunking: none +remote annex keys: 2 +remote annex size: 14 bytes +% git-annex enableremote local1 --with-url +enableremote local1 +git-annex: Specify directory= +failed +enableremote: 1 failed + +% git config set annex.debug true +% git-annex enableremote local1 --with-url +[2025-01-28 12:38:26.198647] (Utility.Process) process [13595] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:38:26.206976] (Utility.Process) process [13595] done ExitSuccess +[2025-01-28 12:38:26.209633] (Utility.Process) process [13596] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:38:26.216084] (Utility.Process) process [13596] done ExitSuccess +[2025-01-28 12:38:26.220051] (Utility.Process) process [13597] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +enableremote local1 +git-annex: Specify directory= +failed +[2025-01-28 12:38:26.228828] (Utility.Process) process [13597] done ExitSuccess +enableremote: 1 failed +% + + +% git config remote.local1.url annex:: +% git config remote.local1.fetch '+refs/heads/*:refs/remotes/foo/*' + + + +% git push --set-upstream local1 master +[2025-01-28 12:45:05.935101] (Utility.Process) process [20178] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:45:05.941251] (Utility.Process) process [20178] done ExitSuccess +[2025-01-28 12:45:05.943896] (Utility.Process) process [20179] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:45:05.950867] (Utility.Process) process [20179] done ExitSuccess +[2025-01-28 12:45:05.95425] (Utility.Process) process [20180] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +Full remote url: annex::7d7560cd-2e80-489d-af4a-ec23b9b9abb7?encryption=none&type=directory +[2025-01-28 12:45:05.96577] (Utility.Process) process [20181] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","for-each-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/","--format=%(objectname) %(refname)"] +[2025-01-28 12:45:05.973785] (Utility.Process) process [20181] done ExitSuccess +[2025-01-28 12:45:05.976602] (Utility.Process) process [20182] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2025-01-28 12:45:05.983816] (Utility.Process) process [20182] done ExitSuccess +[2025-01-28 12:45:05.986197] (Utility.Process) process [20183] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","for-each-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/","--format=%(objectname) %(refname)"] +[2025-01-28 12:45:05.992446] (Utility.Process) process [20183] done ExitSuccess +[2025-01-28 12:45:05.994823] (Utility.Process) process [20184] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/refs/heads/master","cee403c71ad8d180c149c26818b268b4fba67c3f"] +[2025-01-28 12:45:06.002395] (Utility.Process) process [20184] done ExitSuccess +[2025-01-28 12:45:06.006364] (Utility.Process) process [20185] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","bundle","create","--quiet","/var/folders/2p/4z266zp97g9f0w5xxx4hkjz40000gn/T/GITBUNDLE20173-0","--stdin"] +[2025-01-28 12:45:06.027734] (Utility.Process) process [20185] done ExitSuccess +[2025-01-28 12:45:06.028831] (Annex.Perms) freezing content .git/annex/objects/30/fM/GITBUNDLE-s595--7d7560cd-2e80-489d-af4a-ec23b9b9abb7-9aca839242a1231b8ea7d6c4544a582da92ec8a345a87d1f9ec2e7ac9c57917d/GITBUNDLE-s595--7d7560cd-2e80-489d-af4a-ec23b9b9abb7-9aca839242a1231b8ea7d6c4544a582da92ec8a345a87d1f9ec2e7ac9c57917d +[2025-01-28 12:45:06.032936] (Annex.Perms) freezing content directory .git/annex/objects/30/fM/GITBUNDLE-s595--7d7560cd-2e80-489d-af4a-ec23b9b9abb7-9aca839242a1231b8ea7d6c4544a582da92ec8a345a87d1f9ec2e7ac9c57917d +[2025-01-28 12:45:06.035315] (Utility.Process) process [20187] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2025-01-28 12:45:06.041878] (Utility.Process) process [20187] done ExitSuccess +[2025-01-28 12:45:06.044996] (Utility.Process) process [20188] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2025-01-28 12:45:06.051196] (Utility.Process) process [20188] done ExitSuccess +[2025-01-28 12:45:06.053205] (Utility.Process) process [20180] done ExitSuccess +[2025-01-28 12:45:06.057349] (Utility.Process) process [20191] call: cp ["-a","/var/folders/2p/4z266zp97g9f0w5xxx4hkjz40000gn/T/GITMANIFEST20173-3",".git/annex/objects/9q/z3/GITMANIFEST--7d7560cd-2e80-489d-af4a-ec23b9b9abb7.bak/GITMANIFEST--7d7560cd-2e80-489d-af4a-ec23b9b9abb7.bak"] +[2025-01-28 12:45:06.057467] (Utility.Process) process [20191] done ExitFailure (-9) +[2025-01-28 12:45:06.05756] (Annex.Perms) freezing content directory .git/annex/objects/9q/z3/GITMANIFEST--7d7560cd-2e80-489d-af4a-ec23b9b9abb7.bak +Push failed (Failed to upload manifest.) +[2025-01-28 12:45:06.06058] (Utility.Process) process [20192] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","for-each-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/","--format=%(objectname) %(refname)"] +[2025-01-28 12:45:06.068669] (Utility.Process) process [20192] done ExitSuccess +[2025-01-28 12:45:06.071203] (Utility.Process) process [20193] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","-d","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/refs/heads/master","cee403c71ad8d180c149c26818b268b4fba67c3f"] +[2025-01-28 12:45:06.078439] (Utility.Process) process [20193] done ExitSuccess +warning: helper reported unexpected status of push +Everything up-to-date +[2025-01-28 12:45:06.081082] (Utility.Process) process [20194] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:45:06.0879] (Utility.Process) process [20194] done ExitSuccess +[2025-01-28 12:45:06.090302] (Utility.Process) process [20195] read: git ["--version"] +[2025-01-28 12:45:06.095789] (Utility.Process) process [20195] done ExitSuccess +[2025-01-28 12:45:06.096182] (Annex.Perms) freezing content .git/annex/misctmp/gaprobe +[2025-01-28 12:45:06.096262] (Annex.Perms) thawing content .git/annex/misctmp/gaprobe +[2025-01-28 12:45:06.09862] (Utility.Process) process [20196] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +% exit +% git config unset annex.debug +% git push --set-upstream local1 master +Full remote url: annex::7d7560cd-2e80-489d-af4a-ec23b9b9abb7?encryption=none&type=directory +Push failed (Failed to upload manifest.) +warning: helper reported unexpected status of push +Everything up-to-date +% git annex sync +commit +On branch master +nothing to commit, working tree clean +ok +pull local1 +Full remote url: annex::7d7560cd-2e80-489d-af4a-ec23b9b9abb7?encryption=none&type=directory +git-annex: No git repository found in this remote. +ok +push local1 +Full remote url: annex::7d7560cd-2e80-489d-af4a-ec23b9b9abb7?encryption=none&type=directory +Push failed (Failed to upload manifest.) +warning: helper reported unexpected status of push +warning: helper reported unexpected status of push +Everything up-to-date +ok +% git config set annex.debug true +% git annex sync +[2025-01-28 12:46:47.892399] (Utility.Process) process [22027] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:46:47.898879] (Utility.Process) process [22027] done ExitSuccess +[2025-01-28 12:46:47.901826] (Utility.Process) process [22028] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:46:47.908197] (Utility.Process) process [22028] done ExitSuccess +[2025-01-28 12:46:47.911039] (Utility.Process) process [22029] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +commit +[2025-01-28 12:46:47.92136] (Utility.Process) process [22030] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit","-a","-m","git-annex in andrew:/private/tmp/git-annex-testing/repo"] +[2025-01-28 12:46:47.981547] (Utility.Process) process [22035] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2025-01-28 12:46:47.988126] (Utility.Process) process [22035] done ExitSuccess +[2025-01-28 12:46:47.990408] (Utility.Process) process [22036] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2025-01-28 12:46:47.996719] (Utility.Process) process [22036] done ExitSuccess +[2025-01-28 12:46:48.000582] (Utility.Process) process [22037] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff","--cached","--name-only","-z","--diff-filter=ACMRT","--","."] +[2025-01-28 12:46:48.007367] (Utility.Process) process [22037] done ExitSuccess +[2025-01-28 12:46:48.010028] (Utility.Process) process [22038] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2025-01-28 12:46:48.016108] (Utility.Process) process [22038] done ExitSuccess +[2025-01-28 12:46:48.018541] (Utility.Process) process [22039] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2025-01-28 12:46:48.025954] (Utility.Process) process [22039] done ExitSuccess +On branch master +nothing to commit, working tree clean +[2025-01-28 12:46:48.043086] (Utility.Process) process [22030] done ExitFailure 1 +ok +[2025-01-28 12:46:48.045582] (Utility.Process) process [22040] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2025-01-28 12:46:48.05211] (Utility.Process) process [22040] done ExitSuccess +[2025-01-28 12:46:48.054886] (Utility.Process) process [22041] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2025-01-28 12:46:48.062284] (Utility.Process) process [22041] done ExitSuccess +[2025-01-28 12:46:48.064679] (Utility.Process) process [22042] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--verify","-q","refs/heads/synced/master"] +[2025-01-28 12:46:48.071504] (Utility.Process) process [22042] done ExitSuccess +[2025-01-28 12:46:48.073977] (Utility.Process) process [22043] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/heads/synced/master","--pretty=%H","-n1"] +[2025-01-28 12:46:48.080766] (Utility.Process) process [22043] done ExitSuccess +pull local1 +[2025-01-28 12:46:48.084197] (Utility.Process) process [22044] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","fetch","local1"] +[2025-01-28 12:46:48.170637] (Utility.Process) process [22080] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:46:48.179566] (Utility.Process) process [22080] done ExitSuccess +[2025-01-28 12:46:48.183611] (Utility.Process) process [22081] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:46:48.192055] (Utility.Process) process [22081] done ExitSuccess +[2025-01-28 12:46:48.197482] (Utility.Process) process [22086] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +Full remote url: annex::7d7560cd-2e80-489d-af4a-ec23b9b9abb7?encryption=none&type=directory +[2025-01-28 12:46:48.215929] (Utility.Process) process [22087] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:46:48.223041] (Utility.Process) process [22087] done ExitSuccess +[2025-01-28 12:46:48.225642] (Utility.Process) process [22088] read: git ["--version"] +[2025-01-28 12:46:48.231226] (Utility.Process) process [22088] done ExitSuccess +[2025-01-28 12:46:48.231795] (Annex.Perms) freezing content .git/annex/misctmp/gaprobe +[2025-01-28 12:46:48.231889] (Annex.Perms) thawing content .git/annex/misctmp/gaprobe +git-annex: No git repository found in this remote. +[2025-01-28 12:46:48.239358] (Utility.Process) process [22044] done ExitFailure 128 +ok +[2025-01-28 12:46:48.243384] (Utility.Process) process [22089] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:46:48.250955] (Utility.Process) process [22089] done ExitSuccess +[2025-01-28 12:46:48.253625] (Utility.Process) process [22090] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:46:48.260937] (Utility.Process) process [22090] done ExitSuccess +[2025-01-28 12:46:48.264403] (Utility.Process) process [22091] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2025-01-28 12:46:48.270737] (Utility.Process) process [22091] done ExitSuccess +[2025-01-28 12:46:48.273298] (Utility.Process) process [22092] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2025-01-28 12:46:48.279777] (Utility.Process) process [22092] done ExitSuccess +[2025-01-28 12:46:48.282444] (Utility.Process) process [22093] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","branch","-f","synced/master","refs/heads/master"] +[2025-01-28 12:46:48.290967] (Utility.Process) process [22093] done ExitSuccess +[2025-01-28 12:46:48.293498] (Utility.Process) process [22094] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--verify","-q","refs/remotes/local1/synced/master"] +[2025-01-28 12:46:48.300336] (Utility.Process) process [22094] done ExitFailure 1 +push local1 +[2025-01-28 12:46:48.302852] (Utility.Process) process [22095] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","local1","master:synced/master","+git-annex:synced/git-annex"] +[2025-01-28 12:46:48.383262] (Utility.Process) process [22102] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:46:48.389482] (Utility.Process) process [22102] done ExitSuccess +[2025-01-28 12:46:48.391771] (Utility.Process) process [22103] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:46:48.398109] (Utility.Process) process [22103] done ExitSuccess +[2025-01-28 12:46:48.402067] (Utility.Process) process [22104] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +Full remote url: annex::7d7560cd-2e80-489d-af4a-ec23b9b9abb7?encryption=none&type=directory +[2025-01-28 12:46:48.41247] (Utility.Process) process [22105] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","for-each-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/","--format=%(objectname) %(refname)"] +[2025-01-28 12:46:48.419106] (Utility.Process) process [22105] done ExitSuccess +[2025-01-28 12:46:48.42299] (Utility.Process) process [22106] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:46:48.430151] (Utility.Process) process [22106] done ExitSuccess +[2025-01-28 12:46:48.434139] (Utility.Process) process [22107] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2025-01-28 12:46:48.440655] (Utility.Process) process [22107] done ExitSuccess +[2025-01-28 12:46:48.44324] (Utility.Process) process [22108] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","for-each-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/","--format=%(objectname) %(refname)"] +[2025-01-28 12:46:48.449279] (Utility.Process) process [22108] done ExitSuccess +[2025-01-28 12:46:48.452854] (Utility.Process) process [22109] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/refs/heads/synced/git-annex","b6817903fc3363e53ff863fcf79e90ab200130c3"] +[2025-01-28 12:46:48.460699] (Utility.Process) process [22109] done ExitSuccess +[2025-01-28 12:46:48.463036] (Utility.Process) process [22110] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/refs/heads/synced/master","cee403c71ad8d180c149c26818b268b4fba67c3f"] +[2025-01-28 12:46:48.470252] (Utility.Process) process [22110] done ExitSuccess +[2025-01-28 12:46:48.473412] (Utility.Process) process [22111] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","bundle","create","--quiet","/var/folders/2p/4z266zp97g9f0w5xxx4hkjz40000gn/T/GITBUNDLE22097-0","--stdin"] +[2025-01-28 12:46:48.494608] (Utility.Process) process [22111] done ExitSuccess +[2025-01-28 12:46:48.49778] (Utility.Process) process [22104] done ExitSuccess +[2025-01-28 12:46:48.505294] (Utility.Process) process [22116] call: cp ["-a","/var/folders/2p/4z266zp97g9f0w5xxx4hkjz40000gn/T/GITMANIFEST22097-3",".git/annex/objects/9q/z3/GITMANIFEST--7d7560cd-2e80-489d-af4a-ec23b9b9abb7.bak/GITMANIFEST--7d7560cd-2e80-489d-af4a-ec23b9b9abb7.bak"] +[2025-01-28 12:46:48.505403] (Utility.Process) process [22116] done ExitFailure (-9) +[2025-01-28 12:46:48.505502] (Annex.Perms) freezing content directory .git/annex/objects/9q/z3/GITMANIFEST--7d7560cd-2e80-489d-af4a-ec23b9b9abb7.bak +Push failed (Failed to upload manifest.) +[2025-01-28 12:46:48.507999] (Utility.Process) process [22117] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","for-each-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/","--format=%(objectname) %(refname)"] +[2025-01-28 12:46:48.514991] (Utility.Process) process [22117] done ExitSuccess +[2025-01-28 12:46:48.518649] (Utility.Process) process [22118] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","-d","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/refs/heads/synced/git-annex","b6817903fc3363e53ff863fcf79e90ab200130c3"] +[2025-01-28 12:46:48.525217] (Utility.Process) process [22118] done ExitSuccess +[2025-01-28 12:46:48.528742] (Utility.Process) process [22119] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","-d","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/refs/heads/synced/master","cee403c71ad8d180c149c26818b268b4fba67c3f"] +[2025-01-28 12:46:48.535612] (Utility.Process) process [22119] done ExitSuccess +warning: helper reported unexpected status of push +warning: helper reported unexpected status of push +Everything up-to-date +[2025-01-28 12:46:48.5386] (Utility.Process) process [22120] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:46:48.545982] (Utility.Process) process [22120] done ExitSuccess +[2025-01-28 12:46:48.549657] (Utility.Process) process [22121] read: git ["--version"] +[2025-01-28 12:46:48.555248] (Utility.Process) process [22121] done ExitSuccess +[2025-01-28 12:46:48.555667] (Annex.Perms) freezing content .git/annex/misctmp/gaprobe +[2025-01-28 12:46:48.555747] (Annex.Perms) thawing content .git/annex/misctmp/gaprobe +[2025-01-28 12:46:48.558896] (Utility.Process) process [22122] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2025-01-28 12:46:48.574158] (Utility.Process) process [22095] done ExitSuccess +[2025-01-28 12:46:48.576864] (Utility.Process) process [22123] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","local1","git-annex"] +[2025-01-28 12:46:48.808953] (Utility.Process) process [22123] done ExitSuccess +[2025-01-28 12:46:48.813005] (Utility.Process) process [22148] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","local1","master"] +[2025-01-28 12:46:49.062622] (Utility.Process) process [22148] done ExitSuccess +ok +[2025-01-28 12:46:49.063559] (Utility.Process) process [22029] done ExitSuccess +% + + +% cat .git/config +[core] + repositoryformatversion = 0 + filemode = true + bare = false + logallrefupdates = true + ignorecase = true + precomposeunicode = true +[annex] + uuid = bb331ff4-2e85-4657-869a-202d269d4b3c + version = 10 +[filter "annex"] + smudge = git-annex smudge -- %f + clean = git-annex smudge --clean -- %f + process = git-annex filter-process +[remote "local1"] + annex-directory = /tmp/git-annex-testing/specialdirectoryremote + annex-uuid = 7d7560cd-2e80-489d-af4a-ec23b9b9abb7 + skipFetchAll = true + url = annex:: + fetch = +refs/heads/*:refs/remotes/foo/* +% +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Yup! + From 9965b98fc8137ef5164a85e2031b6053f7e29efe Mon Sep 17 00:00:00 2001 From: cwg Date: Tue, 28 Jan 2025 19:18:02 +0000 Subject: [PATCH 04/14] --- ...view_-_git_annex_list_--compact__63__.mdwn | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 doc/forum/Keeping_the_overview_-_git_annex_list_--compact__63__.mdwn diff --git a/doc/forum/Keeping_the_overview_-_git_annex_list_--compact__63__.mdwn b/doc/forum/Keeping_the_overview_-_git_annex_list_--compact__63__.mdwn new file mode 100644 index 0000000000..4e234a77a3 --- /dev/null +++ b/doc/forum/Keeping_the_overview_-_git_annex_list_--compact__63__.mdwn @@ -0,0 +1,31 @@ +Is there a way to obtain/keep the overview over a git annex repository with many directories and files? For example to answer the following question in a compact way: which subdirectories of the current directory are complete locally? + +The needed information is provided by both `git annex list` and `git annex info`, but the output of both command is very verbose and difficult to parse visually. + +For example part of the output of `git annex list` could be: + +``` +XXX__ foo/bar/a +XXX__ foo/bar/b +XXX__ foo/bar/c +XXX__ foo/hey/d +XXX__ foo/hey/d +XXX__ foo/hey/e +``` + +Assuming that the `foo` directory contains no further files this could be compacted to + +``` +XXX__ foo/bar/ +XXX__ foo/hey/ +``` + +or even + +``` +XXX__ foo/ +``` + +Of course the above procedure would only compact directories if all files have the same presence status. But I guess that this will be true in many cases. + +I’m thinking about implementing this as a filter to the output of `git annex list`, but wanted to check whether similar ideas haven’t been considered here before. (I could not find anything.) From 0512905316783561124238b3fc408fca12e19644 Mon Sep 17 00:00:00 2001 From: cwg Date: Tue, 28 Jan 2025 19:29:21 +0000 Subject: [PATCH 05/14] correct typo --- .../Keeping_the_overview_-_git_annex_list_--compact__63__.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/Keeping_the_overview_-_git_annex_list_--compact__63__.mdwn b/doc/forum/Keeping_the_overview_-_git_annex_list_--compact__63__.mdwn index 4e234a77a3..31d1bd90f7 100644 --- a/doc/forum/Keeping_the_overview_-_git_annex_list_--compact__63__.mdwn +++ b/doc/forum/Keeping_the_overview_-_git_annex_list_--compact__63__.mdwn @@ -9,8 +9,8 @@ XXX__ foo/bar/a XXX__ foo/bar/b XXX__ foo/bar/c XXX__ foo/hey/d -XXX__ foo/hey/d XXX__ foo/hey/e +XXX__ foo/hey/f ``` Assuming that the `foo` directory contains no further files this could be compacted to From 75a6a0030774acea2644683b3cd444907f46b4f9 Mon Sep 17 00:00:00 2001 From: jnkl Date: Tue, 28 Jan 2025 21:20:17 +0000 Subject: [PATCH 06/14] Added a comment --- .../comment_4_48d98de3582217dcbcdc6b6968c5152d._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_4_48d98de3582217dcbcdc6b6968c5152d._comment diff --git a/doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_4_48d98de3582217dcbcdc6b6968c5152d._comment b/doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_4_48d98de3582217dcbcdc6b6968c5152d._comment new file mode 100644 index 0000000000..8f5d03e544 --- /dev/null +++ b/doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_4_48d98de3582217dcbcdc6b6968c5152d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="jnkl" + avatar="http://cdn.libravatar.org/avatar/2ab576f3bf2e0d96b1ee935bb7f33dbe" + subject="comment 4" + date="2025-01-28T21:20:17Z" + content=""" +Thank you. So bare repositories are only needed when I want to save disk space with git annex? +"""]] From 58e897b96613975673e7a751bf6f26320e5c60bf Mon Sep 17 00:00:00 2001 From: jnkl Date: Tue, 28 Jan 2025 21:32:18 +0000 Subject: [PATCH 07/14] --- .../Reasonable_annex.bloomcapacity__63__.mdwn | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 doc/forum/Reasonable_annex.bloomcapacity__63__.mdwn diff --git a/doc/forum/Reasonable_annex.bloomcapacity__63__.mdwn b/doc/forum/Reasonable_annex.bloomcapacity__63__.mdwn new file mode 100644 index 0000000000..b5cafce72e --- /dev/null +++ b/doc/forum/Reasonable_annex.bloomcapacity__63__.mdwn @@ -0,0 +1,15 @@ +What is a reasonable value for ```annex.bloomcapacity``` in this situation and in what unit? + + local annex keys: 670671 + local annex size: 2.62 terabytes + annexed files in working tree: 1410199 + size of annexed files in working tree: 3.52 terabytes + combined annex size of all repositories: 2.63 terabytes + annex sizes of repositories: + 2.62 TB: b2e77041-584e-4699-947d-ef5004273901 -- pudong [here] + 7.39 GB: 42a53a13-7ad6-43a5-95ef-813d4f8c1a6f -- [batam_payload] + backend usage: + SHA256E: 1410199 + bloom filter size: 32 mebibytes (appears too small for this repository; adjust annex.bloomcapacity) + +I don't really understand what happens when the bloom filter is too small. From 2f278fd4b396bc44a8435965adbc6666f17d3f1b Mon Sep 17 00:00:00 2001 From: Atemu Date: Tue, 28 Jan 2025 21:57:42 +0000 Subject: [PATCH 08/14] Added a comment --- .../comment_5_051d14d4b7439d10f9879aff505a923c._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_5_051d14d4b7439d10f9879aff505a923c._comment diff --git a/doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_5_051d14d4b7439d10f9879aff505a923c._comment b/doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_5_051d14d4b7439d10f9879aff505a923c._comment new file mode 100644 index 0000000000..65d97fce89 --- /dev/null +++ b/doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_5_051d14d4b7439d10f9879aff505a923c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="Atemu" + avatar="http://cdn.libravatar.org/avatar/6ac9c136a74bb8760c66f422d3d6dc32" + subject="comment 5" + date="2025-01-28T21:57:42Z" + content=""" +Indeed. However the space savings would likely be marginal in a typical git-annex repo. +"""]] From acdefd77a6ecff2170b09616ad06a355e0bf9dc5 Mon Sep 17 00:00:00 2001 From: matrss Date: Wed, 29 Jan 2025 09:56:13 +0000 Subject: [PATCH 09/14] Added a comment --- ..._b6b1c8e9dc9e1d818036385fd073ed21._comment | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 doc/todo/compute_special_remote/comment_13_b6b1c8e9dc9e1d818036385fd073ed21._comment diff --git a/doc/todo/compute_special_remote/comment_13_b6b1c8e9dc9e1d818036385fd073ed21._comment b/doc/todo/compute_special_remote/comment_13_b6b1c8e9dc9e1d818036385fd073ed21._comment new file mode 100644 index 0000000000..372c1feb7b --- /dev/null +++ b/doc/todo/compute_special_remote/comment_13_b6b1c8e9dc9e1d818036385fd073ed21._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="matrss" + avatar="http://cdn.libravatar.org/avatar/cd1c0b3be1af288012e49197918395f0" + subject="comment 13" + date="2025-01-29T09:56:12Z" + content=""" +> @m.risse in your example the \"data.nc\" file gets new content when retrieved from the special remote and the source file has changed. + +True, that can happen, and the user was explicit in that they either don't care about it (non-checksum backend, URL in my PoC), or do care (checksum backend) and git-annex would fail the checksum verification. + +> But if you already have data.nc file present in a repository, it does not get updated immediately when you update the source \"data.grib\" file. +> +> So, a drop and re-get of a file changes the version of the file you have available. For that matter, if the old version has been stored on other remotes, a get may retrieve either an old or a new version. That is not intuitive and it makes me wonder if using a special remote is really a good fit for what you're wanting to do + +This I haven't entirely thought through. I'd say if the key uses a non-checksum backend, then it can only be assumed and is the users responsibility that the resulting file is functionally, even if not bit-by-bit, identical. E.g. with netCDF checksums can differ due to small details like chunking, but the data might be the same. With a checksum backend git-annex would just fail the next recompute, but the interactions with copies on other remotes could indeed get confusing. + +> In your \"cdo\" example, it's not clear to me if the new version of the software generates an identical file to the old, or if it has a bug fix that causes it to generate a significantly different output. If the two outputs are significantly different then treating them as the same git-annex key seems questionable to me. + +Again, two possible cases depending on if the key uses a checksum or a non-checksum backend. With a checksum: if the new version produces the same output everything is fine; if the new version produces different output then git-annex would indicate this discrepancy on the next recompute and the user has to decide how to handle it (probably by checking that the output of the new version is either functionally the same or in some way \"better\" than the old one and updating the repository to record this new key as that file). + +Without a checksum backend the user would again have been explicit in that they don't care if the data changes for whatever reason, the key is essentially just a placeholder for the computation without a guarantee about its content. + +Something like VURL would be a compromise between the two: it would avoid the upfront cost of computing all files (which might be very expensive), but still instruct git-annex to error out if the checksum changes at some point after the first compute. A regular migration of the computed-files-so-far to a checksum backend could achieve the same. +"""]] From 1e0a48fad0f7fb40fe9472058d6dcd437667bd64 Mon Sep 17 00:00:00 2001 From: matrss Date: Wed, 29 Jan 2025 10:13:59 +0000 Subject: [PATCH 10/14] Added a comment --- ...mment_14_f0a575875e1f8809906ba4021e879b43._comment | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/todo/compute_special_remote/comment_14_f0a575875e1f8809906ba4021e879b43._comment diff --git a/doc/todo/compute_special_remote/comment_14_f0a575875e1f8809906ba4021e879b43._comment b/doc/todo/compute_special_remote/comment_14_f0a575875e1f8809906ba4021e879b43._comment new file mode 100644 index 0000000000..7d3624c83d --- /dev/null +++ b/doc/todo/compute_special_remote/comment_14_f0a575875e1f8809906ba4021e879b43._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="matrss" + avatar="http://cdn.libravatar.org/avatar/cd1c0b3be1af288012e49197918395f0" + subject="comment 14" + date="2025-01-29T10:13:59Z" + content=""" +Some thoughts regarding your ideas: + +- Multiple output files could always be emulated by generating a single archive file and registering additional compute instructions that simply extract each output file from that archive. I think there could be some convenience functionality on the CLI side to set that up and the key of the archive file might not even need to correspond to an actual file in the tree. +- For my use-cases (and I think DataLad at large) it is important to make this feature work across repository boundaries. E.g. I would like to use this feature to build a derived dataset from , where exactly this conversion from grib to netcdf happens in the compute step. I'd like to have the netcdf outputs as a separate dataset as some users might only be interested in the grib files, and it would scale better when there is more than just one kind of output that can be derived from an input by computation. `git annex get` doesn't work recursively across submodules/subdatasets though, and `datalad get` does not understand keys, just paths (at least so far). +"""]] From ec5f30f746dc2e8c7b911c14ce80791489f325df Mon Sep 17 00:00:00 2001 From: dmcardle Date: Wed, 29 Jan 2025 14:44:38 +0000 Subject: [PATCH 11/14] Added a comment --- ...1_bfceb1a2ce7d9c70b1c79252a8e12822._comment | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/forum/b2_rclone_initremote_failed__58___empty_folders/comment_1_bfceb1a2ce7d9c70b1c79252a8e12822._comment diff --git a/doc/forum/b2_rclone_initremote_failed__58___empty_folders/comment_1_bfceb1a2ce7d9c70b1c79252a8e12822._comment b/doc/forum/b2_rclone_initremote_failed__58___empty_folders/comment_1_bfceb1a2ce7d9c70b1c79252a8e12822._comment new file mode 100644 index 0000000000..751c577e29 --- /dev/null +++ b/doc/forum/b2_rclone_initremote_failed__58___empty_folders/comment_1_bfceb1a2ce7d9c70b1c79252a8e12822._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="dmcardle" + avatar="http://cdn.libravatar.org/avatar/b79468a0d03ec3ec7acbae547c4fa994" + subject="comment 1" + date="2025-01-29T14:44:37Z" + content=""" +Hi, author of rclone's \"gitannex\" command here. Sorry you're running into trouble with it! + +Based on the text, that error is definitely coming from [gitannex.go](https://github.com/rclone/rclone/blob/6494ac037fdd7ada2052ae9a8e05d230e7e686b2/cmd/gitannex/gitannex.go#L276). + +I believe that my intent was to detect that the following mkdir would fail, and offer up a more specific error message rather than letting it fail. + +I don't know anything about Backblaze B2, unfortunately. I suppose we could work around the issue by creating an empty file underneath the place we want the empty directory. Sounds plausible, right? + +Would you mind trying to make an empty directory on your B2 remote to verify it fails? Something like `rclone mkdir myremote:newdir`. + +And also try touching a file in a new directory to verify it's possible in one go? Something like `rclone touch --recursive myremote:newdir/newfile.txt`. +"""]] From 381019ee3f11e844e47c3300c5d9868812539bc7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Jan 2025 12:06:22 -0400 Subject: [PATCH 12/14] response --- ...1_0361164f67f44520f3af5ba88adc3868._comment | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/forum/Reasonable_annex.bloomcapacity__63__/comment_1_0361164f67f44520f3af5ba88adc3868._comment diff --git a/doc/forum/Reasonable_annex.bloomcapacity__63__/comment_1_0361164f67f44520f3af5ba88adc3868._comment b/doc/forum/Reasonable_annex.bloomcapacity__63__/comment_1_0361164f67f44520f3af5ba88adc3868._comment new file mode 100644 index 0000000000..2a63d58ddd --- /dev/null +++ b/doc/forum/Reasonable_annex.bloomcapacity__63__/comment_1_0361164f67f44520f3af5ba88adc3868._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2025-01-29T15:54:19Z" + content=""" +What will happen with too small a bloom filter is `git-annex unused` may +think that some keys are used which are really not. And +`git-annex sync --content` may operate on some keys that are not in the +work tree. + +The `git-annex info` command displays how much memory the configured bloom +filters use, which is why it's reporting 32 membibytes. But the +annex.bloomcapacity setting is the number of expected files in the work +tree, by default 500000. + +It would probably make sense for you to set it to 2000000 or so unless +your system has an unusually small amount of RAM. +"""]] From 9c9baf7c6581235643661b94f70f76f8a10ae60e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Jan 2025 14:06:54 -0400 Subject: [PATCH 13/14] fix toOsPath on windows After recent changes to use OsPath, test suite fails on windows with: Exception: Uneven number of bytes: 5. This is not a Word16 bytestream. Hopefully this fixes it. The innefficiency of the conversion is unfortunate. --- Utility/OsPath.hs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Utility/OsPath.hs b/Utility/OsPath.hs index 5a62e61004..bd7baf85eb 100644 --- a/Utility/OsPath.hs +++ b/Utility/OsPath.hs @@ -22,13 +22,16 @@ import Utility.FileSystemEncoding import System.OsPath import "os-string" System.OsString.Internal.Types import qualified Data.ByteString.Short as S +#if defined(mingw32_HOST_OS) +import GHC.IO (unsafePerformIO) +#endif -{- Unlike System.OsString.fromBytes, on Windows this does not ensure a - - valid USC-2LE encoding. The input ByteString must be in a valid encoding - - already or uses of the OsPath will fail. -} toOsPath :: RawFilePath -> OsPath #if defined(mingw32_HOST_OS) -toOsPath = OsString . WindowsString . S.toShort +-- On Windows, OsString contains a ShortByteString that is +-- utf-16 encoded. So have to convert the input to that. +-- This is relatively expensive. +toOsPath = unsafePerformIO . encodeFS . fromRawFilePath #else toOsPath = OsString . PosixString . S.toShort #endif From 2940cfcd180d52e199c93ac6dca2093acecaaee3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Jan 2025 14:19:25 -0400 Subject: [PATCH 14/14] fix fromOsPath on windows Same reasons as 9c9baf7c6581235643661b94f70f76f8a10ae60e --- Utility/OsPath.hs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Utility/OsPath.hs b/Utility/OsPath.hs index bd7baf85eb..9fa4e0f57e 100644 --- a/Utility/OsPath.hs +++ b/Utility/OsPath.hs @@ -24,6 +24,8 @@ import "os-string" System.OsString.Internal.Types import qualified Data.ByteString.Short as S #if defined(mingw32_HOST_OS) import GHC.IO (unsafePerformIO) +import System.OsString.Encoding.Internal (cWcharsToChars_UCS2) +import qualified System.OsString.Data.ByteString.Short.Word16 as BS16 #endif toOsPath :: RawFilePath -> OsPath @@ -38,7 +40,10 @@ toOsPath = OsString . PosixString . S.toShort fromOsPath :: OsPath -> RawFilePath #if defined(mingw32_HOST_OS) -fromOsPath = S.fromShort . getWindowsString . getOsString +-- On Windows, OsString contains a ShortByteString that is +-- utf-16 encoded. So have to convert the input from that. +-- This is relatively expensive. +fromOsPath = toRawFilePath . cWcharsToChars_UCS2 . BS16.unpack . getWindowsString #else fromOsPath = S.fromShort . getPosixString . getOsString #endif