diff --git a/Utility/OsPath.hs b/Utility/OsPath.hs index b487a2976d..0af093da92 100644 --- a/Utility/OsPath.hs +++ b/Utility/OsPath.hs @@ -30,6 +30,11 @@ import System.OsPath as X hiding (OsPath, OsString, unsafeFromChar) import System.OsPath import "os-string" System.OsString.Internal.Types import qualified System.FilePath.ByteString as PB +#if defined(mingw32_HOST_OS) +import GHC.IO (unsafePerformIO) +import System.OsString.Encoding.Internal (cWcharsToChars_UCS2) +import qualified System.OsString.Data.ByteString.Short.Word16 as BS16 +#endif #else import System.FilePath.ByteString as X hiding (RawFilePath, getSearchPath) import System.FilePath.ByteString (getSearchPath) @@ -49,26 +54,38 @@ instance OsPathConv FilePath where #ifdef WITH_OSPATH instance OsPathConv RawFilePath where toOsPath = bytesToOsPath . S.toShort +#if defined(mingw32_HOST_OS) + fromOsPath = bytesFromOsPath +#else fromOsPath = S.fromShort . bytesFromOsPath +#endif instance OsPathConv ShortByteString where toOsPath = bytesToOsPath +#if defined(mingw32_HOST_OS) + fromOsPath = S.toShort . bytesFromOsPath +#else fromOsPath = bytesFromOsPath +#endif -{- Unlike System.OsString.fromBytes, on Windows this does not ensure a - - valid USC-2LE encoding. The input ByteString must be in a valid encoding - - already or uses of the OsPath will fail. -} bytesToOsPath :: ShortByteString -> OsPath #if defined(mingw32_HOST_OS) -bytesToOsPath = OsString . WindowsString +-- On Windows, OsString contains a ShortByteString that is +-- utf-16 encoded. So have to convert the input to that. +-- This is relatively expensive. +bytesToOsPath = unsafePerformIO . encodeFS . fromRawFilePath #else bytesToOsPath = OsString . PosixString #endif -bytesFromOsPath :: OsPath -> ShortByteString #if defined(mingw32_HOST_OS) -bytesFromOsPath = getWindowsString . getOsString +bytesFromOsPath :: OsPath -> RawFilePath +-- On Windows, OsString contains a ShortByteString that is +-- utf-16 encoded. So have to convert the input from that. +-- This is relatively expensive. +bytesFromOsPath = toRawFilePath . cWcharsToChars_UCS2 . BS16.unpack . getWindowsString #else +bytesFromOsPath :: OsPath -> ShortByteString bytesFromOsPath = getPosixString . getOsString #endif diff --git a/doc/bugs/git-remote-annex_for_directory_on_macOS.mdwn b/doc/bugs/git-remote-annex_for_directory_on_macOS.mdwn new file mode 100644 index 0000000000..435fce443d --- /dev/null +++ b/doc/bugs/git-remote-annex_for_directory_on_macOS.mdwn @@ -0,0 +1,362 @@ +### Please describe the problem. +Running `git-annex enableremote local1 --with-url` for directory special remote `local` fails on macOS. + +Trying to sync to directory special remote configured manually as git-remote-annex fails on macOS. + +### What steps will reproduce the problem? +[[!format sh """ +% pwd +/tmp/git-annex-testing +% mkdir repo +% cd repo +% git init +% git annex init +% echo "file a" > a.txt +% echo "file b" > b.txt +% git annex add a.txt +% git annex add b.txt +% git commit -m "added files" +% git annex info +... +local annex keys: 2 +local annex size: 14 bytes +annexed files in working tree: 2 +.. +% mkdir ../specialdirectoryremote +% git annex initremote local1 type=directory directory=/tmp/git-annex-testing/specialdirectoryremote encryption=none +initremote local1 ok +(recording state in git...) +% git annex copy --to=local1 +% git annex info local1 +uuid: 7d7560cd-2e80-489d-af4a-ec23b9b9abb7 +description: [local1] +trust: semitrusted +remote: local1 +cost: 100.0 +type: directory +available: true +directory: /tmp/git-annex-testing/specialdirectoryremote +encryption: none +chunking: none +remote annex keys: 2 +remote annex size: 14 bytes +% git-annex enableremote local1 --with-url +enableremote local1 +git-annex: Specify directory= +failed +enableremote: 1 failed +"""]] + + +### What version of git-annex are you using? On what operating system? +[[!format sh """ +% git --version +git version 2.48.0 +% git-annex version +git-annex version: 10.20250103-gbd5d7e936d15abc1376f64ea9a124bba0b6e2d11 +build flags: Assistant Webapp Pairing FsEvents TorrentParser MagicMime Servant Benchmark Feeds Testsuite S3 WebDAV +dependency versions: aws-0.24.3 bloomfilter-2.0.1.2 crypton-1.0.1 DAV-1.3.4 feed-1.3.2.1 ghc-9.8.4 http-client-0.7.18 persistent-sqlite-2.13.3.0 torrent-10000.1.3 uuid-1.3.16 yesod-1.6.2.1 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL GITBUNDLE GITMANIFEST VURL X* +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg rclone hook external +operating system: darwin x86_64 +supported repository versions: 8 9 10 +upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10 +local repository version: 10 +% sw_vers +ProductName: macOS +ProductVersion: 12.6.3 +BuildVersion: 21G419 +% uname -a +Darwin bumblebee.local 21.6.0 Darwin Kernel Version 21.6.0: Mon Dec 19 20:46:01 PST 2022; root:xnu-8020.240.18~2/RELEASE_ARM64_T8101 arm64 +% +"""]] + +### Please provide any additional information below. + +[[!format sh """ +% git --version +git version 2.48.0 +% git-annex version +git-annex version: 10.20250103-gbd5d7e936d15abc1376f64ea9a124bba0b6e2d11 +build flags: Assistant Webapp Pairing FsEvents TorrentParser MagicMime Servant Benchmark Feeds Testsuite S3 WebDAV +dependency versions: aws-0.24.3 bloomfilter-2.0.1.2 crypton-1.0.1 DAV-1.3.4 feed-1.3.2.1 ghc-9.8.4 http-client-0.7.18 persistent-sqlite-2.13.3.0 torrent-10000.1.3 uuid-1.3.16 yesod-1.6.2.1 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL GITBUNDLE GITMANIFEST VURL X* +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg rclone hook external +operating system: darwin x86_64 +supported repository versions: 8 9 10 +upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10 +local repository version: 10 +% sw_vers +ProductName: macOS +ProductVersion: 12.6.3 +BuildVersion: 21G419 +% uname -a +Darwin bumblebee.local 21.6.0 Darwin Kernel Version 21.6.0: Mon Dec 19 20:46:01 PST 2022; root:xnu-8020.240.18~2/RELEASE_ARM64_T8101 arm64 +% + + +% pwd +/tmp/git-annex-testing +% mkdir repo +% cd repo +% git init +% git annex init +% echo "file a" > a.txt +% echo "file b" > b.txt +% git annex add a.txt +% git annex add b.txt +% git commit -m "added files" +% git annex info +... +local annex keys: 2 +local annex size: 14 bytes +annexed files in working tree: 2 +.. +% mkdir ../specialdirectoryremote +% git annex initremote local1 type=directory directory=/tmp/git-annex-testing/specialdirectoryremote encryption=none +initremote local1 ok +(recording state in git...) +% git annex copy --to=local1 +% git annex info local1 +uuid: 7d7560cd-2e80-489d-af4a-ec23b9b9abb7 +description: [local1] +trust: semitrusted +remote: local1 +cost: 100.0 +type: directory +available: true +directory: /tmp/git-annex-testing/specialdirectoryremote +encryption: none +chunking: none +remote annex keys: 2 +remote annex size: 14 bytes +% git-annex enableremote local1 --with-url +enableremote local1 +git-annex: Specify directory= +failed +enableremote: 1 failed + +% git config set annex.debug true +% git-annex enableremote local1 --with-url +[2025-01-28 12:38:26.198647] (Utility.Process) process [13595] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:38:26.206976] (Utility.Process) process [13595] done ExitSuccess +[2025-01-28 12:38:26.209633] (Utility.Process) process [13596] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:38:26.216084] (Utility.Process) process [13596] done ExitSuccess +[2025-01-28 12:38:26.220051] (Utility.Process) process [13597] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +enableremote local1 +git-annex: Specify directory= +failed +[2025-01-28 12:38:26.228828] (Utility.Process) process [13597] done ExitSuccess +enableremote: 1 failed +% + + +% git config remote.local1.url annex:: +% git config remote.local1.fetch '+refs/heads/*:refs/remotes/foo/*' + + + +% git push --set-upstream local1 master +[2025-01-28 12:45:05.935101] (Utility.Process) process [20178] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:45:05.941251] (Utility.Process) process [20178] done ExitSuccess +[2025-01-28 12:45:05.943896] (Utility.Process) process [20179] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:45:05.950867] (Utility.Process) process [20179] done ExitSuccess +[2025-01-28 12:45:05.95425] (Utility.Process) process [20180] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +Full remote url: annex::7d7560cd-2e80-489d-af4a-ec23b9b9abb7?encryption=none&type=directory +[2025-01-28 12:45:05.96577] (Utility.Process) process [20181] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","for-each-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/","--format=%(objectname) %(refname)"] +[2025-01-28 12:45:05.973785] (Utility.Process) process [20181] done ExitSuccess +[2025-01-28 12:45:05.976602] (Utility.Process) process [20182] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2025-01-28 12:45:05.983816] (Utility.Process) process [20182] done ExitSuccess +[2025-01-28 12:45:05.986197] (Utility.Process) process [20183] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","for-each-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/","--format=%(objectname) %(refname)"] +[2025-01-28 12:45:05.992446] (Utility.Process) process [20183] done ExitSuccess +[2025-01-28 12:45:05.994823] (Utility.Process) process [20184] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/refs/heads/master","cee403c71ad8d180c149c26818b268b4fba67c3f"] +[2025-01-28 12:45:06.002395] (Utility.Process) process [20184] done ExitSuccess +[2025-01-28 12:45:06.006364] (Utility.Process) process [20185] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","bundle","create","--quiet","/var/folders/2p/4z266zp97g9f0w5xxx4hkjz40000gn/T/GITBUNDLE20173-0","--stdin"] +[2025-01-28 12:45:06.027734] (Utility.Process) process [20185] done ExitSuccess +[2025-01-28 12:45:06.028831] (Annex.Perms) freezing content .git/annex/objects/30/fM/GITBUNDLE-s595--7d7560cd-2e80-489d-af4a-ec23b9b9abb7-9aca839242a1231b8ea7d6c4544a582da92ec8a345a87d1f9ec2e7ac9c57917d/GITBUNDLE-s595--7d7560cd-2e80-489d-af4a-ec23b9b9abb7-9aca839242a1231b8ea7d6c4544a582da92ec8a345a87d1f9ec2e7ac9c57917d +[2025-01-28 12:45:06.032936] (Annex.Perms) freezing content directory .git/annex/objects/30/fM/GITBUNDLE-s595--7d7560cd-2e80-489d-af4a-ec23b9b9abb7-9aca839242a1231b8ea7d6c4544a582da92ec8a345a87d1f9ec2e7ac9c57917d +[2025-01-28 12:45:06.035315] (Utility.Process) process [20187] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2025-01-28 12:45:06.041878] (Utility.Process) process [20187] done ExitSuccess +[2025-01-28 12:45:06.044996] (Utility.Process) process [20188] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2025-01-28 12:45:06.051196] (Utility.Process) process [20188] done ExitSuccess +[2025-01-28 12:45:06.053205] (Utility.Process) process [20180] done ExitSuccess +[2025-01-28 12:45:06.057349] (Utility.Process) process [20191] call: cp ["-a","/var/folders/2p/4z266zp97g9f0w5xxx4hkjz40000gn/T/GITMANIFEST20173-3",".git/annex/objects/9q/z3/GITMANIFEST--7d7560cd-2e80-489d-af4a-ec23b9b9abb7.bak/GITMANIFEST--7d7560cd-2e80-489d-af4a-ec23b9b9abb7.bak"] +[2025-01-28 12:45:06.057467] (Utility.Process) process [20191] done ExitFailure (-9) +[2025-01-28 12:45:06.05756] (Annex.Perms) freezing content directory .git/annex/objects/9q/z3/GITMANIFEST--7d7560cd-2e80-489d-af4a-ec23b9b9abb7.bak +Push failed (Failed to upload manifest.) +[2025-01-28 12:45:06.06058] (Utility.Process) process [20192] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","for-each-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/","--format=%(objectname) %(refname)"] +[2025-01-28 12:45:06.068669] (Utility.Process) process [20192] done ExitSuccess +[2025-01-28 12:45:06.071203] (Utility.Process) process [20193] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","-d","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/refs/heads/master","cee403c71ad8d180c149c26818b268b4fba67c3f"] +[2025-01-28 12:45:06.078439] (Utility.Process) process [20193] done ExitSuccess +warning: helper reported unexpected status of push +Everything up-to-date +[2025-01-28 12:45:06.081082] (Utility.Process) process [20194] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:45:06.0879] (Utility.Process) process [20194] done ExitSuccess +[2025-01-28 12:45:06.090302] (Utility.Process) process [20195] read: git ["--version"] +[2025-01-28 12:45:06.095789] (Utility.Process) process [20195] done ExitSuccess +[2025-01-28 12:45:06.096182] (Annex.Perms) freezing content .git/annex/misctmp/gaprobe +[2025-01-28 12:45:06.096262] (Annex.Perms) thawing content .git/annex/misctmp/gaprobe +[2025-01-28 12:45:06.09862] (Utility.Process) process [20196] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +% exit +% git config unset annex.debug +% git push --set-upstream local1 master +Full remote url: annex::7d7560cd-2e80-489d-af4a-ec23b9b9abb7?encryption=none&type=directory +Push failed (Failed to upload manifest.) +warning: helper reported unexpected status of push +Everything up-to-date +% git annex sync +commit +On branch master +nothing to commit, working tree clean +ok +pull local1 +Full remote url: annex::7d7560cd-2e80-489d-af4a-ec23b9b9abb7?encryption=none&type=directory +git-annex: No git repository found in this remote. +ok +push local1 +Full remote url: annex::7d7560cd-2e80-489d-af4a-ec23b9b9abb7?encryption=none&type=directory +Push failed (Failed to upload manifest.) +warning: helper reported unexpected status of push +warning: helper reported unexpected status of push +Everything up-to-date +ok +% git config set annex.debug true +% git annex sync +[2025-01-28 12:46:47.892399] (Utility.Process) process [22027] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:46:47.898879] (Utility.Process) process [22027] done ExitSuccess +[2025-01-28 12:46:47.901826] (Utility.Process) process [22028] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:46:47.908197] (Utility.Process) process [22028] done ExitSuccess +[2025-01-28 12:46:47.911039] (Utility.Process) process [22029] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +commit +[2025-01-28 12:46:47.92136] (Utility.Process) process [22030] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit","-a","-m","git-annex in andrew:/private/tmp/git-annex-testing/repo"] +[2025-01-28 12:46:47.981547] (Utility.Process) process [22035] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2025-01-28 12:46:47.988126] (Utility.Process) process [22035] done ExitSuccess +[2025-01-28 12:46:47.990408] (Utility.Process) process [22036] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2025-01-28 12:46:47.996719] (Utility.Process) process [22036] done ExitSuccess +[2025-01-28 12:46:48.000582] (Utility.Process) process [22037] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","diff","--cached","--name-only","-z","--diff-filter=ACMRT","--","."] +[2025-01-28 12:46:48.007367] (Utility.Process) process [22037] done ExitSuccess +[2025-01-28 12:46:48.010028] (Utility.Process) process [22038] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2025-01-28 12:46:48.016108] (Utility.Process) process [22038] done ExitSuccess +[2025-01-28 12:46:48.018541] (Utility.Process) process [22039] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2025-01-28 12:46:48.025954] (Utility.Process) process [22039] done ExitSuccess +On branch master +nothing to commit, working tree clean +[2025-01-28 12:46:48.043086] (Utility.Process) process [22030] done ExitFailure 1 +ok +[2025-01-28 12:46:48.045582] (Utility.Process) process [22040] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2025-01-28 12:46:48.05211] (Utility.Process) process [22040] done ExitSuccess +[2025-01-28 12:46:48.054886] (Utility.Process) process [22041] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2025-01-28 12:46:48.062284] (Utility.Process) process [22041] done ExitSuccess +[2025-01-28 12:46:48.064679] (Utility.Process) process [22042] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--verify","-q","refs/heads/synced/master"] +[2025-01-28 12:46:48.071504] (Utility.Process) process [22042] done ExitSuccess +[2025-01-28 12:46:48.073977] (Utility.Process) process [22043] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/master..refs/heads/synced/master","--pretty=%H","-n1"] +[2025-01-28 12:46:48.080766] (Utility.Process) process [22043] done ExitSuccess +pull local1 +[2025-01-28 12:46:48.084197] (Utility.Process) process [22044] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","fetch","local1"] +[2025-01-28 12:46:48.170637] (Utility.Process) process [22080] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:46:48.179566] (Utility.Process) process [22080] done ExitSuccess +[2025-01-28 12:46:48.183611] (Utility.Process) process [22081] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:46:48.192055] (Utility.Process) process [22081] done ExitSuccess +[2025-01-28 12:46:48.197482] (Utility.Process) process [22086] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +Full remote url: annex::7d7560cd-2e80-489d-af4a-ec23b9b9abb7?encryption=none&type=directory +[2025-01-28 12:46:48.215929] (Utility.Process) process [22087] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:46:48.223041] (Utility.Process) process [22087] done ExitSuccess +[2025-01-28 12:46:48.225642] (Utility.Process) process [22088] read: git ["--version"] +[2025-01-28 12:46:48.231226] (Utility.Process) process [22088] done ExitSuccess +[2025-01-28 12:46:48.231795] (Annex.Perms) freezing content .git/annex/misctmp/gaprobe +[2025-01-28 12:46:48.231889] (Annex.Perms) thawing content .git/annex/misctmp/gaprobe +git-annex: No git repository found in this remote. +[2025-01-28 12:46:48.239358] (Utility.Process) process [22044] done ExitFailure 128 +ok +[2025-01-28 12:46:48.243384] (Utility.Process) process [22089] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:46:48.250955] (Utility.Process) process [22089] done ExitSuccess +[2025-01-28 12:46:48.253625] (Utility.Process) process [22090] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:46:48.260937] (Utility.Process) process [22090] done ExitSuccess +[2025-01-28 12:46:48.264403] (Utility.Process) process [22091] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2025-01-28 12:46:48.270737] (Utility.Process) process [22091] done ExitSuccess +[2025-01-28 12:46:48.273298] (Utility.Process) process [22092] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2025-01-28 12:46:48.279777] (Utility.Process) process [22092] done ExitSuccess +[2025-01-28 12:46:48.282444] (Utility.Process) process [22093] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","branch","-f","synced/master","refs/heads/master"] +[2025-01-28 12:46:48.290967] (Utility.Process) process [22093] done ExitSuccess +[2025-01-28 12:46:48.293498] (Utility.Process) process [22094] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--verify","-q","refs/remotes/local1/synced/master"] +[2025-01-28 12:46:48.300336] (Utility.Process) process [22094] done ExitFailure 1 +push local1 +[2025-01-28 12:46:48.302852] (Utility.Process) process [22095] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","local1","master:synced/master","+git-annex:synced/git-annex"] +[2025-01-28 12:46:48.383262] (Utility.Process) process [22102] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:46:48.389482] (Utility.Process) process [22102] done ExitSuccess +[2025-01-28 12:46:48.391771] (Utility.Process) process [22103] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:46:48.398109] (Utility.Process) process [22103] done ExitSuccess +[2025-01-28 12:46:48.402067] (Utility.Process) process [22104] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +Full remote url: annex::7d7560cd-2e80-489d-af4a-ec23b9b9abb7?encryption=none&type=directory +[2025-01-28 12:46:48.41247] (Utility.Process) process [22105] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","for-each-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/","--format=%(objectname) %(refname)"] +[2025-01-28 12:46:48.419106] (Utility.Process) process [22105] done ExitSuccess +[2025-01-28 12:46:48.42299] (Utility.Process) process [22106] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2025-01-28 12:46:48.430151] (Utility.Process) process [22106] done ExitSuccess +[2025-01-28 12:46:48.434139] (Utility.Process) process [22107] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/master"] +[2025-01-28 12:46:48.440655] (Utility.Process) process [22107] done ExitSuccess +[2025-01-28 12:46:48.44324] (Utility.Process) process [22108] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","for-each-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/","--format=%(objectname) %(refname)"] +[2025-01-28 12:46:48.449279] (Utility.Process) process [22108] done ExitSuccess +[2025-01-28 12:46:48.452854] (Utility.Process) process [22109] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/refs/heads/synced/git-annex","b6817903fc3363e53ff863fcf79e90ab200130c3"] +[2025-01-28 12:46:48.460699] (Utility.Process) process [22109] done ExitSuccess +[2025-01-28 12:46:48.463036] (Utility.Process) process [22110] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/refs/heads/synced/master","cee403c71ad8d180c149c26818b268b4fba67c3f"] +[2025-01-28 12:46:48.470252] (Utility.Process) process [22110] done ExitSuccess +[2025-01-28 12:46:48.473412] (Utility.Process) process [22111] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","bundle","create","--quiet","/var/folders/2p/4z266zp97g9f0w5xxx4hkjz40000gn/T/GITBUNDLE22097-0","--stdin"] +[2025-01-28 12:46:48.494608] (Utility.Process) process [22111] done ExitSuccess +[2025-01-28 12:46:48.49778] (Utility.Process) process [22104] done ExitSuccess +[2025-01-28 12:46:48.505294] (Utility.Process) process [22116] call: cp ["-a","/var/folders/2p/4z266zp97g9f0w5xxx4hkjz40000gn/T/GITMANIFEST22097-3",".git/annex/objects/9q/z3/GITMANIFEST--7d7560cd-2e80-489d-af4a-ec23b9b9abb7.bak/GITMANIFEST--7d7560cd-2e80-489d-af4a-ec23b9b9abb7.bak"] +[2025-01-28 12:46:48.505403] (Utility.Process) process [22116] done ExitFailure (-9) +[2025-01-28 12:46:48.505502] (Annex.Perms) freezing content directory .git/annex/objects/9q/z3/GITMANIFEST--7d7560cd-2e80-489d-af4a-ec23b9b9abb7.bak +Push failed (Failed to upload manifest.) +[2025-01-28 12:46:48.507999] (Utility.Process) process [22117] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","for-each-ref","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/","--format=%(objectname) %(refname)"] +[2025-01-28 12:46:48.514991] (Utility.Process) process [22117] done ExitSuccess +[2025-01-28 12:46:48.518649] (Utility.Process) process [22118] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","-d","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/refs/heads/synced/git-annex","b6817903fc3363e53ff863fcf79e90ab200130c3"] +[2025-01-28 12:46:48.525217] (Utility.Process) process [22118] done ExitSuccess +[2025-01-28 12:46:48.528742] (Utility.Process) process [22119] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","-d","refs/namespaces/git-remote-annex/7d7560cd-2e80-489d-af4a-ec23b9b9abb7/refs/heads/synced/master","cee403c71ad8d180c149c26818b268b4fba67c3f"] +[2025-01-28 12:46:48.535612] (Utility.Process) process [22119] done ExitSuccess +warning: helper reported unexpected status of push +warning: helper reported unexpected status of push +Everything up-to-date +[2025-01-28 12:46:48.5386] (Utility.Process) process [22120] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2025-01-28 12:46:48.545982] (Utility.Process) process [22120] done ExitSuccess +[2025-01-28 12:46:48.549657] (Utility.Process) process [22121] read: git ["--version"] +[2025-01-28 12:46:48.555248] (Utility.Process) process [22121] done ExitSuccess +[2025-01-28 12:46:48.555667] (Annex.Perms) freezing content .git/annex/misctmp/gaprobe +[2025-01-28 12:46:48.555747] (Annex.Perms) thawing content .git/annex/misctmp/gaprobe +[2025-01-28 12:46:48.558896] (Utility.Process) process [22122] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2025-01-28 12:46:48.574158] (Utility.Process) process [22095] done ExitSuccess +[2025-01-28 12:46:48.576864] (Utility.Process) process [22123] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","local1","git-annex"] +[2025-01-28 12:46:48.808953] (Utility.Process) process [22123] done ExitSuccess +[2025-01-28 12:46:48.813005] (Utility.Process) process [22148] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","push","local1","master"] +[2025-01-28 12:46:49.062622] (Utility.Process) process [22148] done ExitSuccess +ok +[2025-01-28 12:46:49.063559] (Utility.Process) process [22029] done ExitSuccess +% + + +% cat .git/config +[core] + repositoryformatversion = 0 + filemode = true + bare = false + logallrefupdates = true + ignorecase = true + precomposeunicode = true +[annex] + uuid = bb331ff4-2e85-4657-869a-202d269d4b3c + version = 10 +[filter "annex"] + smudge = git-annex smudge -- %f + clean = git-annex smudge --clean -- %f + process = git-annex filter-process +[remote "local1"] + annex-directory = /tmp/git-annex-testing/specialdirectoryremote + annex-uuid = 7d7560cd-2e80-489d-af4a-ec23b9b9abb7 + skipFetchAll = true + url = annex:: + fetch = +refs/heads/*:refs/remotes/foo/* +% +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) +Yup! + diff --git a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn index 1307c892cd..2c6d62e9c2 100644 --- a/doc/design/assistant/polls/prioritizing_special_remotes.mdwn +++ b/doc/design/assistant/polls/prioritizing_special_remotes.mdwn @@ -6,7 +6,7 @@ locally paired systems, and remote servers with rsync. Help me prioritize my work: What special remote would you most like to use with the git-annex assistant? -[[!poll open=yes 18 "Amazon S3 (done)" 13 "Amazon Glacier (done)" 10 "Box.com (done)" 79 "My phone (or MP3 player)" 29 "Tahoe-LAFS" 19 "OpenStack SWIFT" 37 "Google Drive"]] +[[!poll open=yes 18 "Amazon S3 (done)" 13 "Amazon Glacier (done)" 10 "Box.com (done)" 81 "My phone (or MP3 player)" 29 "Tahoe-LAFS" 19 "OpenStack SWIFT" 37 "Google Drive"]] This poll is ordered with the options I consider easiest to build listed first. Mostly because git-annex already supports them and they diff --git a/doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_4_48d98de3582217dcbcdc6b6968c5152d._comment b/doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_4_48d98de3582217dcbcdc6b6968c5152d._comment new file mode 100644 index 0000000000..8f5d03e544 --- /dev/null +++ b/doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_4_48d98de3582217dcbcdc6b6968c5152d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="jnkl" + avatar="http://cdn.libravatar.org/avatar/2ab576f3bf2e0d96b1ee935bb7f33dbe" + subject="comment 4" + date="2025-01-28T21:20:17Z" + content=""" +Thank you. So bare repositories are only needed when I want to save disk space with git annex? +"""]] diff --git a/doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_5_051d14d4b7439d10f9879aff505a923c._comment b/doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_5_051d14d4b7439d10f9879aff505a923c._comment new file mode 100644 index 0000000000..65d97fce89 --- /dev/null +++ b/doc/forum/Deduplication_between_two_repos_on_the_same_drive__63__/comment_5_051d14d4b7439d10f9879aff505a923c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="Atemu" + avatar="http://cdn.libravatar.org/avatar/6ac9c136a74bb8760c66f422d3d6dc32" + subject="comment 5" + date="2025-01-28T21:57:42Z" + content=""" +Indeed. However the space savings would likely be marginal in a typical git-annex repo. +"""]] diff --git a/doc/forum/Keeping_the_overview_-_git_annex_list_--compact__63__.mdwn b/doc/forum/Keeping_the_overview_-_git_annex_list_--compact__63__.mdwn new file mode 100644 index 0000000000..31d1bd90f7 --- /dev/null +++ b/doc/forum/Keeping_the_overview_-_git_annex_list_--compact__63__.mdwn @@ -0,0 +1,31 @@ +Is there a way to obtain/keep the overview over a git annex repository with many directories and files? For example to answer the following question in a compact way: which subdirectories of the current directory are complete locally? + +The needed information is provided by both `git annex list` and `git annex info`, but the output of both command is very verbose and difficult to parse visually. + +For example part of the output of `git annex list` could be: + +``` +XXX__ foo/bar/a +XXX__ foo/bar/b +XXX__ foo/bar/c +XXX__ foo/hey/d +XXX__ foo/hey/e +XXX__ foo/hey/f +``` + +Assuming that the `foo` directory contains no further files this could be compacted to + +``` +XXX__ foo/bar/ +XXX__ foo/hey/ +``` + +or even + +``` +XXX__ foo/ +``` + +Of course the above procedure would only compact directories if all files have the same presence status. But I guess that this will be true in many cases. + +I’m thinking about implementing this as a filter to the output of `git annex list`, but wanted to check whether similar ideas haven’t been considered here before. (I could not find anything.) diff --git a/doc/forum/Reasonable_annex.bloomcapacity__63__.mdwn b/doc/forum/Reasonable_annex.bloomcapacity__63__.mdwn new file mode 100644 index 0000000000..b5cafce72e --- /dev/null +++ b/doc/forum/Reasonable_annex.bloomcapacity__63__.mdwn @@ -0,0 +1,15 @@ +What is a reasonable value for ```annex.bloomcapacity``` in this situation and in what unit? + + local annex keys: 670671 + local annex size: 2.62 terabytes + annexed files in working tree: 1410199 + size of annexed files in working tree: 3.52 terabytes + combined annex size of all repositories: 2.63 terabytes + annex sizes of repositories: + 2.62 TB: b2e77041-584e-4699-947d-ef5004273901 -- pudong [here] + 7.39 GB: 42a53a13-7ad6-43a5-95ef-813d4f8c1a6f -- [batam_payload] + backend usage: + SHA256E: 1410199 + bloom filter size: 32 mebibytes (appears too small for this repository; adjust annex.bloomcapacity) + +I don't really understand what happens when the bloom filter is too small. diff --git a/doc/forum/Reasonable_annex.bloomcapacity__63__/comment_1_0361164f67f44520f3af5ba88adc3868._comment b/doc/forum/Reasonable_annex.bloomcapacity__63__/comment_1_0361164f67f44520f3af5ba88adc3868._comment new file mode 100644 index 0000000000..2a63d58ddd --- /dev/null +++ b/doc/forum/Reasonable_annex.bloomcapacity__63__/comment_1_0361164f67f44520f3af5ba88adc3868._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2025-01-29T15:54:19Z" + content=""" +What will happen with too small a bloom filter is `git-annex unused` may +think that some keys are used which are really not. And +`git-annex sync --content` may operate on some keys that are not in the +work tree. + +The `git-annex info` command displays how much memory the configured bloom +filters use, which is why it's reporting 32 membibytes. But the +annex.bloomcapacity setting is the number of expected files in the work +tree, by default 500000. + +It would probably make sense for you to set it to 2000000 or so unless +your system has an unusually small amount of RAM. +"""]] diff --git a/doc/forum/b2_rclone_initremote_failed__58___empty_folders/comment_1_bfceb1a2ce7d9c70b1c79252a8e12822._comment b/doc/forum/b2_rclone_initremote_failed__58___empty_folders/comment_1_bfceb1a2ce7d9c70b1c79252a8e12822._comment new file mode 100644 index 0000000000..751c577e29 --- /dev/null +++ b/doc/forum/b2_rclone_initremote_failed__58___empty_folders/comment_1_bfceb1a2ce7d9c70b1c79252a8e12822._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="dmcardle" + avatar="http://cdn.libravatar.org/avatar/b79468a0d03ec3ec7acbae547c4fa994" + subject="comment 1" + date="2025-01-29T14:44:37Z" + content=""" +Hi, author of rclone's \"gitannex\" command here. Sorry you're running into trouble with it! + +Based on the text, that error is definitely coming from [gitannex.go](https://github.com/rclone/rclone/blob/6494ac037fdd7ada2052ae9a8e05d230e7e686b2/cmd/gitannex/gitannex.go#L276). + +I believe that my intent was to detect that the following mkdir would fail, and offer up a more specific error message rather than letting it fail. + +I don't know anything about Backblaze B2, unfortunately. I suppose we could work around the issue by creating an empty file underneath the place we want the empty directory. Sounds plausible, right? + +Would you mind trying to make an empty directory on your B2 remote to verify it fails? Something like `rclone mkdir myremote:newdir`. + +And also try touching a file in a new directory to verify it's possible in one go? Something like `rclone touch --recursive myremote:newdir/newfile.txt`. +"""]] diff --git a/doc/todo/compute_special_remote/comment_13_b6b1c8e9dc9e1d818036385fd073ed21._comment b/doc/todo/compute_special_remote/comment_13_b6b1c8e9dc9e1d818036385fd073ed21._comment new file mode 100644 index 0000000000..372c1feb7b --- /dev/null +++ b/doc/todo/compute_special_remote/comment_13_b6b1c8e9dc9e1d818036385fd073ed21._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="matrss" + avatar="http://cdn.libravatar.org/avatar/cd1c0b3be1af288012e49197918395f0" + subject="comment 13" + date="2025-01-29T09:56:12Z" + content=""" +> @m.risse in your example the \"data.nc\" file gets new content when retrieved from the special remote and the source file has changed. + +True, that can happen, and the user was explicit in that they either don't care about it (non-checksum backend, URL in my PoC), or do care (checksum backend) and git-annex would fail the checksum verification. + +> But if you already have data.nc file present in a repository, it does not get updated immediately when you update the source \"data.grib\" file. +> +> So, a drop and re-get of a file changes the version of the file you have available. For that matter, if the old version has been stored on other remotes, a get may retrieve either an old or a new version. That is not intuitive and it makes me wonder if using a special remote is really a good fit for what you're wanting to do + +This I haven't entirely thought through. I'd say if the key uses a non-checksum backend, then it can only be assumed and is the users responsibility that the resulting file is functionally, even if not bit-by-bit, identical. E.g. with netCDF checksums can differ due to small details like chunking, but the data might be the same. With a checksum backend git-annex would just fail the next recompute, but the interactions with copies on other remotes could indeed get confusing. + +> In your \"cdo\" example, it's not clear to me if the new version of the software generates an identical file to the old, or if it has a bug fix that causes it to generate a significantly different output. If the two outputs are significantly different then treating them as the same git-annex key seems questionable to me. + +Again, two possible cases depending on if the key uses a checksum or a non-checksum backend. With a checksum: if the new version produces the same output everything is fine; if the new version produces different output then git-annex would indicate this discrepancy on the next recompute and the user has to decide how to handle it (probably by checking that the output of the new version is either functionally the same or in some way \"better\" than the old one and updating the repository to record this new key as that file). + +Without a checksum backend the user would again have been explicit in that they don't care if the data changes for whatever reason, the key is essentially just a placeholder for the computation without a guarantee about its content. + +Something like VURL would be a compromise between the two: it would avoid the upfront cost of computing all files (which might be very expensive), but still instruct git-annex to error out if the checksum changes at some point after the first compute. A regular migration of the computed-files-so-far to a checksum backend could achieve the same. +"""]] diff --git a/doc/todo/compute_special_remote/comment_14_f0a575875e1f8809906ba4021e879b43._comment b/doc/todo/compute_special_remote/comment_14_f0a575875e1f8809906ba4021e879b43._comment new file mode 100644 index 0000000000..7d3624c83d --- /dev/null +++ b/doc/todo/compute_special_remote/comment_14_f0a575875e1f8809906ba4021e879b43._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="matrss" + avatar="http://cdn.libravatar.org/avatar/cd1c0b3be1af288012e49197918395f0" + subject="comment 14" + date="2025-01-29T10:13:59Z" + content=""" +Some thoughts regarding your ideas: + +- Multiple output files could always be emulated by generating a single archive file and registering additional compute instructions that simply extract each output file from that archive. I think there could be some convenience functionality on the CLI side to set that up and the key of the archive file might not even need to correspond to an actual file in the tree. +- For my use-cases (and I think DataLad at large) it is important to make this feature work across repository boundaries. E.g. I would like to use this feature to build a derived dataset from , where exactly this conversion from grib to netcdf happens in the compute step. I'd like to have the netcdf outputs as a separate dataset as some users might only be interested in the grib files, and it would scale better when there is more than just one kind of output that can be derived from an input by computation. `git annex get` doesn't work recursively across submodules/subdatasets though, and `datalad get` does not understand keys, just paths (at least so far). +"""]]