From 67baa63e5bd45ef8b688e8ffb64f1e720316c288 Mon Sep 17 00:00:00 2001 From: "asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" Date: Wed, 20 Oct 2021 22:54:39 +0000 Subject: [PATCH 01/19] Added a comment --- ..._93c9d3ea6d7a1f2d035f8a52c81790ad._comment | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_10_93c9d3ea6d7a1f2d035f8a52c81790ad._comment diff --git a/doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_10_93c9d3ea6d7a1f2d035f8a52c81790ad._comment b/doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_10_93c9d3ea6d7a1f2d035f8a52c81790ad._comment new file mode 100644 index 0000000000..15b06ccc7f --- /dev/null +++ b/doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_10_93c9d3ea6d7a1f2d035f8a52c81790ad._comment @@ -0,0 +1,71 @@ +[[!comment format=mdwn + username="asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" + nickname="asakurareiko" + avatar="http://cdn.libravatar.org/avatar/a865743e357add9d15081840179ce082" + subject="comment 10" + date="2021-10-20T22:54:39Z" + content=""" +The error I get previously (before [[!commit 0f38ad9a6]]) with my test case is + +``` +init + Detected a filesystem without fifo support. + + Disabling ssh connection caching. +ok +(recording state in git...) +get a (from origin...) +ok +get b (from origin...) +ok +(recording state in git...) +sqlite worker thread crashed: user error (SQLite3 returned ErrorProtocol while attempting to perform prepare \"SELECT null from content limit 1\": locking protocol(while opening database connection)) +git-annex: sqlite query crashed: thread blocked indefinitely in an MVar operation +CallStack (from HasCallStack): + error, called at ./Database/Handle.hs:102:40 in main:Database.Handle +error: external filter 'git-annex smudge --clean -- %f' failed 1 +error: external filter 'git-annex smudge --clean -- %f' failed +sqlite worker thread crashed: user error (SQLite3 returned ErrorProtocol while attempting to perform prepare \"SELECT null from content limit 1\": locking protocol(while opening database connection)) +git-annex: sqlite query crashed: thread blocked indefinitely in an MVar operation +CallStack (from HasCallStack): + error, called at ./Database/Handle.hs:102:40 in main:Database.Handle +error: external filter 'git-annex smudge --clean -- %f' failed 1 +error: external filter 'git-annex smudge --clean -- %f' failed +``` + +With [[!commit d0ef8303c]], the test case still works, but adjusted branches still have the same error. + +``` +git init +git annex init +git config annex.crippledfilesystem true +echo aaa > a +cp a b +git annex add . +git commit -m . +git annex adjust --unlock +``` + +produces + +``` +adjust +sqlite worker thread crashed: user error (SQLite3 returned ErrorProtocol while attempting to perform prepare \"SELECT null from content limit 1\": locking protocol(while opening database connection)) +git-annex: thread blocked indefinitely in an MVar operation +error: external filter 'git-annex smudge -- %f' failed 1 +error: external filter 'git-annex smudge -- %f' failed +sqlite worker thread crashed: user error (SQLite3 returned ErrorProtocol while attempting to perform prepare \"SELECT null from content limit 1\": locking protocol(while opening database connection)) +git-annex: thread blocked indefinitely in an MVar operation +error: external filter 'git-annex smudge -- %f' failed 1 +error: external filter 'git-annex smudge -- %f' failed +Switched to branch 'adjusted/master(unlocked)' +sqlite worker thread crashed: user error (SQLite3 returned ErrorProtocol while attempting to perform prepare \"SELECT null from content limit 1\": locking protocol(while opening database connection)) +git-annex: sqlite query crashed: thread blocked indefinitely in an MVar operation +CallStack (from HasCallStack): + error, called at ./Database/Handle.hs:79:40 in main:Database.Handle +failed +adjust: 1 failed +``` + +About `git-annex version`, I'm using `make install-home` to do an incremental build but the version does not update. +"""]] From c6d1c7d2098c01cf4b419b4b48ef1b3e3ea951d4 Mon Sep 17 00:00:00 2001 From: jwodder Date: Thu, 21 Oct 2021 01:35:48 +0000 Subject: [PATCH 02/19] --- doc/bugs/metadata_cmd._vs._--json-error-messages.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/bugs/metadata_cmd._vs._--json-error-messages.mdwn diff --git a/doc/bugs/metadata_cmd._vs._--json-error-messages.mdwn b/doc/bugs/metadata_cmd._vs._--json-error-messages.mdwn new file mode 100644 index 0000000000..9f5524636c --- /dev/null +++ b/doc/bugs/metadata_cmd._vs._--json-error-messages.mdwn @@ -0,0 +1,6 @@ +(Sorry about the title; I was trying to work within the character limit.) + +When invoking `git-annex metadata --batch --json --json-error-messages`, if an error occurs in response to some input — say, because the name of a nonexistent file was supplied (or, in my case, because the name of a file downloaded milliseconds ago in a parallel addurl process was supplied) — then `git-annex metadata` will output "git-annex: not an annexed file: {filepath}" to standard error and immediately exit. Not only is this in contrast to what it seems `--json-error-messages` should do, but the "exiting immediately" bit is in contrast to my understanding of how batch mode is supposed to work. Surely this should be fixed? + +[[!meta author=jwodder]] +[[!tag projects/dandi]] From ae80b1214ca8375e58e8206ba973a5239574d53c Mon Sep 17 00:00:00 2001 From: "asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" Date: Thu, 21 Oct 2021 02:06:50 +0000 Subject: [PATCH 03/19] Added a comment --- ..._94d00ed84dfebebd7889fa861172df61._comment | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_11_94d00ed84dfebebd7889fa861172df61._comment diff --git a/doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_11_94d00ed84dfebebd7889fa861172df61._comment b/doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_11_94d00ed84dfebebd7889fa861172df61._comment new file mode 100644 index 0000000000..b01e115845 --- /dev/null +++ b/doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_11_94d00ed84dfebebd7889fa861172df61._comment @@ -0,0 +1,46 @@ +[[!comment format=mdwn + username="asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" + nickname="asakurareiko" + avatar="http://cdn.libravatar.org/avatar/a865743e357add9d15081840179ce082" + subject="comment 11" + date="2021-10-21T02:06:50Z" + content=""" +I found a new type of failure which occurs when there are new unlocked files in the index. + +``` +git init +git annex init +git config annex.crippledfilesystem true +git config annex.addunlocked true +touch a +git annex add . # OK +touch b +git annex add . # 1 error +touch c +git annex add . # 2 errors +``` + +Something is happening to the files already in the index and the error is triggered once per file in the index. + +``` +add c +ok +sqlite worker thread crashed: user error (SQLite3 returned ErrorProtocol while attempting to perform prepare \"SELECT null from content limit 1\": locking protocol(while opening database connection)) +git-annex: sqlite query crashed: thread blocked indefinitely in an MVar operation +CallStack (from HasCallStack): + error, called at ./Database/Handle.hs:102:40 in main:Database.Handle +error: external filter 'git-annex smudge --clean -- %f' failed 1 +error: external filter 'git-annex smudge --clean -- %f' failed +add a +ok +sqlite worker thread crashed: user error (SQLite3 returned ErrorProtocol while attempting to perform prepare \"SELECT null from content limit 1\": locking protocol(while opening database connection)) +git-annex: sqlite query crashed: thread blocked indefinitely in an MVar operation +CallStack (from HasCallStack): + error, called at ./Database/Handle.hs:102:40 in main:Database.Handle +error: external filter 'git-annex smudge --clean -- %f' failed 1 +error: external filter 'git-annex smudge --clean -- %f' failed +add b +ok +(recording state in git...) +``` +"""]] From f8faeabb709f54d0c3dd8424735730cf55274745 Mon Sep 17 00:00:00 2001 From: "asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" Date: Thu, 21 Oct 2021 02:09:02 +0000 Subject: [PATCH 04/19] Added a comment --- ..._76f765e3befd5e263a8863b56cad139b._comment | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_12_76f765e3befd5e263a8863b56cad139b._comment diff --git a/doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_12_76f765e3befd5e263a8863b56cad139b._comment b/doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_12_76f765e3befd5e263a8863b56cad139b._comment new file mode 100644 index 0000000000..adaae56df5 --- /dev/null +++ b/doc/bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol/comment_12_76f765e3befd5e263a8863b56cad139b._comment @@ -0,0 +1,45 @@ +[[!comment format=mdwn + username="asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" + nickname="asakurareiko" + avatar="http://cdn.libravatar.org/avatar/a865743e357add9d15081840179ce082" + subject="comment 12" + date="2021-10-21T02:09:02Z" + content=""" +Forgot to add in the previous comment. The index looks fine afterwards + +``` +On branch master + +No commits yet + +Changes to be committed: + (use \"git rm --cached ...\" to unstage) + new file: a + new file: b + new file: c +``` + +``` +diff --git a/a b/a +new file mode 100755 +index 0000000..f8e47b9 +--- /dev/null ++++ b/a +@@ -0,0 +1 @@ ++/annex/objects/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +diff --git a/b b/b +new file mode 100755 +index 0000000..f8e47b9 +--- /dev/null ++++ b/b +@@ -0,0 +1 @@ ++/annex/objects/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +diff --git a/c b/c +new file mode 100755 +index 0000000..f8e47b9 +--- /dev/null ++++ b/c +@@ -0,0 +1 @@ ++/annex/objects/SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +``` +"""]] From 471464e4a0831f94068738551541c78962254aea Mon Sep 17 00:00:00 2001 From: jwodder Date: Thu, 21 Oct 2021 04:54:26 +0000 Subject: [PATCH 05/19] --- .../metadata_--batch_--json_should_fail_on_bad_fields.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/bugs/metadata_--batch_--json_should_fail_on_bad_fields.mdwn diff --git a/doc/bugs/metadata_--batch_--json_should_fail_on_bad_fields.mdwn b/doc/bugs/metadata_--batch_--json_should_fail_on_bad_fields.mdwn new file mode 100644 index 0000000000..0441664fcd --- /dev/null +++ b/doc/bugs/metadata_--batch_--json_should_fail_on_bad_fields.mdwn @@ -0,0 +1,3 @@ +When setting file metadata using `git-annex metadata --batch --json --json-error-messages`, if the "fields" field of an input line is not 100% an object whose values are arrays of strings, then git-annex will silently ignore the "fields" field and act as though the user simply requested the metadata for the given file/key. It would be more useful if, whenever the input contains a "fields" field that does not match the required schema, git annex treats it as an error. This would make it easier for users to figure out that they are doing something wrong. + +[[!meta author=jwodder]] From efaa1fe94a1c6d1831918df058666777427dddc3 Mon Sep 17 00:00:00 2001 From: "git-annex.visiteur@e9d364191d2ffc1b163c8d9e4c57dbadf58aad8e" Date: Thu, 21 Oct 2021 08:24:55 +0000 Subject: [PATCH 06/19] Added a comment --- ..._678a0136f143d2d7874226b3fc744eb2._comment | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 doc/bugs/git-annex_died_of_signal_11_when_syncing_content/comment_4_678a0136f143d2d7874226b3fc744eb2._comment diff --git a/doc/bugs/git-annex_died_of_signal_11_when_syncing_content/comment_4_678a0136f143d2d7874226b3fc744eb2._comment b/doc/bugs/git-annex_died_of_signal_11_when_syncing_content/comment_4_678a0136f143d2d7874226b3fc744eb2._comment new file mode 100644 index 0000000000..2665ce27e8 --- /dev/null +++ b/doc/bugs/git-annex_died_of_signal_11_when_syncing_content/comment_4_678a0136f143d2d7874226b3fc744eb2._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="git-annex.visiteur@e9d364191d2ffc1b163c8d9e4c57dbadf58aad8e" + nickname="git-annex.visiteur" + avatar="http://cdn.libravatar.org/avatar/59640df9d44f100f0bf98c1cbb430037" + subject="comment 4" + date="2021-10-21T08:24:55Z" + content=""" +Yes the problem occurs each time I want to do git annex get . on a repository with about 20 000 files. + +As i say, I've perfom memtest86 (several times) and memtester (several times) without any problem. Do you think it could be hardware problem despite these results ? + +You suggest using gdb. I get + +Thread 1 \"git-annex\" received signal SIGSEGV, Segmentation fault. +0x0000000003c6be67 in ?? () + +(gdb) backtrace + +#0 0x0000000003c6be67 in ?? () +#1 0x0000000000000000 in ?? () + +which is not very helpful. Do you have any advices to investigate? + +"""]] From f9e09277c5a587fac93a2405290aa7ad76d1079f Mon Sep 17 00:00:00 2001 From: "asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" Date: Thu, 21 Oct 2021 17:17:42 +0000 Subject: [PATCH 07/19] --- doc/bugs/git_commit_smudges_unncessarily.mdwn | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/bugs/git_commit_smudges_unncessarily.mdwn diff --git a/doc/bugs/git_commit_smudges_unncessarily.mdwn b/doc/bugs/git_commit_smudges_unncessarily.mdwn new file mode 100644 index 0000000000..b0aae6b491 --- /dev/null +++ b/doc/bugs/git_commit_smudges_unncessarily.mdwn @@ -0,0 +1,20 @@ +### Please describe the problem. + +git commit smudges unlocked files in the index when they were already smudged with git status. + +### What steps will reproduce the problem? + +``` +for i in {1..200}; do echo $i > $i; done +git init; git annex init; git config annex.thin true; git config annex.crippledfilesystem true; git config annex.addunlocked true; git annex add .; git status; time git commit -m . +``` + +git commit should take less than a second but takes several seconds to smudge the files in the index. + +### What version of git-annex are you using? On what operating system? + +Through bisection, the problem was found to be introduced in [[!commit 428c91606b434512d1986622e751c795edf4df44]]. Problem occurs both in Linux and WSL1. + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +git-annex has worked wonderfully for managing my files across different machines and cloud storage services. From 201ac941a7f3b7166d68ac679bbec867dc29f09f Mon Sep 17 00:00:00 2001 From: "asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" Date: Thu, 21 Oct 2021 21:24:24 +0000 Subject: [PATCH 08/19] --- ...git_checkout_slow_with_unlocked_files.mdwn | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 doc/bugs/git_checkout_slow_with_unlocked_files.mdwn diff --git a/doc/bugs/git_checkout_slow_with_unlocked_files.mdwn b/doc/bugs/git_checkout_slow_with_unlocked_files.mdwn new file mode 100644 index 0000000000..6f99b87f9c --- /dev/null +++ b/doc/bugs/git_checkout_slow_with_unlocked_files.mdwn @@ -0,0 +1,28 @@ +### Please describe the problem. + +git checkout is very slow to checkout unlocked files. + +### What steps will reproduce the problem? + +Setup: + +``` +for i in {1..1000}; do echo $i > $i; done +git init +git annex init +git config annex.addunlocked true +git annex add . +git commit -m . +git annex lock . +git reset . +``` + +Doing a `git checkout .` at this point took about a minute. In comparison doing a `git annex unlock . && git status` took 30 seconds. + +### What version of git-annex are you using? On what operating system? + +8.20210428 in Linux. + +### Please provide any additional information below. + +The relevance of this problem is that I tried to go from a lock-adjusted branch to its basis with about 50000 unlocked files with `git checkout`. It was taking longer than expected and there were many warnings about a cosmetic problem and advised me to run `git update-index -q --refresh `. Eventually I deleted the entire working tree, did `git reset --hard` to recreate the unlocked file stubs, locked every file and unlocked them again. From 29dfc0a9bf9c6d028b8c015952aef632214ca8a8 Mon Sep 17 00:00:00 2001 From: "asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" Date: Fri, 22 Oct 2021 15:58:11 +0000 Subject: [PATCH 09/19] Added a comment --- ..._a87baf5ad6b7688bb40a6c463cb1b8db._comment | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 doc/todo/More_space_savings_for_annex.thin/comment_4_a87baf5ad6b7688bb40a6c463cb1b8db._comment diff --git a/doc/todo/More_space_savings_for_annex.thin/comment_4_a87baf5ad6b7688bb40a6c463cb1b8db._comment b/doc/todo/More_space_savings_for_annex.thin/comment_4_a87baf5ad6b7688bb40a6c463cb1b8db._comment new file mode 100644 index 0000000000..bf7dab77fc --- /dev/null +++ b/doc/todo/More_space_savings_for_annex.thin/comment_4_a87baf5ad6b7688bb40a6c463cb1b8db._comment @@ -0,0 +1,47 @@ +[[!comment format=mdwn + username="asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" + nickname="asakurareiko" + avatar="http://cdn.libravatar.org/avatar/a865743e357add9d15081840179ce082" + subject="comment 4" + date="2021-10-22T15:58:11Z" + content=""" +I understand `annex.thinmode=forcehardlink` presents a greater risk. Personally that's something I can manage. In case anyone else is interested in doing the same, this patch will make `git annex fix` always create hardlinks with `annex.thin=true`. + +``` +diff --git a/Annex/Content/LowLevel.hs b/Annex/Content/LowLevel.hs +index c686e462e..b7cf95d24 100644 +--- a/Annex/Content/LowLevel.hs ++++ b/Annex/Content/LowLevel.hs +@@ -57,10 +57,8 @@ linkOrCopy' canhardlink key src dest destmode = catchDefaultIO Nothing $ + where + hardlink = do + s <- getstat +- if linkCount s > 1 +- then copy s +- else liftIO (R.createLink src dest >> preserveGitMode dest destmode >> return (Just Linked)) +- `catchIO` const (copy s) ++ liftIO (R.createLink src dest >> preserveGitMode dest destmode >> return (Just Linked)) ++ `catchIO` const (copy s) + copy s = ifM (checkedCopyFile' key src dest destmode s) + ( return (Just Copied) + , return Nothing +diff --git a/Command/Fix.hs b/Command/Fix.hs +index 6a84652ce..40a93b357 100644 +--- a/Command/Fix.hs ++++ b/Command/Fix.hs +@@ -62,10 +62,10 @@ start fixwhat si file key = do + thin <- annexThin <$> Annex.getGitConfig + fs <- liftIO $ catchMaybeIO $ R.getFileStatus file + os <- liftIO $ catchMaybeIO $ R.getFileStatus obj +- case (linkCount <$> fs, linkCount <$> os, thin) of +- (Just 1, Just 1, True) -> ++ case (fileID <$> fs, fileID <$> os, thin) of ++ (Just n, Just n', True) | n /= n' -> + fixby $ makeHardLink file key +- (Just n, Just n', False) | n > 1 && n == n' -> ++ (Just n, Just n', False) | n == n' -> + fixby $ breakHardLink file key obj + _ -> stop + +``` +"""]] From ebe022e9cf2fa2fd078f989f89180397a1b1ff1a Mon Sep 17 00:00:00 2001 From: "asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" Date: Fri, 22 Oct 2021 22:13:46 +0000 Subject: [PATCH 10/19] --- .../Using_git-annex_on_NTFS_with_WSL1.mdwn | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 doc/tips/Using_git-annex_on_NTFS_with_WSL1.mdwn diff --git a/doc/tips/Using_git-annex_on_NTFS_with_WSL1.mdwn b/doc/tips/Using_git-annex_on_NTFS_with_WSL1.mdwn new file mode 100644 index 0000000000..3b257d7f7d --- /dev/null +++ b/doc/tips/Using_git-annex_on_NTFS_with_WSL1.mdwn @@ -0,0 +1,32 @@ +The following steps are tested on Windows 10 21h1 and are designed to avoid these two bugs: + +* [[bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol]] +* [[bugs/WSL1__58___git-annex-add_fails_in_DrvFs_filesystem]] + +**Setup** + +* Enable Developer mode in Windows settings so that symlinks can be created without elevated privileges. +* Mount the NTFS drive with metadata option. [`/etc/wsl.conf`](https://docs.microsoft.com/en-us/windows/wsl/wsl-config) can be used or a line such as `C: /mnt/c drvfs metadata` can be added in `/etc/fstab`. +* Create an empty directory for the repo. + * With File Explorer go to Properties/Security/Advanced. + * Give "Authenticated Users" permission to [“Write attributes”, “Create files”, “Create folders” and “Delete subfolders and files”](https://devblogs.microsoft.com/commandline/improved-per-directory-case-sensitivity-support-in-wsl/) for this folder and subfolders. + * Enable case sensitivity with `setfattr -n system.wsl_case_sensitive -v 1 `. + * This attribute will be inherited by new subdirectories but will not be automatically recursively applied to existing subdirectories. + * Set this attribute to 0 as appropriate if you do not have files that differ only by case, or if you have non-default git-annex [[tuning]]. + * This attribute can be queried with `getfattr -n system.wsl_case_sensitive `. +* Initialize the repo. + * Set `git config annex.crippledfilesystem true` immediately after `git annex init` to avoid triggering the above two bugs. +* Open Properties/Security/Advanced for .git/annex/objects. + * Deny "Authenticated Users" write permissions for files only and select "Replace all child object permission entries with inheritable permission entries from this object". + +** Setup with repo cloned from SSH ** + +* If you encounter problems with older versions of git-annex. + * Create a local git-annex branch from the remote branch. + * Remove the origin remote. + * Do `git annex init` and other setup. + * Add back the origin remote. + +** Using symlinks and locked files ** + +* Symlinks might be broken and can be fixed by recreating them. You can also delete them and do a `git checkout`. From 42eaa6f8c90303b93495415492b7410f8c5ada68 Mon Sep 17 00:00:00 2001 From: "asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" Date: Fri, 22 Oct 2021 22:14:40 +0000 Subject: [PATCH 11/19] Moved WSL1 guide to a tips page --- doc/todo/windows_support.mdwn | 40 +---------------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/doc/todo/windows_support.mdwn b/doc/todo/windows_support.mdwn index f68aee8569..11e1d7d522 100644 --- a/doc/todo/windows_support.mdwn +++ b/doc/todo/windows_support.mdwn @@ -131,42 +131,4 @@ Seems like this would need Windows 10. > > > > But here's a bug about sqlite in WSL: > > > > [[bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol]] --[[Joey]] -## Update Oct 2021: Steps for using git-annex on NTFS with WSL1 (an experimental setup for those adventurous enough) - -The following steps are tested on Windows 10 21h1 with Ubuntu 18.04/20.04 and are specifically designed to work around these two bugs: - -* [[bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol]] -* [[bugs/WSL1__58___git-annex-add_fails_in_DrvFs_filesystem]] - -Do the following: - -1. Enable Developer mode in Windows settings so that symlinks can be created without elevated privileges. - -2. Mount the NTFS drive with metadata option. [`/etc/wsl.conf`](https://docs.microsoft.com/en-us/windows/wsl/wsl-config) can be used or a line such as `C: /mnt/c drvfs metadata` can be added in `/etc/fstab`. - -3. Create an empty directory where your repo will be. Then enable case sensitivity `setfattr -n system.wsl_case_sensitive -v 1 `. This attribute will be inherited by new subdirectories. If setfattr(1) errs out with permission denied, you can also effect the same change in CMD.EXE / Windows Powershell as admin with `fsutil file setCaseSensitiveInfo enable`.[^1] You can check that the setting is enabled with `getfattr -n system.wsl_case_sensitive ` under WSL1. - - If you do not have files which may be differ only case, you do not need to set this on the entire repository, only for `.git/annex`. If in addition you have repository [[tuning]] set to use only lowercase hash directories, you do not need to set this at all. - -4. Create the repo however you like (see steps below for cloning a repo with ssh). Immediately after `git annex init`, do `git config annex.crippledfilesystem true`. If you set `crippledfilesystem` before init, then git annex will try to enter an adjusted branch and trigger the first bug. If you do not set `crippledfilesystem` after init, you will trigger the second bug when doing `git annex add`. - -[^1]: This works because Administrators usually have Full Control over most files. What Windows actually looks for is "Write attributes", "Create files", "Create folders" and "Delete subfolders and files" permissions on the directory required for changing case-sensitivity. As a regular user (or without UAC) you might not have those permissions by default for instance on external drives, so adjust accordingly. For more info about about the `system.wsl_case_sensitive` attribute see this blog post: [[https://devblogs.microsoft.com/commandline/improved-per-directory-case-sensitivity-support-in-wsl/]] - -### Cloning a repo with ssh - -When cloning a repo with ssh, `git annex init` will fail to enable ssh remotes if `crippledfilesystem` is not set, but you also cannot set it before init. Follow these steps to avoid unrelated history in the `git-annex` branch. - - git clone annex - cd annex - git branch git-annex origin/git-annex - git remote remove origin - git annex init - git config annex.crippledfilesystem true - git remote add origin - git annex sync - -### Using symlinks and locked files - -* You can now use symlinks and locked files but please remember that locked files can still be overwritten. So make sure to unlock them before you edit them. -* After you `git annex get` files, the symlinks for those files will still be broken. Recreate the symlinks to fix them. You can make a script or delete them and `git checkout`. -* It can be difficult to use symlinks on Windows because programs will see the link target rather than the link, which makes it impossible to do things like navigating between files in the same directory or using opened file history. You can unlock the files or access them through another filesystem layer such as SMB. +See also [[tips/Using_git-annex_on_NTFS_with_WSL1]]. From 6f3cadcbda64edc7719e4a4075a26e06c9e0bb2c Mon Sep 17 00:00:00 2001 From: username Date: Sat, 23 Oct 2021 15:53:57 +0000 Subject: [PATCH 12/19] Added a comment --- ...ment_20_a80b9ce3a78fb44c9d60e877146d9e59._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_20_a80b9ce3a78fb44c9d60e877146d9e59._comment diff --git a/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_20_a80b9ce3a78fb44c9d60e877146d9e59._comment b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_20_a80b9ce3a78fb44c9d60e877146d9e59._comment new file mode 100644 index 0000000000..27047bee59 --- /dev/null +++ b/doc/forum/Managing_a_large_number_of_files_archived_on_many_pieces_of_read-only_medium___40__E.G._DVDs__41__/comment_20_a80b9ce3a78fb44c9d60e877146d9e59._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="username" + avatar="http://cdn.libravatar.org/avatar/3c17ce77d299219a458fc2eff973239a" + subject="comment 20" + date="2021-10-23T15:53:57Z" + content=""" +Aha, that's what I was missing! + +I was working under the wrong assumption that the git-annex location tracking was tied in some way to the master branch and that alternate branches required shared history with master to have their contents tracked. + +Wholly independent branches for each disc is exactly what I want, thanks Lukey and Joey for your suggestions. +"""]] From 4471aae22f201369678029c373cdaf610228926b Mon Sep 17 00:00:00 2001 From: jkniiv Date: Sun, 24 Oct 2021 15:10:13 +0000 Subject: [PATCH 13/19] still think we should highlight this as new info --- doc/todo/windows_support.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/todo/windows_support.mdwn b/doc/todo/windows_support.mdwn index 11e1d7d522..e2ea77a613 100644 --- a/doc/todo/windows_support.mdwn +++ b/doc/todo/windows_support.mdwn @@ -131,4 +131,6 @@ Seems like this would need Windows 10. > > > > But here's a bug about sqlite in WSL: > > > > [[bugs/WSL_adjusted_braches__58___smudge_fails_with_sqlite_thread_crashed_-_locking_protocol]] --[[Joey]] -See also [[tips/Using_git-annex_on_NTFS_with_WSL1]]. +## update Oct 2021: + +Git-annex has become more reliable on WSL1 recently, see [[tips/Using_git-annex_on_NTFS_with_WSL1]]. From 63313e0b40b552072cdb508b0cf361bc6fbfcb38 Mon Sep 17 00:00:00 2001 From: "asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" Date: Sun, 24 Oct 2021 19:12:23 +0000 Subject: [PATCH 14/19] --- ...l_freezeContent_after_move_into_annex.mdwn | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 doc/todo/__91__PATCH__93___Call_freezeContent_after_move_into_annex.mdwn diff --git a/doc/todo/__91__PATCH__93___Call_freezeContent_after_move_into_annex.mdwn b/doc/todo/__91__PATCH__93___Call_freezeContent_after_move_into_annex.mdwn new file mode 100644 index 0000000000..eed83c62b5 --- /dev/null +++ b/doc/todo/__91__PATCH__93___Call_freezeContent_after_move_into_annex.mdwn @@ -0,0 +1,41 @@ +``` +From d863d357d89e7e584755dd09c6af18eedefc9557 Mon Sep 17 00:00:00 2001 +From: Reiko Asakura +Date: Sun, 24 Oct 2021 14:44:44 -0400 +Subject: [PATCH] Call freezeContent after move into annex + +This change better supports Windows ACL management using +annex.freezecontent-command and annex.thawcontent-command and matches +the behaviour of adding an unlocked file. + +By calling freezeContent after the file has moved into the annex, +the file's delete permission can be denied. If the file's delete +permission is denied before moving into the annex, the file cannot +be moved or deleted. If the file's delete permission is not denied after +moving into the annex, it will likely inherit a grant for the delete +permission which allows it to be deleted irrespective of the permissions +of the parent directory. +--- + Annex/Content.hs | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Annex/Content.hs b/Annex/Content.hs +index da65143ab..3664c6ea2 100644 +--- a/Annex/Content.hs ++++ b/Annex/Content.hs +@@ -342,10 +342,10 @@ moveAnnex key af src = ifM (checkSecureHashes' key) + storeobject dest = ifM (liftIO $ R.doesPathExist dest) + ( alreadyhave + , adjustedBranchRefresh af $ modifyContent dest $ do +- freezeContent src + liftIO $ moveFile + (fromRawFilePath src) + (fromRawFilePath dest) ++ freezeContent dest + g <- Annex.gitRepo + fs <- map (`fromTopFilePath` g) + <$> Database.Keys.getAssociatedFiles key +-- +2.30.2 + +``` From be3cf1115e68e81402e3ba0327e3243c2e09b7c9 Mon Sep 17 00:00:00 2001 From: jwodder Date: Sun, 24 Oct 2021 19:21:54 +0000 Subject: [PATCH 15/19] --- ...upkey_does_not_work_from_subdirectory.mdwn | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/bugs/lookupkey_does_not_work_from_subdirectory.mdwn diff --git a/doc/bugs/lookupkey_does_not_work_from_subdirectory.mdwn b/doc/bugs/lookupkey_does_not_work_from_subdirectory.mdwn new file mode 100644 index 0000000000..c60f4a08b7 --- /dev/null +++ b/doc/bugs/lookupkey_does_not_work_from_subdirectory.mdwn @@ -0,0 +1,21 @@ +Running `git-annex lookupkey` while in a subdirectory of a git-annex repository does not work, regardless of whether the supplied path is relative to the current directory or the root of the repository. + +### What steps will reproduce the problem? + +[[!format sh """ +git init +git-annex init +git-annex addurl --file latex/lshort.pdf https://tobi.oetiker.ch/lshort/lshort.pdf +cd latex +git-annex lookupkey lshort.pdf # Fails! +git-annex lookupkey latex/lshort.pdf # Fails! +cd .. +git-annex lookupkey latex/lshort.pdf # Succeeds +"""]] + + +### What version of git-annex are you using? On what operating system? + +8.20211011 on macOS 11.6 + +[[!meta author=jwodder]] From b2c797e35b264ed722717aed8f3d54df457a965b Mon Sep 17 00:00:00 2001 From: "asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" Date: Sun, 24 Oct 2021 21:23:56 +0000 Subject: [PATCH 16/19] Added a comment --- ...t_1_65087993c9c1b3a538a0f7b017adeafb._comment | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/bugs/lookupkey_does_not_work_from_subdirectory/comment_1_65087993c9c1b3a538a0f7b017adeafb._comment diff --git a/doc/bugs/lookupkey_does_not_work_from_subdirectory/comment_1_65087993c9c1b3a538a0f7b017adeafb._comment b/doc/bugs/lookupkey_does_not_work_from_subdirectory/comment_1_65087993c9c1b3a538a0f7b017adeafb._comment new file mode 100644 index 0000000000..6e39054a94 --- /dev/null +++ b/doc/bugs/lookupkey_does_not_work_from_subdirectory/comment_1_65087993c9c1b3a538a0f7b017adeafb._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="asakurareiko@f3d908c71c009580228b264f63f21c7274df7476" + nickname="asakurareiko" + avatar="http://cdn.libravatar.org/avatar/a865743e357add9d15081840179ce082" + subject="comment 1" + date="2021-10-24T21:23:56Z" + content=""" +Broken since [[!commit 9012fa01877a78542c64f104bd60a80825ce3c86]] where the output became like this + +``` +~/test-annex/a$ git annex lookupkey a +(\"f'\",\"a\",\"..\",False) +``` + + +"""]] From cc877dfe45ce2252a2d5b4ba167c6e9e201434b4 Mon Sep 17 00:00:00 2001 From: pat Date: Sun, 24 Oct 2021 22:42:35 +0000 Subject: [PATCH 17/19] Added a comment --- .../comment_4_558abc4b928649ff3b199a7154c052bf._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/How_to_see_what_will_go_to_annex_vs_git/comment_4_558abc4b928649ff3b199a7154c052bf._comment diff --git a/doc/forum/How_to_see_what_will_go_to_annex_vs_git/comment_4_558abc4b928649ff3b199a7154c052bf._comment b/doc/forum/How_to_see_what_will_go_to_annex_vs_git/comment_4_558abc4b928649ff3b199a7154c052bf._comment new file mode 100644 index 0000000000..e15ea7c92a --- /dev/null +++ b/doc/forum/How_to_see_what_will_go_to_annex_vs_git/comment_4_558abc4b928649ff3b199a7154c052bf._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="pat" + avatar="http://cdn.libravatar.org/avatar/6b552550673a6a6df3b33364076f8ea8" + subject="comment 4" + date="2021-10-24T22:42:35Z" + content=""" +Would you consider adding a command like this to `git-annex`? I almost always want to work with untracked files at first, which means using `git add` to add them (because `git-annex` doesn't have an `--unlocked` option afaik). Adding them to git by mistake is not good. And really it's just helpful to know when I have new files, are they going to git or to annex. +"""]] From 9f1bf486968a2bae531c35e8d1cfbde1592cbaca Mon Sep 17 00:00:00 2001 From: Lukey Date: Mon, 25 Oct 2021 10:16:35 +0000 Subject: [PATCH 18/19] --- doc/bugs/buggy_private_special_remote.mdwn | 61 ++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 doc/bugs/buggy_private_special_remote.mdwn diff --git a/doc/bugs/buggy_private_special_remote.mdwn b/doc/bugs/buggy_private_special_remote.mdwn new file mode 100644 index 0000000000..8de77f4f6a --- /dev/null +++ b/doc/bugs/buggy_private_special_remote.mdwn @@ -0,0 +1,61 @@ +### Please describe the problem. +Hello Joey,
+I added a private special remote (great feature BTW!) some time ago and everything seemed to work fine. Today I wanted to change the preferred content expression for another repo, but the repo that syncs with the special remote refused to accept the new expression and always showed (and used) the old one. + +It looked like git-annex on the affected repo still used the `preferred-content.log` from the private journal and didn't merge it with the one from the git-annex branch. + +As a workaround I manually merged both versions, put the result in `.git/annex/journal-private/preferred-content.log` and that solved the issue. + +### What steps will reproduce the problem? +[[!format sh """ +#!/bin/bash -x + +# Initialize repo a and clone it to b +git init a +cd a +git annex init a +touch a +git annex add a +git annex sync +cd .. +git clone a b +cd b +git annex init b +git annex wanted b nothing +git annex sync +cd ../a +git remote add b ../b +git annex sync + +# Add private special remote +mkdir ../special +git annex initremote --private special type=directory directory=../special encryption=none exporttree=yes importtree=no +git config remote.special.annex-tracking-branch master +git annex wanted special anything +git annex sync + +# change preferred content expression for b +git annex wanted b anything + +git annex wanted b +# preferred content expression for b is still "nothing" + +cd ../b +git annex sync +git annex wanted b +# In the other repo it's correct + + +# End of transcript or log. +"""]] + +### What version of git-annex are you using? On what operating system? + + +### Please provide any additional information below. + + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + From 0a04986ac8b6318beadfa56b66d6c4bf1da6cd4a Mon Sep 17 00:00:00 2001 From: Lukey Date: Mon, 25 Oct 2021 10:25:05 +0000 Subject: [PATCH 19/19] Added a comment --- ..._38b73aeff1f436e0629d9093c0b8497e._comment | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 doc/bugs/buggy_private_special_remote/comment_1_38b73aeff1f436e0629d9093c0b8497e._comment diff --git a/doc/bugs/buggy_private_special_remote/comment_1_38b73aeff1f436e0629d9093c0b8497e._comment b/doc/bugs/buggy_private_special_remote/comment_1_38b73aeff1f436e0629d9093c0b8497e._comment new file mode 100644 index 0000000000..2e9622f173 --- /dev/null +++ b/doc/bugs/buggy_private_special_remote/comment_1_38b73aeff1f436e0629d9093c0b8497e._comment @@ -0,0 +1,33 @@ +[[!comment format=mdwn + username="Lukey" + avatar="http://cdn.libravatar.org/avatar/c7c08e2efd29c692cc017c4a4ca3406b" + subject="comment 1" + date="2021-10-25T10:25:05Z" + content=""" +Update: It also seems to confuse location tracking in the affected repo. For example for a recently added file: +[[!format sh \"\"\" +# On the affected repo +# \"gigaset\" is the private special remote +$ git annex whereis --key SHA256E-s8073298--5b19e75adce5f4a90205ab47ce3d5a96b0a7b7f3fd465b8323a84946c652ba08.mp3 +whereis SHA256E-s8073298--5b19e75adce5f4a90205ab47ce3d5a96b0a7b7f3fd465b8323a84946c652ba08.mp3 (1 copy) + 0500a119-8fa9-47d7-ba6f-36a0ddde829e -- [gigaset] + + The following untrusted locations may also have copies: + c40ae4eb-1737-49ad-8781-c37d8d7d4f86 -- luklap [here] +ok + +# And now on another repo +$ git annex whereis --key SHA256E-s8073298--5b19e75adce5f4a90205ab47ce3d5a96b0a7b7f3fd465b8323a84946c652ba08.mp3 +whereis SHA256E-s8073298--5b19e75adce5f4a90205ab47ce3d5a96b0a7b7f3fd465b8323a84946c652ba08.mp3 (3 copies) + 173e456f-b72b-4557-8570-6c3737fe864c -- WD40EFRX-68W + d1368ba2-6c20-4b51-815e-87a47d8cc67a -- Seagate_BUP [here] + da266c97-5af7-4f50-a596-0fedfec89bb9 -- lapserv [origin] + + The following untrusted locations may also have copies: + 452d0592-bf98-4c2b-af86-f8d03327e10b -- pandora + c40ae4eb-1737-49ad-8781-c37d8d7d4f86 -- luklap +ok + + +\"\"\"]] +"""]]