From 6a59c628ccfbd5d658b75e9c6131d67ae4e30efa Mon Sep 17 00:00:00 2001 From: spwhitton Date: Sat, 23 May 2015 02:46:13 +0000 Subject: [PATCH 1/7] report gcrypt bug --- ..._a_sync_fails_because_network_is_down.mdwn | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 doc/bugs/gcrypt_gives_false___34__remote_ID_has_changed__34___warning_after_a_sync_fails_because_network_is_down.mdwn diff --git a/doc/bugs/gcrypt_gives_false___34__remote_ID_has_changed__34___warning_after_a_sync_fails_because_network_is_down.mdwn b/doc/bugs/gcrypt_gives_false___34__remote_ID_has_changed__34___warning_after_a_sync_fails_because_network_is_down.mdwn new file mode 100644 index 0000000000..35e5111b63 --- /dev/null +++ b/doc/bugs/gcrypt_gives_false___34__remote_ID_has_changed__34___warning_after_a_sync_fails_because_network_is_down.mdwn @@ -0,0 +1,43 @@ +### What steps will reproduce the problem? + +1. Create a fresh annex repo. Create a dummy test file and add it to the annex. +2. `git init --bare` in an empty directory on a remote machine. +3. `git initremote testremote type=gcrypt encryption=hybrid gitrepo=ssh://machine/the/remote/machine/dir keyid=my_key_id` +4. `git annex sync testremote --content` +5. Unplug network/switch off WiFi. +6. `git annex sync testremote --content`, which fails due to the broken network. +7. Reconnect network, check that can ssh to remote host. +8. `git annex sync testremote --content` + +gcrypt issues warning `gcrypt: WARNING: Remote ID has changed!` + +### What version of git-annex are you using? On what operating system? + +git-annex 5.20141125 on Debian Wheezy 32-bit. + +### Please provide any additional information below. + +This is essentially a gcrypt bug, so I don't know if you want to fix it, and I know that the gcrypt author is inactive. + +My diagnosis is that when running `git annex sync testremote --content` when the network is disconnected, git can't SSH to the remote and gcrypt makes the mistake of regenerating the remote ID and setting up a new remote. So when the network comes back online, the local record of the remote's gcrypt ID is just wrong. gcrypt ought not to "set up a new repository" when there is a network failure. + + gcrypt: Development version -- Repository format MAY CHANGE + gcrypt: Repository not found: ssh://url-here + gcrypt: Setting up new repository + gcrypt: Remote ID is :id:agVyn7wBG/JGwN9LW5Qn + Counting objects: 22, done. + Compressing objects: 100% (17/17), done. + Total 22 (delta 4), reused 0 (delta 0) + gcrypt: Encrypting to: -r my_key_id_here + gcrypt: Requesting manifest signature + ssh: Could not resolve hostname my_remote_host_here: No such file or directory + fatal: Could not read from remote repository. + + Please make sure you have the correct access rights + and the repository exists. + error: failed to push some refs to 'gcrypt::ssh://url-here + + Pushing to testremote failed. + + (non-fast-forward problems can be solved by setting receive.denyNonFastforwards to false in the remote's git config) + failed From 7fa8e6a2849389f602bb80dc80c46b8ad39dccb5 Mon Sep 17 00:00:00 2001 From: "https://launchpad.net/~giomasce" Date: Sat, 23 May 2015 14:35:14 +0000 Subject: [PATCH 2/7] --- ...are_not_present_in_current_repository.mdwn | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 doc/bugs/incremental_fsck-ing_a_remote_does_not_work_for_files_that_are_not_present_in_current_repository.mdwn diff --git a/doc/bugs/incremental_fsck-ing_a_remote_does_not_work_for_files_that_are_not_present_in_current_repository.mdwn b/doc/bugs/incremental_fsck-ing_a_remote_does_not_work_for_files_that_are_not_present_in_current_repository.mdwn new file mode 100644 index 0000000000..ffc3acb6b1 --- /dev/null +++ b/doc/bugs/incremental_fsck-ing_a_remote_does_not_work_for_files_that_are_not_present_in_current_repository.mdwn @@ -0,0 +1,103 @@ +### Please describe the problem. +Incremental fsck keeps information about which time a file was last fsck-ed by setting mtime of the file's parent directory in `.git/annex/objects/`. When we are doing incremental fsck from a remote, files that are not available locally are never marked as checked (since said directory does not exist), so they are checked at every invocation of `git annex fsck --more`. + +### What steps will reproduce the problem? +Create a git-annex repository with some random content. Then add any remote, copy files there, remove them locally and run an incremental fsck from the remote. Interrupt it and run again with `--more`. It will check again all the files, including those that have already been checked. + +### What version of git-annex are you using? On what operating system? +Debian official package, 5.20141125, on Debian sid (more or less up-to-date). + +### Please provide any additional information below. + +[[!format sh """ +# Create a test repository +giovanni@amalgama:~$ cd /tmp/ +giovanni@amalgama:/tmp$ mkdir test +giovanni@amalgama:/tmp$ cd test/ +giovanni@amalgama:/tmp/test$ git init +Inizializzato un repository Git in /tmp/test/.git/ +giovanni@amalgama:/tmp/test (master)$ git annex init +init ok +(Recording state in git...) +# Create random content +giovanni@amalgama:/tmp/test (master)$ dd if=/dev/urandom bs=1M count=20 of=test1 +20+0 record dentro +20+0 record fuori +20971520 byte (21 MB) copiati, 1,15928 s, 18,1 MB/s +giovanni@amalgama:/tmp/test (master)$ dd if=/dev/urandom bs=1M count=20 of=test2 +20+0 record dentro +20+0 record fuori +20971520 byte (21 MB) copiati, 1,12974 s, 18,6 MB/s +giovanni@amalgama:/tmp/test (master)$ dd if=/dev/urandom bs=1M count=20 of=test3 +20+0 record dentro +20+0 record fuori +20971520 byte (21 MB) copiati, 1,16881 s, 17,9 MB/s +giovanni@amalgama:/tmp/test (master)$ dd if=/dev/urandom bs=1M count=20 of=test4 +20+0 record dentro +20+0 record fuori +20971520 byte (21 MB) copiati, 1,14387 s, 18,3 MB/s +giovanni@amalgama:/tmp/test (master)$ git annex add . +add test1 ok +add test2 ok +add test3 ok +add test4 ok +(Recording state in git...) +# Create a remote of type directory and move content there +giovanni@amalgama:/tmp/test (master)$ mkdir /tmp/dir +giovanni@amalgama:/tmp/test (master)$ git annex initremote test type=directory encryption=none directory=/tmp/dir +initremote test ok +(Recording state in git...) +giovanni@amalgama:/tmp/test (master)$ git annex move --to test +move test1 (to test...) +ok +move test2 (to test...) +ok +move test3 (to test...) +ok +move test4 (to test...) +ok +(Recording state in git...) +# Launch a remote incremental fsck +giovanni@amalgama:/tmp/test (master)$ git annex fsck --from test --incremental +fsck test1 (checksum...) +ok +fsck test2 (checksum...) +ok +fsck test3 (checksum...) +ok +fsck test4 (checksum...) +ok +# Continue it; here I would expect nothing to happen, since all content has already been checked +giovanni@amalgama:/tmp/test (master)$ git annex fsck --from test --more +fsck test1 (checksum...) +ok +fsck test2 (checksum...) +ok +fsck test3 (checksum...) +ok +fsck test4 (checksum...) +ok +# Bring back content locally and launch again fsck +giovanni@amalgama:/tmp/test (master)$ git annex get +get test1 (from test...) +ok +get test2 (from test...) +ok +get test3 (from test...) +ok +get test4 (from test...) +ok +(Recording state in git...) +giovanni@amalgama:/tmp/test (master)$ git annex fsck --from test --incremental +fsck test1 (checksum...) +ok +fsck test2 (checksum...) +ok +fsck test3 (checksum...) +ok +fsck test4 (checksum...) +ok +# Now --more semantics is respected +giovanni@amalgama:/tmp/test (master)$ git annex fsck --from test --more +giovanni@amalgama:/tmp/test (master)$ +"""]] From 97cad7691614b717ae837c38bf1c5d02583839ac Mon Sep 17 00:00:00 2001 From: "junk@5e3eeba2290e8a3fcf938d9f93b0dfa2565dc7b1" Date: Sun, 24 May 2015 13:48:33 +0000 Subject: [PATCH 3/7] Added a comment: Tahoe-LAFS helper: multiple FURLs for the same grid --- ..._d30f5958cca5dbcb93e4ea56a25586da._comment | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/special_remotes/tahoe/comment_1_d30f5958cca5dbcb93e4ea56a25586da._comment diff --git a/doc/special_remotes/tahoe/comment_1_d30f5958cca5dbcb93e4ea56a25586da._comment b/doc/special_remotes/tahoe/comment_1_d30f5958cca5dbcb93e4ea56a25586da._comment new file mode 100644 index 0000000000..63bb8f44a7 --- /dev/null +++ b/doc/special_remotes/tahoe/comment_1_d30f5958cca5dbcb93e4ea56a25586da._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="junk@5e3eeba2290e8a3fcf938d9f93b0dfa2565dc7b1" + nickname="junk" + subject="Tahoe-LAFS helper: multiple FURLs for the same grid" + date="2015-05-24T13:48:33Z" + content=""" +Hi, + +I would like to uses git-annex in combination with Tahoe-LAFS. The grid will consist of private Servers connected though slow DSL-Lines. Thus I would like to use the Tahoe-LAFS helper feature (like a Tahoe-LAFS upload proxy): + +https://tahoe-lafs.org/trac/tahoe-lafs/browser/trunk/docs/helper.rst + +This will result in a different FURL for each location pointing to the same Tahoe-LAFS grid. + +How can I setup two git-annex clients to use two different FURLs for the same remote (the same Tahoe-LAFS grid)? + +Thank you very much for your help! + +Oliver +"""]] From 83b63172b6957f94b9c975092ea900420c5a3318 Mon Sep 17 00:00:00 2001 From: eigengrau Date: Sun, 24 May 2015 14:55:31 +0000 Subject: [PATCH 4/7] --- doc/bugs/git_annex_fsck_on_btrfs_devices.mdwn | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/git_annex_fsck_on_btrfs_devices.mdwn diff --git a/doc/bugs/git_annex_fsck_on_btrfs_devices.mdwn b/doc/bugs/git_annex_fsck_on_btrfs_devices.mdwn new file mode 100644 index 0000000000..1ad2f65f94 --- /dev/null +++ b/doc/bugs/git_annex_fsck_on_btrfs_devices.mdwn @@ -0,0 +1,14 @@ +### Please describe the problem. +btrfs automatically validates checksums when data is read. If a checksum fails, instead of giving the corrupted file contents, the read will throw an I/O error. In result, it seems that git-annex fsck will not recognize the file as faulty, and will instead fail with a sha1sum parse error, without dropping the corresponding file as “bad”. + +[[!format sh """ +git annex fsck file +fsck file (checksum...) +sha1sum: .git/annex/objects/…: Input/output error +git-annex: sha1sum parse error +# End of transcript or log. +"""]] + +### What version of git-annex are you using? On what operating system? +git-annex 5.20150508 +linux 4.0.4 From 836d129244bff609cebf3ea51c4abcea3646eb0d Mon Sep 17 00:00:00 2001 From: eigengrau Date: Sun, 24 May 2015 14:58:30 +0000 Subject: [PATCH 5/7] Added a comment --- .../comment_2_ce97390480a04918cf20c12944ee65ef._comment | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/bugs/Stale_keys_not_forgotten_upon_git-annex_forget/comment_2_ce97390480a04918cf20c12944ee65ef._comment diff --git a/doc/bugs/Stale_keys_not_forgotten_upon_git-annex_forget/comment_2_ce97390480a04918cf20c12944ee65ef._comment b/doc/bugs/Stale_keys_not_forgotten_upon_git-annex_forget/comment_2_ce97390480a04918cf20c12944ee65ef._comment new file mode 100644 index 0000000000..d757eefa9a --- /dev/null +++ b/doc/bugs/Stale_keys_not_forgotten_upon_git-annex_forget/comment_2_ce97390480a04918cf20c12944ee65ef._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="eigengrau" + subject="comment 2" + date="2015-05-24T14:58:30Z" + content=""" +Thanks! This would be useful. While it probably takes up only little disk space, the noise it creates during fsck would make it harder to discern which data has been unintentionally lost, as opposed to migrated to another back-end. +"""]] From 15ef7780b40be9a0124b761cc6c94ac5c5b9a2be Mon Sep 17 00:00:00 2001 From: eigengrau Date: Sun, 24 May 2015 15:07:34 +0000 Subject: [PATCH 6/7] Added a comment --- .../comment_1_f33d4cb9689f89f07bb97460e4f30be8._comment | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/bugs/git_annex_fsck_on_btrfs_devices/comment_1_f33d4cb9689f89f07bb97460e4f30be8._comment diff --git a/doc/bugs/git_annex_fsck_on_btrfs_devices/comment_1_f33d4cb9689f89f07bb97460e4f30be8._comment b/doc/bugs/git_annex_fsck_on_btrfs_devices/comment_1_f33d4cb9689f89f07bb97460e4f30be8._comment new file mode 100644 index 0000000000..d186370d3b --- /dev/null +++ b/doc/bugs/git_annex_fsck_on_btrfs_devices/comment_1_f33d4cb9689f89f07bb97460e4f30be8._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="eigengrau" + subject="comment 1" + date="2015-05-24T15:07:34Z" + content=""" +I suppose since an I/O error can be intermittent, the file can’t be outright regarded as bad. Also I’m not sure whether the read system call returns a dedicated error code for checksum errors. +"""]] From adac761093b549180ca6c0e8b032502a2062c6ed Mon Sep 17 00:00:00 2001 From: clacke Date: Sun, 24 May 2015 17:41:18 +0000 Subject: [PATCH 7/7] Added a comment --- .../comment_4_77e59a25c859b6afec8b75f74885ef5e._comment | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/bugs/rsync:_protocol_version_mismatch/comment_4_77e59a25c859b6afec8b75f74885ef5e._comment diff --git a/doc/bugs/rsync:_protocol_version_mismatch/comment_4_77e59a25c859b6afec8b75f74885ef5e._comment b/doc/bugs/rsync:_protocol_version_mismatch/comment_4_77e59a25c859b6afec8b75f74885ef5e._comment new file mode 100644 index 0000000000..cad761d149 --- /dev/null +++ b/doc/bugs/rsync:_protocol_version_mismatch/comment_4_77e59a25c859b6afec8b75f74885ef5e._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="clacke" + subject="comment 4" + date="2015-05-24T17:41:17Z" + content=""" +Hm, not so sure that \"rebooted, did not help\" was actually true. I take that back. + +Now I saw a stray `git-annex-shell recv-key` process mentioning that file. I killed it and now everything seems fine. I will keep this in mind for next time, to see if I can verify that this was actually the cause of the message, but maybe it's a clue. +"""]]