From 82538a9cd351b41ffb5a334610b73f88c8c6df9b Mon Sep 17 00:00:00 2001 From: "mike@2d6d71f56ce2a992244350475251df87c26fe351" Date: Fri, 27 Sep 2024 07:39:06 +0000 Subject: [PATCH 1/3] Added a comment: corruption using git-annex-remote-rclone --- ..._414559099884a972943586cf03d613a4._comment | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 doc/special_remotes/rclone/comment_5_414559099884a972943586cf03d613a4._comment diff --git a/doc/special_remotes/rclone/comment_5_414559099884a972943586cf03d613a4._comment b/doc/special_remotes/rclone/comment_5_414559099884a972943586cf03d613a4._comment new file mode 100644 index 0000000000..5e7837a6cd --- /dev/null +++ b/doc/special_remotes/rclone/comment_5_414559099884a972943586cf03d613a4._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="mike@2d6d71f56ce2a992244350475251df87c26fe351" + nickname="mike" + avatar="http://cdn.libravatar.org/avatar/183fa439752e2f0c6f39ede658d81050" + subject="corruption using git-annex-remote-rclone" + date="2024-09-27T07:39:06Z" + content=""" +Perhaps Joey can help me out here a bit with some background knowledge: + +I've been seeing sporadic corruption with this setup: +- chunking +- encryption +- old helper program git-annex-remote-rclone +- rclone's pcloud backend + +At it seems, rclone keeps partial files under the name of the full file when a transfer is interrupted, for the pcloud backend. (This is for rclone <= 1.67.0; 1.68.0 has changes for pcloud, which may fix this.) My theory how the corruption might have happened: + +- First interrupted run of git-annex uploads chunks A and a partial(!) chunk B +- Second run skips chunks A and B(!) and, say, the rest of the chunks (C and D) +- At the end we have uploaded A, C and D and a corrupted/partial chunk B + +Joey: Is this a possible error scenario? + +"""]] From 39e02528f0488fd1eae23538317a0c567d084ad9 Mon Sep 17 00:00:00 2001 From: "mike@2d6d71f56ce2a992244350475251df87c26fe351" Date: Fri, 27 Sep 2024 12:18:41 +0000 Subject: [PATCH 2/3] Added a comment: corruption using git-annex-remote-rclone --- ..._f38408b0e8f62a531f5daeb24fbb36a3._comment | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 doc/special_remotes/rclone/comment_6_f38408b0e8f62a531f5daeb24fbb36a3._comment diff --git a/doc/special_remotes/rclone/comment_6_f38408b0e8f62a531f5daeb24fbb36a3._comment b/doc/special_remotes/rclone/comment_6_f38408b0e8f62a531f5daeb24fbb36a3._comment new file mode 100644 index 0000000000..9c2da62f7b --- /dev/null +++ b/doc/special_remotes/rclone/comment_6_f38408b0e8f62a531f5daeb24fbb36a3._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="mike@2d6d71f56ce2a992244350475251df87c26fe351" + nickname="mike" + avatar="http://cdn.libravatar.org/avatar/183fa439752e2f0c6f39ede658d81050" + subject="corruption using git-annex-remote-rclone" + date="2024-09-27T12:18:41Z" + content=""" +Perhaps Joey can help me out here a bit with some background knowledge: + +I've been seeing sporadic corruption with this setup: + +* chunking +* encryption +* old helper program git-annex-remote-rclone +* rclone's pcloud backend + +As it seems, rclone keeps partial files under the name of the full file when a transfer is interrupted, for the pcloud backend. (This is for rclone <= 1.67.0; 1.68.0 has changes for pcloud, which may fix this.) My theory how the corruption might have happened: + +* First interrupted run of git-annex uploads chunks A and a partial(!) chunk B +* Second run skips chunks A and B(!); and proceedsto upload the rest of the chunks (C and D) +* At the end we have uploaded A, C and D and a corrupted/partial chunk B + +Joey: Is this a possible error scenario? + +"""]] From 7b5dda33e07962dc3717560232c367a9febbd47c Mon Sep 17 00:00:00 2001 From: "mike@2d6d71f56ce2a992244350475251df87c26fe351" Date: Fri, 27 Sep 2024 12:18:59 +0000 Subject: [PATCH 3/3] removed --- ..._414559099884a972943586cf03d613a4._comment | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 doc/special_remotes/rclone/comment_5_414559099884a972943586cf03d613a4._comment diff --git a/doc/special_remotes/rclone/comment_5_414559099884a972943586cf03d613a4._comment b/doc/special_remotes/rclone/comment_5_414559099884a972943586cf03d613a4._comment deleted file mode 100644 index 5e7837a6cd..0000000000 --- a/doc/special_remotes/rclone/comment_5_414559099884a972943586cf03d613a4._comment +++ /dev/null @@ -1,24 +0,0 @@ -[[!comment format=mdwn - username="mike@2d6d71f56ce2a992244350475251df87c26fe351" - nickname="mike" - avatar="http://cdn.libravatar.org/avatar/183fa439752e2f0c6f39ede658d81050" - subject="corruption using git-annex-remote-rclone" - date="2024-09-27T07:39:06Z" - content=""" -Perhaps Joey can help me out here a bit with some background knowledge: - -I've been seeing sporadic corruption with this setup: -- chunking -- encryption -- old helper program git-annex-remote-rclone -- rclone's pcloud backend - -At it seems, rclone keeps partial files under the name of the full file when a transfer is interrupted, for the pcloud backend. (This is for rclone <= 1.67.0; 1.68.0 has changes for pcloud, which may fix this.) My theory how the corruption might have happened: - -- First interrupted run of git-annex uploads chunks A and a partial(!) chunk B -- Second run skips chunks A and B(!) and, say, the rest of the chunks (C and D) -- At the end we have uploaded A, C and D and a corrupted/partial chunk B - -Joey: Is this a possible error scenario? - -"""]]