From 957dc80c3f4babfeb1784953e1bdc49d03dbb65e Mon Sep 17 00:00:00 2001 From: Ilya_Shlyakhter Date: Thu, 27 Feb 2020 20:33:44 +0000 Subject: [PATCH 1/6] Added a comment: batch presence checking --- .../comment_1_181a908a9f77a4c51d1d09f1aff526b7._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/todo/CHECKPRESENT-MULTI/comment_1_181a908a9f77a4c51d1d09f1aff526b7._comment diff --git a/doc/todo/CHECKPRESENT-MULTI/comment_1_181a908a9f77a4c51d1d09f1aff526b7._comment b/doc/todo/CHECKPRESENT-MULTI/comment_1_181a908a9f77a4c51d1d09f1aff526b7._comment new file mode 100644 index 0000000000..d81b7be40e --- /dev/null +++ b/doc/todo/CHECKPRESENT-MULTI/comment_1_181a908a9f77a4c51d1d09f1aff526b7._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="batch presence checking" + date="2020-02-27T20:33:44Z" + content=""" +Beyond individual large files, maybe this could speed up checking that a whole directory is present in an export remote, by using a command like [`gsutil ls`](https://cloud.google.com/storage/docs/gsutil/commands/ls) to list the present files, instead of calling [`gsutil stat`](https://cloud.google.com/storage/docs/gsutil/commands/stat) on each file individually. +"""]] From e156a2b74242b6cf38d64ab9499bda049654ea89 Mon Sep 17 00:00:00 2001 From: Ilya_Shlyakhter Date: Thu, 27 Feb 2020 21:39:57 +0000 Subject: [PATCH 2/6] Added a comment: another example of git-annex-fsck reporting dead keys as errors --- ..._cee96c6f5d5d97f6816b9c3524822327._comment | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 doc/bugs/git-annex-fsck_reports_dead_keys_as_errors/comment_1_cee96c6f5d5d97f6816b9c3524822327._comment diff --git a/doc/bugs/git-annex-fsck_reports_dead_keys_as_errors/comment_1_cee96c6f5d5d97f6816b9c3524822327._comment b/doc/bugs/git-annex-fsck_reports_dead_keys_as_errors/comment_1_cee96c6f5d5d97f6816b9c3524822327._comment new file mode 100644 index 0000000000..ac9c790184 --- /dev/null +++ b/doc/bugs/git-annex-fsck_reports_dead_keys_as_errors/comment_1_cee96c6f5d5d97f6816b9c3524822327._comment @@ -0,0 +1,43 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="another example of git-annex-fsck reporting dead keys as errors" + date="2020-02-27T21:39:57Z" + content=""" +[[!format sh \"\"\" +(master_env_v174_py36) 16:36 [viral-ngs-benchmarks] $ git annex fsck wdl/quay.io-broadinstitute-viral-ngs-dev_1.25.0-11-g6fe46aa-is-f\ +ix-gatk-mindepth_sha256_20bbb1d96cf58235d325a4ea2def8c3d840d9c2ab5dddf573f9471c57f7414f1/assemble_denovo/tasks_assembly.wdl +fsck wdl/quay.io-broadinstitute-viral-ngs-dev_1.25.0-11-g6fe46aa-is-fix-gatk-mindepth_sha256_20bbb1d96cf58235d325a4ea2def8c3d840d9c2ab\ +5dddf573f9471c57f7414f1/assemble_denovo/tasks_assembly.wdl + ** Required content wdl/quay.io-broadinstitute-viral-ngs-dev_1.25.0-11-g6fe46aa-is-fix-gatk-mindepth_sha256_20bbb1d96cf58235d325a4ea\ +2def8c3d840d9c2ab5dddf573f9471c57f7414f1/assemble_denovo/tasks_assembly.wdl is missing from these repositories: + 449efa47-a0e1-4376-a17f-42c7a1f509d1 -- Benchmarks for viral-ngs, stored on Amazon S3 [viral-ngs-benchmarks-s3] + + ** No known copies exist of wdl/quay.io-broadinstitute-viral-ngs-dev_1.25.0-11-g6fe46aa-is-fix-gatk-mindepth_sha256_20bbb1d96cf58235\ +d325a4ea2def8c3d840d9c2ab5dddf573f9471c57f7414f1/assemble_denovo/tasks_assembly.wdl + These dead repositories used to have copies + 461e3f01-1ff4-43d6-8dfa-71b839a25881 -- crogit runner crogrun_200207_174047__8035____git_9cb32e7230b +failed +(recording state in git...) +git-annex: fsck: 1 failed +(master_env_v174_py36) 16:36 [viral-ngs-benchmarks] $ git annex dead --key `git annex lookupkey wdl/quay.io-broadinstitute-viral-ngs-\ +dev_1.25.0-11-g6fe46aa-is-fix-gatk-mindepth_sha256_20bbb1d96cf58235d325a4ea2def8c3d840d9c2ab5dddf573f9471c57f7414f1/assemble_denovo/ta\ +sks_assembly.wdl` +dead MD5E-s15002--4f40b2c4a2ba81128db649e6096b5fec.wdl ok +(master_env_v174_py36) 16:37 [viral-ngs-benchmarks] $ git annex fsck wdl/quay.io-broadinstitute-viral-ngs-dev_1.25.0-11-g6fe46aa-is-f\ +ix-gatk-mindepth_sha256_20bbb1d96cf58235d325a4ea2def8c3d840d9c2ab5dddf573f9471c57f7414f1/assemble_denovo/tasks_assembly.wdl +fsck wdl/quay.io-broadinstitute-viral-ngs-dev_1.25.0-11-g6fe46aa-is-fix-gatk-mindepth_sha256_20bbb1d96cf58235d325a4ea2def8c3d840d9c2ab\ +5dddf573f9471c57f7414f1/assemble_denovo/tasks_assembly.wdl + ** Required content wdl/quay.io-broadinstitute-viral-ngs-dev_1.25.0-11-g6fe46aa-is-fix-gatk-mindepth_sha256_20bbb1d96cf58235d325a4ea\ +2def8c3d840d9c2ab5dddf573f9471c57f7414f1/assemble_denovo/tasks_assembly.wdl is missing from these repositories: + 449efa47-a0e1-4376-a17f-42c7a1f509d1 -- Benchmarks for viral-ngs, stored on Amazon S3 [viral-ngs-benchmarks-s3] + + ** No known copies exist of wdl/quay.io-broadinstitute-viral-ngs-dev_1.25.0-11-g6fe46aa-is-fix-gatk-mindepth_sha256_20bbb1d96cf58235\ +d325a4ea2def8c3d840d9c2ab5dddf573f9471c57f7414f1/assemble_denovo/tasks_assembly.wdl + These dead repositories used to have copies + 461e3f01-1ff4-43d6-8dfa-71b839a25881 -- crogit runner crogrun_200207_174047__8035____git_9cb32e7230b +failed +(recording state in git...) +git-annex: fsck: 1 failed +\"\"\"]] +"""]] From f39aa14f6231cec43153091c2a94c16347023e43 Mon Sep 17 00:00:00 2001 From: codecircuit Date: Fri, 28 Feb 2020 11:16:24 +0000 Subject: [PATCH 3/6] --- ...iles_encrypted_on_special_remote_with_gcrypt.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/Store_annexed_files_encrypted_on_special_remote_with_gcrypt.mdwn diff --git a/doc/forum/Store_annexed_files_encrypted_on_special_remote_with_gcrypt.mdwn b/doc/forum/Store_annexed_files_encrypted_on_special_remote_with_gcrypt.mdwn new file mode 100644 index 0000000000..4812c6f0b4 --- /dev/null +++ b/doc/forum/Store_annexed_files_encrypted_on_special_remote_with_gcrypt.mdwn @@ -0,0 +1,12 @@ +Hello, + +my final goal is to store files on an untrusted external server/USB drive in an encrypted git repository. + +- First I set up the external git repository with `git init --bare /path/to/external` +- Second I add the special remote with `git annex initremote encrypted-origin type=gcrypt gitrepo=/path/to/external encryption=hybrid keyid=$keyid` +- Then I sync with `git annex sync` +- Then I copy the files to the remote with `git annex copy -t encrypted-origin` + +I can now see the annexed files in the remote repository **unencrypted**. I do not want the copied files to be readable on the remote. + +Is that the intended behaviour, or did I do something wrong? From f56dfe791cbab99fb8f944b20034ec060ff30d8d Mon Sep 17 00:00:00 2001 From: codecircuit Date: Fri, 28 Feb 2020 11:29:27 +0000 Subject: [PATCH 4/6] Added a comment --- .../comment_1_5a643827a1ced7c0b9bba5303f71a52c._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Store_annexed_files_encrypted_on_special_remote_with_gcrypt/comment_1_5a643827a1ced7c0b9bba5303f71a52c._comment diff --git a/doc/forum/Store_annexed_files_encrypted_on_special_remote_with_gcrypt/comment_1_5a643827a1ced7c0b9bba5303f71a52c._comment b/doc/forum/Store_annexed_files_encrypted_on_special_remote_with_gcrypt/comment_1_5a643827a1ced7c0b9bba5303f71a52c._comment new file mode 100644 index 0000000000..c6d9c686cb --- /dev/null +++ b/doc/forum/Store_annexed_files_encrypted_on_special_remote_with_gcrypt/comment_1_5a643827a1ced7c0b9bba5303f71a52c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="codecircuit" + avatar="http://cdn.libravatar.org/avatar/8a2041fb0484e333aa769009d49c4061" + subject="comment 1" + date="2020-02-28T11:29:27Z" + content=""" +Ok, I assume it is better to just take the `rsync` or `directory` special remote +"""]] From 55e95643b2152fca7d0357e2644c5de22b809639 Mon Sep 17 00:00:00 2001 From: "basile.pinsard@f1a7fae9f3bd9d5282fca11f62ad53b45a8eb317" Date: Fri, 28 Feb 2020 15:44:50 +0000 Subject: [PATCH 5/6] Added a comment --- ..._41720da2eebb09b23cc739a45ca5d653._comment | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 doc/bugs/S3-remote_fail_to_get_data_on_some_clones__47__machines/comment_10_41720da2eebb09b23cc739a45ca5d653._comment diff --git a/doc/bugs/S3-remote_fail_to_get_data_on_some_clones__47__machines/comment_10_41720da2eebb09b23cc739a45ca5d653._comment b/doc/bugs/S3-remote_fail_to_get_data_on_some_clones__47__machines/comment_10_41720da2eebb09b23cc739a45ca5d653._comment new file mode 100644 index 0000000000..001986ae4d --- /dev/null +++ b/doc/bugs/S3-remote_fail_to_get_data_on_some_clones__47__machines/comment_10_41720da2eebb09b23cc739a45ca5d653._comment @@ -0,0 +1,53 @@ +[[!comment format=mdwn + username="basile.pinsard@f1a7fae9f3bd9d5282fca11f62ad53b45a8eb317" + nickname="basile.pinsard" + avatar="http://cdn.libravatar.org/avatar/87e1f73acf277ad0337b90fc0253c62e" + subject="comment 10" + date="2020-02-28T15:44:50Z" + content=""" +@joey + +I am now trying from scratch, created a new repository and trying to create the remote from the server to the S3 on the same network. + +``` +git-annex config --set annex.security.allowed-ip-addresses 10.10.10.20 +git annex initremote -d s3.test type=S3 encryption=none autoenable=true host=s3.unf-montreal.ca port=443 protocol=https chunk=1GiB bucket=test requeststyle=path +``` + +I get the following error despite relaxing the security option. (same thing with 'all' instead of 10.10.10.20). + +Maybe this security check is broken, and this is why I have problem just getting data from the local network? + + +``` + (InternalException (ConnectionRestricted \"Configuration of annex.security.allowed-ip-addresses does not allow accessing address 10.10.10.20\")) +``` + +Also, when I add the option exporttree=no + +``` +git-annex: Unexpected parameters: exporttree +failed +``` + +while the option exists in the doc, and I used it in the past. + +Version + +``` +$ git-annex version +git-annex version: 7.20200219-g5094a3f +build flags: Assistant Webapp Pairing S3 WebDAV Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite +dependency versions: aws-0.21.1 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.3 feed-1.0.1.0 ghc-8.6.5 http-client-0.5.14 persistent-sqlite-2.9.3 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs hook external +operating system: linux x86_64 +supported repository versions: 7 +upgrade supported from repository versions: 0 1 2 3 4 5 6 +local repository version: 7 +``` + + +Thank you for your help. + +"""]] From e0507812f4db6ac314c5368015d8966670f68cdb Mon Sep 17 00:00:00 2001 From: "lykos@d125a37d89b1cfac20829f12911656c40cb70018" Date: Fri, 28 Feb 2020 16:17:08 +0000 Subject: [PATCH 6/6] Added a comment --- .../comment_2_92e1aef2ce26210b15a908e2cf729b80._comment | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/forum/Store_annexed_files_encrypted_on_special_remote_with_gcrypt/comment_2_92e1aef2ce26210b15a908e2cf729b80._comment diff --git a/doc/forum/Store_annexed_files_encrypted_on_special_remote_with_gcrypt/comment_2_92e1aef2ce26210b15a908e2cf729b80._comment b/doc/forum/Store_annexed_files_encrypted_on_special_remote_with_gcrypt/comment_2_92e1aef2ce26210b15a908e2cf729b80._comment new file mode 100644 index 0000000000..44b211a270 --- /dev/null +++ b/doc/forum/Store_annexed_files_encrypted_on_special_remote_with_gcrypt/comment_2_92e1aef2ce26210b15a908e2cf729b80._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="lykos@d125a37d89b1cfac20829f12911656c40cb70018" + nickname="lykos" + avatar="http://cdn.libravatar.org/avatar/085df7b04d3408ba23c19f9c49be9ea2" + subject="comment 2" + date="2020-02-28T16:17:08Z" + content=""" +Which version do you use? There was a [bug](https://git-annex.branchable.com/bugs/gcrypt_special_remote_not_working/) in an earlier version of this month which was [fixed](https://git-annex.branchable.com/devblog/day_619__important_bugfix_release_and_v8_too/) with [7.20200226](https://git-annex.branchable.com/news/version_7.20200226/). +"""]]