diff --git a/doc/bugs/git_annex_get_is_silently_stuck_on__P2P___62___GET_0.mdwn b/doc/bugs/git_annex_get_is_silently_stuck_on__P2P___62___GET_0.mdwn new file mode 100644 index 0000000000..811281ed9a --- /dev/null +++ b/doc/bugs/git_annex_get_is_silently_stuck_on__P2P___62___GET_0.mdwn @@ -0,0 +1,83 @@ +### Please describe the problem. + +Trying to get fresh data from a server but `annex get` is just stuck without any output. Here is how it looks with `--debug`: + +``` +yoh@typhon:/mnt/DATA/data/yoh/1076_spacetop$ git annex get --debug sub-0002/ses-01/dwi/sub-0002_ses-01_acq-96dirX6b0Xmb_dwi.nii.gz +[2024-05-29 14:08:30.203689795] (Utility.Process) process [3922981] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","ls-files","--stage","-z","--error-unmatch","--","sub-0002/ses-01/dwi/sub-0002_ses-01_acq-96dirX6b0Xmb_dwi.nii.gz"] +[2024-05-29 14:08:30.204159722] (Utility.Process) process [3922982] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)","--buffer"] +[2024-05-29 14:08:30.204574378] (Utility.Process) process [3922983] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","cat-file","--batch=%(objectname) %(objecttype) %(objectsize)","--buffer"] +[2024-05-29 14:08:30.205075804] (Utility.Process) process [3922984] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","git-annex"] +[2024-05-29 14:08:30.207328301] (Utility.Process) process [3922984] done ExitSuccess +[2024-05-29 14:08:30.207536385] (Utility.Process) process [3922985] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","--hash","refs/heads/git-annex"] +[2024-05-29 14:08:30.209355606] (Utility.Process) process [3922985] done ExitSuccess +[2024-05-29 14:08:30.209800913] (Utility.Process) process [3922986] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","cat-file","--batch=%(objectname) %(objecttype) %(objectsize)","--buffer"] +get sub-0002/ses-01/dwi/sub-0002_ses-01_acq-96dirX6b0Xmb_dwi.nii.gz [2024-05-29 14:08:30.215082484] (Utility.Process) process [3922987] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","cat-file","--batch"] +(from origin...) +[2024-05-29 14:08:30.225575902] (Utility.Process) process [3922988] read: ssh ["-O","stop","-S","bids@rolando.cns.dartmouth.edu","-o","ControlMaster=auto","-o","ControlPersist=yes","localhost"] in ".git/annex/ssh/" +[2024-05-29 14:08:30.230512383] (Utility.Process) process [3922988] done ExitSuccess +[2024-05-29 14:08:30.231078838] (Utility.Process) process [3922989] read: ssh ["-o","BatchMode=true","-S",".git/annex/ssh/bids@rolando.cns.dartmouth.edu","-o","ControlMaster=auto","-o","ControlPersist=yes","-n","-T","bids@rolando.cns.dartmouth.edu","true"] +[2024-05-29 14:08:31.042361711] (Utility.Process) process [3922989] done ExitSuccess +[2024-05-29 14:08:31.043219566] (Utility.Process) process [3923085] chat: ssh ["bids@rolando.cns.dartmouth.edu","-S",".git/annex/ssh/bids@rolando.cns.dartmouth.edu","-o","ControlMaster=auto","-o","ControlPersist=yes","-T","git-annex-shell 'p2pstdio' '/inbox/BIDS/Wager/Wager/1076_spacetop' '--debug' '97b6f5e4-4642-43a7-988a-c483caf553c5' --uuid 590b4fd0-0142-4e9d-8964-d1158c242c6a"] +[2024-05-29 14:08:31.540468659] (P2P.IO) [ThreadId 4] P2P > AUTH-SUCCESS 590b4fd0-0142-4e9d-8964-d1158c242c6a +[2024-05-29 14:08:31.539643152] (P2P.IO) [ssh connection Just 3923085] [ThreadId 4] P2P < AUTH-SUCCESS 590b4fd0-0142-4e9d-8964-d1158c242c6a +[2024-05-29 14:08:31.539846045] (P2P.IO) [ssh connection Just 3923085] [ThreadId 4] P2P > VERSION 1 +[2024-05-29 14:08:31.542774097] (P2P.IO) [ThreadId 4] P2P < VERSION 1 +[2024-05-29 14:08:31.543007073] (P2P.IO) [ThreadId 4] P2P > VERSION 1 +[2024-05-29 14:08:31.541288127] (P2P.IO) [ssh connection Just 3923085] [ThreadId 4] P2P < VERSION 1 +[2024-05-29 14:08:31.541362199] (P2P.IO) [ssh connection Just 3923085] [ThreadId 4] P2P > GET 0 sub-0002/ses-01/dwi/sub-0002_ses-01_acq-96dirX6b0Xmb_dwi.nii.gz MD5E-s239384952--c3aaaebbed3ef5932b4390ddb47d2150.nii.gz +[2024-05-29 14:08:31.544298446] (P2P.IO) [ThreadId 4] P2P < GET 0 sub-0002/ses-01/dwi/sub-0002_ses-01_acq-96dirX6b0Xmb_dwi.nii.gz MD5E-s239384952--c3aaaebbed3ef5932b4390ddb47d2150.nii.gz + +``` + +and on the server it looks like ( I killed one, client reinitiated, this is current one) + +``` +bids 27577 0.3 0.0 1074226428 11120 ? Ssl 14:10 0:00 /data/home/bids/git-annexes/10.20231129+git83-g86dbe9a825/usr/lib/git-annex.linux/exe/git-annex-shell --library-path /data/home/bids/git-annexes/10.20231129+git83-g86dbe9a825/usr/lib/git-annex.linux//lib/x86_64-linux-gnu: /data/home/bids/git-annexes/10.20231129+git83-g86dbe9a825/usr/lib/git-annex.linux/shimmed/git-annex-shell/git-annex-shell p2pstdio /inbox/BIDS/Wager/Wager/1076_spacetop.git --debug 97b6f5e4-4642-43a7-988a-c483caf553c5 --uuid 40795e62-527c-4d26-ae8c-af42a6e2da5a +``` + +and on server config is + +``` +[annex] + sshcaching = false + autoupgraderepository = false + pidlock = true + +[safe] + directory = /inbox/BIDS + +``` + +and that directory `/inbox/BIDS` is NFS mounted. + +Not sure what nobs to turn to help diagnose the issue. + +The git-annex on client side is `10.20240129-1~ndall+1` + + +[[!meta author=yoh]] +[[!tag projects/repronim]] + + + + +### What steps will reproduce the problem? + + +### What version of git-annex are you using? On what operating system? + + +### Please provide any additional information below. + +[[!format sh """ +# If you can, paste a complete transcript of the problem occurring here. +# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log + + +# End of transcript or log. +"""]] + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + diff --git a/doc/bugs/git_annex_get_is_silently_stuck_on__P2P___62___GET_0/comment_1_360cee1c036eca92a25fe92bedd2e2af._comment b/doc/bugs/git_annex_get_is_silently_stuck_on__P2P___62___GET_0/comment_1_360cee1c036eca92a25fe92bedd2e2af._comment new file mode 100644 index 0000000000..b4fa142437 --- /dev/null +++ b/doc/bugs/git_annex_get_is_silently_stuck_on__P2P___62___GET_0/comment_1_360cee1c036eca92a25fe92bedd2e2af._comment @@ -0,0 +1,43 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="odd odd odd" + date="2024-05-29T18:25:11Z" + content=""" +and I cannot copy from the server to the client: + +``` +[bids@rolando 1076_spacetop.git] > git annex copy --debug --to typhon --fast --not --in typhon +yoh@typhon.dartmouth.edu's password: +[2024-05-29 14:21:40.601369476] (Utility.Process) process [28828] read: git [\"--git-dir=.\",\"--literal-pathspecs\",\"-c\",\"annex.debug=true\",\"ls-tree\",\"--full-tree\",\"-z\",\"-r\",\"--\",\"refs/heads/git-annex\"] +[2024-05-29 14:21:40.602439011] (Utility.Process) process [28829] chat: git [\"--git-dir=.\",\"--literal-pathspecs\",\"-c\",\"annex.debug=true\",\"cat-file\",\"--batch=%(objectname) %(objecttype) %(objectsize)\",\"--buffer\"] +[2024-05-29 14:21:40.775458494] (Messages.explain) [ MD5E-s292602526--2069b86f10e1a39bda5a6ed8996078d4.nii.gz does not match:not in=typhon[TRUE] ] + +[2024-05-29 14:21:40.793913739] (Messages.explain) [ MD5E-s2615170--589b64f4cd3067f468408d809c7f1037.nii.gz does not match:not in=typhon[TRUE] ] + +[2024-05-29 14:21:40.811856562] (Messages.explain) [ MD5E-s597779--1108c4d91a2386573c50c251a0e967c2.tgz matches:not in=typhon[FALSE] ] + +copy MD5E-s597779--1108c4d91a2386573c50c251a0e967c2.tgz (to typhon...) [2024-05-29 14:21:40.974993044] (Utility.Process) process [28817] done ExitSuccess +[2024-05-29 14:21:40.975527841] (Utility.Process) process [28816] done ExitSuccess + + + You have enabled concurrency, but git-annex is not able to use ssh connection caching. This may result in multiple ssh processes prompting for passwords at the same time. + + annex.sshcaching is not set to true +[2024-05-29 14:21:40.977267402] (Utility.Process) process [28831] chat: ssh [\"yoh@typhon.dartmouth.edu\",\"-T\",\"git-annex-shell 'p2pstdio' '/mnt/DATA/data/yoh/1076_spacetop' '--debug' '40795e62-527c-4d26-ae8c-af42a6e2da5a' --uuid 97b6f5e4-4642-43a7-988a-c483caf553c5\"] +yoh@typhon.dartmouth.edu's password: +[2024-05-29 14:21:57.87732028] (P2P.IO) [ThreadId 4] P2P > AUTH-SUCCESS 97b6f5e4-4642-43a7-988a-c483caf553c5 +[2024-05-29 14:21:57.879974047] (P2P.IO) [ssh connection Just 28831] [ThreadId 4] P2P < AUTH-SUCCESS 97b6f5e4-4642-43a7-988a-c483caf553c5 +[2024-05-29 14:21:57.880228709] (P2P.IO) [ssh connection Just 28831] [ThreadId 4] P2P > VERSION 1 +[2024-05-29 14:21:57.878927215] (P2P.IO) [ThreadId 4] P2P < VERSION 1 +[2024-05-29 14:21:57.878991142] (P2P.IO) [ThreadId 4] P2P > VERSION 1 +[2024-05-29 14:21:57.881305277] (P2P.IO) [ssh connection Just 28831] [ThreadId 4] P2P < VERSION 1 +[2024-05-29 14:21:57.881487685] (P2P.IO) [ssh connection Just 28831] [ThreadId 4] P2P > PUT MD5E-s597779--1108c4d91a2386573c50c251a0e967c2.tgz +[2024-05-29 14:21:57.880214299] (P2P.IO) [ThreadId 4] P2P < PUT MD5E-s597779--1108c4d91a2386573c50c251a0e967c2.tgz +[2024-05-29 14:21:57.880467298] (P2P.IO) [ThreadId 4] P2P > PUT-FROM 0 +[2024-05-29 14:21:57.882841829] (P2P.IO) [ssh connection Just 28831] [ThreadId 4] P2P < PUT-FROM 0 + +``` + +NB we had issue with a recent 10GB switch update which was worked around by me reducing MTU on my servers from Jumbo frames to regular 1500 . But it manifested that scp did not work. Here scp worked (I did scp on typhon from rolando). And for heejung it does not work to `pull` data from roland to discovery. +"""]] diff --git a/doc/bugs/git_annex_get_is_silently_stuck_on__P2P___62___GET_0/comment_2_cd9e001bc07ab51e6ff1f9abbbb043ac._comment b/doc/bugs/git_annex_get_is_silently_stuck_on__P2P___62___GET_0/comment_2_cd9e001bc07ab51e6ff1f9abbbb043ac._comment new file mode 100644 index 0000000000..76d50e988a --- /dev/null +++ b/doc/bugs/git_annex_get_is_silently_stuck_on__P2P___62___GET_0/comment_2_cd9e001bc07ab51e6ff1f9abbbb043ac._comment @@ -0,0 +1,43 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="odd odd odd" + date="2024-05-29T18:25:23Z" + content=""" +and I cannot copy from the server to the client: + +``` +[bids@rolando 1076_spacetop.git] > git annex copy --debug --to typhon --fast --not --in typhon +yoh@typhon.dartmouth.edu's password: +[2024-05-29 14:21:40.601369476] (Utility.Process) process [28828] read: git [\"--git-dir=.\",\"--literal-pathspecs\",\"-c\",\"annex.debug=true\",\"ls-tree\",\"--full-tree\",\"-z\",\"-r\",\"--\",\"refs/heads/git-annex\"] +[2024-05-29 14:21:40.602439011] (Utility.Process) process [28829] chat: git [\"--git-dir=.\",\"--literal-pathspecs\",\"-c\",\"annex.debug=true\",\"cat-file\",\"--batch=%(objectname) %(objecttype) %(objectsize)\",\"--buffer\"] +[2024-05-29 14:21:40.775458494] (Messages.explain) [ MD5E-s292602526--2069b86f10e1a39bda5a6ed8996078d4.nii.gz does not match:not in=typhon[TRUE] ] + +[2024-05-29 14:21:40.793913739] (Messages.explain) [ MD5E-s2615170--589b64f4cd3067f468408d809c7f1037.nii.gz does not match:not in=typhon[TRUE] ] + +[2024-05-29 14:21:40.811856562] (Messages.explain) [ MD5E-s597779--1108c4d91a2386573c50c251a0e967c2.tgz matches:not in=typhon[FALSE] ] + +copy MD5E-s597779--1108c4d91a2386573c50c251a0e967c2.tgz (to typhon...) [2024-05-29 14:21:40.974993044] (Utility.Process) process [28817] done ExitSuccess +[2024-05-29 14:21:40.975527841] (Utility.Process) process [28816] done ExitSuccess + + + You have enabled concurrency, but git-annex is not able to use ssh connection caching. This may result in multiple ssh processes prompting for passwords at the same time. + + annex.sshcaching is not set to true +[2024-05-29 14:21:40.977267402] (Utility.Process) process [28831] chat: ssh [\"yoh@typhon.dartmouth.edu\",\"-T\",\"git-annex-shell 'p2pstdio' '/mnt/DATA/data/yoh/1076_spacetop' '--debug' '40795e62-527c-4d26-ae8c-af42a6e2da5a' --uuid 97b6f5e4-4642-43a7-988a-c483caf553c5\"] +yoh@typhon.dartmouth.edu's password: +[2024-05-29 14:21:57.87732028] (P2P.IO) [ThreadId 4] P2P > AUTH-SUCCESS 97b6f5e4-4642-43a7-988a-c483caf553c5 +[2024-05-29 14:21:57.879974047] (P2P.IO) [ssh connection Just 28831] [ThreadId 4] P2P < AUTH-SUCCESS 97b6f5e4-4642-43a7-988a-c483caf553c5 +[2024-05-29 14:21:57.880228709] (P2P.IO) [ssh connection Just 28831] [ThreadId 4] P2P > VERSION 1 +[2024-05-29 14:21:57.878927215] (P2P.IO) [ThreadId 4] P2P < VERSION 1 +[2024-05-29 14:21:57.878991142] (P2P.IO) [ThreadId 4] P2P > VERSION 1 +[2024-05-29 14:21:57.881305277] (P2P.IO) [ssh connection Just 28831] [ThreadId 4] P2P < VERSION 1 +[2024-05-29 14:21:57.881487685] (P2P.IO) [ssh connection Just 28831] [ThreadId 4] P2P > PUT MD5E-s597779--1108c4d91a2386573c50c251a0e967c2.tgz +[2024-05-29 14:21:57.880214299] (P2P.IO) [ThreadId 4] P2P < PUT MD5E-s597779--1108c4d91a2386573c50c251a0e967c2.tgz +[2024-05-29 14:21:57.880467298] (P2P.IO) [ThreadId 4] P2P > PUT-FROM 0 +[2024-05-29 14:21:57.882841829] (P2P.IO) [ssh connection Just 28831] [ThreadId 4] P2P < PUT-FROM 0 + +``` + +NB we had issue with a recent 10GB switch update which was worked around by me reducing MTU on my servers from Jumbo frames to regular 1500 . But it manifested that scp did not work. Here scp worked (I did scp on typhon from rolando). And for heejung it does not work to `pull` data from roland to discovery. +"""]] diff --git a/doc/bugs/git_annex_get_is_silently_stuck_on__P2P___62___GET_0/comment_3_6a9abad163d6cc3c1c0cc90ed3a3f256._comment b/doc/bugs/git_annex_get_is_silently_stuck_on__P2P___62___GET_0/comment_3_6a9abad163d6cc3c1c0cc90ed3a3f256._comment new file mode 100644 index 0000000000..5fc7172268 --- /dev/null +++ b/doc/bugs/git_annex_get_is_silently_stuck_on__P2P___62___GET_0/comment_3_6a9abad163d6cc3c1c0cc90ed3a3f256._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 3" + date="2024-05-29T18:31:16Z" + content=""" +might be (part of) mystery resolved -- dmesg on server shows bunch of + +``` +[Wed May 29 14:12:32 2024] lockd: server dbic-mrinbox not responding, still trying +[Wed May 29 14:12:32 2024] lockd: server dbic-mrinbox not responding, still trying +[Wed May 29 14:21:33 2024] lockd: server dbic-mrinbox not responding, still trying +[Wed May 29 14:23:34 2024] lockd: server dbic-mrinbox not responding, still trying +[Wed May 29 14:26:34 2024] lockd: server dbic-mrinbox not responding, still trying + +``` +although I can still read content just fine, so it is more specifically about locking I guess (despite using pidlock?)? +"""]] diff --git a/doc/todo/make_annex___34__respect__34___.git__47__hooks__47__prepare-commit-msg/comment_7_a766358f9d5635589379d9be16ef8092._comment b/doc/todo/make_annex___34__respect__34___.git__47__hooks__47__prepare-commit-msg/comment_7_a766358f9d5635589379d9be16ef8092._comment new file mode 100644 index 0000000000..8bc1b86bd9 --- /dev/null +++ b/doc/todo/make_annex___34__respect__34___.git__47__hooks__47__prepare-commit-msg/comment_7_a766358f9d5635589379d9be16ef8092._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 7" + date="2024-05-30T14:29:43Z" + content=""" +fixed in [10.20240129-86-g3475b09c3e AKA 10.20240227~29](https://git.kitenet.net/index.cgi/git-annex.git/commit/?id=3475b09c3e93ae7e5f21de02e7ada8f460e490a4) --[[yarikoptic]] +"""]] diff --git a/doc/todo/make_annex___34__respect__34___.git__47__hooks__47__prepare-commit-msg/comment_8_b89d83fbf3f8a1e88fa607f01b182d0f._comment b/doc/todo/make_annex___34__respect__34___.git__47__hooks__47__prepare-commit-msg/comment_8_b89d83fbf3f8a1e88fa607f01b182d0f._comment new file mode 100644 index 0000000000..a6dfe57d7e --- /dev/null +++ b/doc/todo/make_annex___34__respect__34___.git__47__hooks__47__prepare-commit-msg/comment_8_b89d83fbf3f8a1e88fa607f01b182d0f._comment @@ -0,0 +1,41 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 8" + date="2024-05-30T14:34:32Z" + content=""" +FWIW the trick with `.git/last-commit-msg` did not really work for me in direct use of git-annex as I guess annex introduces changes to git-annex when I do `git-annex add` first, so it takes then PRIOR commit message: + +``` +❯ git annex add random2 +(recording state in git...) +add random2 +ok +(recording state in git...) +❯ git commit -a +[master 16107604] Will be committing changes to following files: random2 + 1 file changed, 1 insertion(+) + create mode 120000 random2 +❯ git show git-annex +commit dfb41b9b170e4d504e1e494538362e20bd73943a (git-annex) +Author: Yaroslav Halchenko +Date: Thu May 30 10:31:46 2024 -0400 + + Will be committing changes to following files: random + +diff --git a/a37/1c8/MD5E-s1000--288b6b2b44800acf433b76dc5889695c.log b/a37/1c8/MD5E-s1000--288b6b2b44800acf433b76dc5889695c.log +new file mode 100644 +index 00000000..4a3787e6 +--- /dev/null ++++ b/a37/1c8/MD5E-s1000--288b6b2b44800acf433b76dc5889695c.log +@@ -0,0 +1 @@ ++1717079506s 1 fff52b70-2aa4-4d16-8377-97fee7b2de1c +❯ cat .git/last-commit-msg +Will be committing changes to following files: random2 + +❯ git lg HEAD^^..HEAD +* 16107604 - (HEAD -> master) Will be committing changes to following files: random2 (3 minutes ago) [Yaroslav Halchenko] +* d4840167 - Will be committing changes to following files: random (4 minutes ago) [Yaroslav Halchenko] + +``` +"""]]