From 68adc08502681181abaf050152093f0a6bcff808 Mon Sep 17 00:00:00 2001 From: yarikoptic Date: Tue, 6 Nov 2018 03:43:40 +0000 Subject: [PATCH] initial report on failing to get a file after it actually got to tmp/ fully --- ...ied_by_full_and_correct_full_download.mdwn | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 doc/bugs/annex_is_not_satisfied_by_full_and_correct_full_download.mdwn diff --git a/doc/bugs/annex_is_not_satisfied_by_full_and_correct_full_download.mdwn b/doc/bugs/annex_is_not_satisfied_by_full_and_correct_full_download.mdwn new file mode 100644 index 0000000000..ac953f6780 --- /dev/null +++ b/doc/bugs/annex_is_not_satisfied_by_full_and_correct_full_download.mdwn @@ -0,0 +1,100 @@ +### Please describe the problem. + +It seems fully and correctly obtain `get` a file but it ended up in tmp/, upon attempts to reget tries to reget it with incorrect range (since fully here already): + +[[!format sh """ +$> datalad install -g http://github.com/openneurodatasets/ds000001 +install(notneeded): /mnt/btrfs/scrap/tmp/openneuro/ds000001 (dataset) [dataset was already cloned from 'http://github.com/openneurodatasets/ds000001'] +[WARNING] Running get resulted in stderr output: download failed: Requested Range Not Satisfiable +download failed: Requested Range Not Satisfiable +git-annex: get: 1 failed + +[ERROR ] from s3-PUBLIC...; from s3-PUBLIC...; Unable to access these remotes: s3-PUBLIC; Try making some of these repositories available:; b5dd2e3d-825f-4bc2-b719-cba1059f6bfc -- root@93184394ac19:/datalad/ds000001; deaa691f-c824-4416-9bf8-a94a47dd31b5 -- [s3-PUBLIC]; ; (Note that these git remotes have annex-ignore set: origin) [get(/mnt/btrfs/scrap/tmp/openneuro/ds000001/sub-03/func/sub-03_task-balloonanalogrisktask_run-03_bold.nii.gz)] +[WARNING] could not get some content in /mnt/btrfs/scrap/tmp/openneuro/ds000001 ['/mnt/btrfs/scrap/tmp/openneuro/ds000001/sub-03/func/sub-03_task-balloonanalogrisktask_run-03_bold.nii.gz'] [get(/mnt/btrfs/scrap/tmp/openneuro/ds000001)] +action summary: + get (error: 1, impossible: 1) + install (notneeded: 1) +1 14492 ->1.....................................:Mon 05 Nov 2018 10:36:53 PM EST:. +smaug:/mnt/btrfs/scrap/tmp/openneuro +$> cd ds000001 +CHANGES dataset_description.json sub-01/ sub-03/ sub-05/ sub-07/ sub-09/ sub-11/ sub-13/ sub-15/ task-balloonanalogrisktask_bold.json +README participants.tsv sub-02/ sub-04/ sub-06/ sub-08/ sub-10/ sub-12/ sub-14/ sub-16/ +1 14493.....................................:Mon 05 Nov 2018 10:36:56 PM EST:. +(git)smaug:/mnt/btrfs/scrap/tmp/openneuro/ds000001[master] +$> git annex get --debug +[2018-11-05 22:36:59.665601708] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","symbolic-ref","-q","HEAD"] +[2018-11-05 22:36:59.669131907] process done ExitSuccess +[2018-11-05 22:36:59.669205841] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","refs/heads/master"] +[2018-11-05 22:36:59.672930992] process done ExitSuccess +[2018-11-05 22:36:59.673020181] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--cached","-z","--"] +get sub-03/func/sub-03_task-balloonanalogrisktask_run-03_bold.nii.gz [2018-11-05 22:36:59.686205786] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2018-11-05 22:36:59.690089151] process done ExitSuccess +[2018-11-05 22:36:59.690165258] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2018-11-05 22:36:59.693038961] process done ExitSuccess +[2018-11-05 22:36:59.693248238] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..4fd548ce0d4e2bc67da62795e80880a7ae1f6abc","--pretty=%H","-n1"] +[2018-11-05 22:36:59.697945062] process done ExitSuccess +[2018-11-05 22:36:59.698564444] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2018-11-05 22:36:59.698998042] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"] +(from s3-PUBLIC...) + +[2018-11-05 22:36:59.734434115] Request { + host = "openneuro.org.s3.amazonaws.com" + port = 80 + secure = False + requestHeaders = [("Range","bytes=46216487-"),("Accept-Encoding","identity"),("User-Agent","git-annex/6.20181011+git124-g94aa0e2f6-1~ndall+1")] + path = "/ds000001/sub-03/func/sub-03_task-balloonanalogrisktask_run-03_bold.nii.gz" + queryString = "?versionId=KKf9EeO6JyGQbKG_SyU51ehWSYeXiMX9" + method = "GET" + proxy = Nothing + rawBody = False + redirectCount = 10 + responseTimeout = ResponseTimeoutDefault + requestVersion = HTTP/1.1 +} + +download failed: Requested Range Not Satisfiable +(from s3-PUBLIC...) + +[2018-11-05 22:36:59.815117255] Request { + host = "openneuro.org.s3.amazonaws.com" + port = 80 + secure = False + requestHeaders = [("Range","bytes=46216487-"),("Accept-Encoding","identity"),("User-Agent","git-annex/6.20181011+git124-g94aa0e2f6-1~ndall+1")] + path = "/ds000001/sub-03/func/sub-03_task-balloonanalogrisktask_run-03_bold.nii.gz" + queryString = "?versionId=KKf9EeO6JyGQbKG_SyU51ehWSYeXiMX9" + method = "GET" + proxy = Nothing + rawBody = False + redirectCount = 10 + responseTimeout = ResponseTimeoutDefault + requestVersion = HTTP/1.1 +} + +download failed: Requested Range Not Satisfiable + + Unable to access these remotes: s3-PUBLIC + + Try making some of these repositories available: + b5dd2e3d-825f-4bc2-b719-cba1059f6bfc -- root@93184394ac19:/datalad/ds000001 + deaa691f-c824-4416-9bf8-a94a47dd31b5 -- [s3-PUBLIC] + + (Note that these git remotes have annex-ignore set: origin) +failed +[2018-11-05 22:36:59.852095523] process done ExitSuccess +[2018-11-05 22:36:59.85253563] process done ExitSuccess +git-annex: get: 1 failed + + +$> du -scb .git/annex/tmp/MD5E-s46216487--ec7b8fc23a313f1429e941fd94c2d268.nii.gz +46216487 .git/annex/tmp/MD5E-s46216487--ec7b8fc23a313f1429e941fd94c2d268.nii.gz +46216487 total + +$> md5sum .git/annex/tmp/MD5E-s46216487--ec7b8fc23a313f1429e941fd94c2d268.nii.gz +ec7b8fc23a313f1429e941fd94c2d268 .git/annex/tmp/MD5E-s46216487--ec7b8fc23a313f1429e941fd94c2d268.nii.gz + +$> git annex version +git-annex version: 6.20181011+git124-g94aa0e2f6-1~ndall+1 + +"""]] + +Not sure why it ended up not moved into the proper location but I think upon redownload, size should be verified, if "Full" - try to proceed to checksum verification etc.