From 7a5969cc9263ace14220b8e9943e0f8a5a4f4a7f Mon Sep 17 00:00:00 2001 From: "https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" Date: Mon, 11 Jan 2016 16:47:04 +0000 Subject: [PATCH 1/9] --- ...redownload_even_if_it_already_present.mdwn | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 doc/bugs/addurl_--file__causes_file_redownload_even_if_it_already_present.mdwn diff --git a/doc/bugs/addurl_--file__causes_file_redownload_even_if_it_already_present.mdwn b/doc/bugs/addurl_--file__causes_file_redownload_even_if_it_already_present.mdwn new file mode 100644 index 0000000000..1016966e30 --- /dev/null +++ b/doc/bugs/addurl_--file__causes_file_redownload_even_if_it_already_present.mdwn @@ -0,0 +1,24 @@ +### Please describe the problem. + +IMHO annex shouldn't redownload the file (not yet under annex/git control) entirely if pointed by --file=FILE FILE exists. + +[[!format sh """ +$> chmod a+w -R /tmp/123; rm -rf /tmp/123; mkdir /tmp/123; cd /tmp/123; git init; git annex init; +Initialized empty Git repository in /tmp/123/.git/ +init ok + +$> wget -q http://www.onerussian.com/tmp/banner.png + +$> ls -l +total 28 +-rw------- 1 yoh yoh 25319 Sep 17 13:49 banner.png + +$> git annex addurl --file=banner.png http://www.onerussian.com/tmp/banner.png +addurl banner.png (downloading http://www.onerussian.com/tmp/banner.png ...) +/tmp/123/.git/annex/tmp/URL-s25319--http&c%%w 100%[=================================================================================================>] 24.73K --.-KB/s in 0.003s +ok + +$> git annex version +git-annex version: 6.20160104+gitg0cf96be-1~ndall+1 +"""] +[[!meta author=yoh]] From e540c3ffc576d2fe11ccc55271d967af050a3394 Mon Sep 17 00:00:00 2001 From: "https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" Date: Mon, 11 Jan 2016 16:47:52 +0000 Subject: [PATCH 2/9] --- ...ile__causes_file_redownload_even_if_it_already_present.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/bugs/addurl_--file__causes_file_redownload_even_if_it_already_present.mdwn b/doc/bugs/addurl_--file__causes_file_redownload_even_if_it_already_present.mdwn index 1016966e30..c72084d62a 100644 --- a/doc/bugs/addurl_--file__causes_file_redownload_even_if_it_already_present.mdwn +++ b/doc/bugs/addurl_--file__causes_file_redownload_even_if_it_already_present.mdwn @@ -3,6 +3,7 @@ IMHO annex shouldn't redownload the file (not yet under annex/git control) entirely if pointed by --file=FILE FILE exists. [[!format sh """ + $> chmod a+w -R /tmp/123; rm -rf /tmp/123; mkdir /tmp/123; cd /tmp/123; git init; git annex init; Initialized empty Git repository in /tmp/123/.git/ init ok @@ -20,5 +21,7 @@ ok $> git annex version git-annex version: 6.20160104+gitg0cf96be-1~ndall+1 + """] + [[!meta author=yoh]] From d73469a87fc6f92ecc8d53a0673162dfb7861bd4 Mon Sep 17 00:00:00 2001 From: "https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" Date: Mon, 11 Jan 2016 22:10:57 +0000 Subject: [PATCH 3/9] --- ...ot_forbids_reliable_parsing_of_output.mdwn | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/bugs/inconsistent_output_upon_addurl_--batch_complicates_if_not_forbids_reliable_parsing_of_output.mdwn diff --git a/doc/bugs/inconsistent_output_upon_addurl_--batch_complicates_if_not_forbids_reliable_parsing_of_output.mdwn b/doc/bugs/inconsistent_output_upon_addurl_--batch_complicates_if_not_forbids_reliable_parsing_of_output.mdwn new file mode 100644 index 0000000000..a0019456c6 --- /dev/null +++ b/doc/bugs/inconsistent_output_upon_addurl_--batch_complicates_if_not_forbids_reliable_parsing_of_output.mdwn @@ -0,0 +1,21 @@ +### Please describe the problem. + +if using addurl in --batch mode, there is no --json formatting and text output is somewhat inconsistent. It might include "downloading ..." with 'ok' on a separate line, possibly with a msg on recording state in git, or within the same line report 'ok', or 'failed' in upcoming lines. This complicates parsing of such output to verify correct operation for a given addurl pair + +[[!format sh """ + +$> echo 'http://127.0.0.1:33369/about.txt about.txt' | git -c receive.autogc=false annex addurl --with-files --batch --debug 2>/dev/null +addurl about.txt (downloading http://127.0.0.1:33369/about.txt ...) +ok +(recording state in git...) + +$> echo 'http://127.0.0.1:33369/about.txt about.txt' | git -c receive.autogc=false annex addurl --with-files --batch --debug 2>/dev/null +addurl about.txt ok + +$> echo 'http://127.0.0.1:33369/about.stxt about.txt' | git -c receive.autogc=false annex addurl --with-files --batch --debug 2>/dev/null +addurl about.txt +failed + +"""]] + +[[!meta author=yoh]] From d003501fab4d86e1bda38dca8c9e934752e132fc Mon Sep 17 00:00:00 2001 From: "emanuele.ruffaldi@56b9c9a5c1f873994b869248e9b53aa20f437d20" Date: Tue, 12 Jan 2016 00:32:31 +0000 Subject: [PATCH 4/9] --- ...Accessing_files_in_bare_remotes_without_git-annex.mdwn | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/tips/Accessing_files_in_bare_remotes_without_git-annex.mdwn diff --git a/doc/tips/Accessing_files_in_bare_remotes_without_git-annex.mdwn b/doc/tips/Accessing_files_in_bare_remotes_without_git-annex.mdwn new file mode 100644 index 0000000000..c9af79cdfd --- /dev/null +++ b/doc/tips/Accessing_files_in_bare_remotes_without_git-annex.mdwn @@ -0,0 +1,8 @@ + +git-annex is amazing for my large files backup necessities, but I am a bit scared for the long-term possibility of accessing data, without git-annex. + +For this reason I have prepared a small Python tool that accesses the content of bare git-annex repositories. The tool retrieves the locations of a file, and the path to the file in the current annex. This works with the last v6 version of the git-annex format. + +This is motivated by the fact that in the master branch the files are stores as links to .git/annex/objects/XX/YY/KEY/KEY (as it is correct for non-bare repos), while in the bare the files are stored in annex/objecst/ZZ/WW/KEY/KEY. + +The source code is available here: From ac17961c437a6bb83e98fa157c6644cc5e5ad440 Mon Sep 17 00:00:00 2001 From: "https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" Date: Tue, 12 Jan 2016 03:30:25 +0000 Subject: [PATCH 5/9] initial report --- ...dd_file_into_git_until_pipe_is_closed.mdwn | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 doc/bugs/addurl_--batch__--with-files_doesn__39__t_add_file_into_git_until_pipe_is_closed.mdwn diff --git a/doc/bugs/addurl_--batch__--with-files_doesn__39__t_add_file_into_git_until_pipe_is_closed.mdwn b/doc/bugs/addurl_--batch__--with-files_doesn__39__t_add_file_into_git_until_pipe_is_closed.mdwn new file mode 100644 index 0000000000..46534587df --- /dev/null +++ b/doc/bugs/addurl_--batch__--with-files_doesn__39__t_add_file_into_git_until_pipe_is_closed.mdwn @@ -0,0 +1,96 @@ +### Please describe the problem. + +only happens if file doesn't exist yet and gets downloaded (interestingly, as I reported in another report, if file exists it gets re-downloaded but this bug doesn't repeat) + +[[!format sh """ + +$> chmod a+w -R /tmp/123; rm -rf /tmp/123; mkdir /tmp/123; cd /tmp/123; git init; git annex init; +Initialized empty Git repository in /tmp/123/.git/ +init ok +(recording state in git...) + +$> mkfifo /tmp/pipe + +$> cat /tmp/pipe | git annex --debug addurl --batch --with-files +^Z +[2] + 1084 suspended cat /tmp/pipe | + 1085 suspended git annex --debug addurl --batch --with-files + +$> bg +[2] - 1084 continued cat /tmp/pipe | + 1085 continued git annex --debug addurl --batch --with-files + +$> exec 3>/tmp/pipe + +$> echo "http://www.onerussian.com/tmp/banner.png banner.png" >&3 +[2016-01-11 22:25:34.423037] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"] +[2016-01-11 22:25:34.427649] process done ExitSuccess +[2016-01-11 22:25:34.428067] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2016-01-11 22:25:34.432199] process done ExitSuccess + +$> [2016-01-11 22:25:34.432834] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","log","refs/heads/git-annex..87efc349dceb06df635c8485ef536ae11f48736c","-n1","--pretty=%H"] +[2016-01-11 22:25:34.438533] process done ExitSuccess +[2016-01-11 22:25:34.439666] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","cat-file","--batch"] +[2016-01-11 22:25:34.444875] read: quvi ["--version"] +[2016-01-11 22:25:34.452158] process done ExitSuccess +[2016-01-11 22:25:34.452563] call: quvi ["--verbosity","mute","--support","http://www.onerussian.com/tmp/banner.png"] +[2016-01-11 22:25:34.468071] process done ExitFailure 65 +addurl banner.png (downloading http://www.onerussian.com/tmp/banner.png ...) +[2016-01-11 22:25:34.500939] call: wget ["-q","--show-progress","--clobber","-c","-O","/tmp/123/.git/annex/tmp/URL-s25319--http&c%%www.onerussian.com%tmp%banner.png","http://www.onerussian.com/tmp/banner.png","--user-agent","git-annex/6.20160104+gitg0cf96be-1~ndall+1"] +/tmp/123/.git/annex/tmp/URL-s25319--http&c%%www.on 100%[==============================================================================================================>] 24.73K --.-KB/s in 0s +[2016-01-11 22:25:34.518335] process done ExitSuccess +[2016-01-11 22:25:34.519195] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","check-attr","-z","--stdin","annex.backend","annex.numcopies","--"] +[2016-01-11 22:25:34.5241] read: git ["--version"] +[2016-01-11 22:25:34.527001] process done ExitSuccess +ok + +$> git status # note that file is untracked, although it is a symlink into .git/annex/objects +On branch master + +Initial commit + +Untracked files: + (use "git add ..." to include in what will be committed) + + banner.png + +nothing added to commit but untracked files present (use "git add" to track) + +$> # closing the pipe now +$> exec 3>&- +(recording state in git...) +[2016-01-11 22:26:02.947133] feed: xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","--"] +$> [2016-01-11 22:26:02.95701] process done ExitSuccess +[2016-01-11 22:26:02.961588] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","hash-object","-w","--stdin-paths","--no-filters"] +[2016-01-11 22:26:02.964507] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-index","-z","--index-info"] +[2016-01-11 22:26:02.971853] process done ExitSuccess +[2016-01-11 22:26:02.972524] process done ExitSuccess +[2016-01-11 22:26:02.973043] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"] +[2016-01-11 22:26:02.975871] process done ExitSuccess +[2016-01-11 22:26:02.977613] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","write-tree"] +[2016-01-11 22:26:02.982308] process done ExitSuccess +[2016-01-11 22:26:02.982744] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","commit-tree","0b4d294f555bc86c17de89bca35193433408bbef","--no-gpg-sign","-p","refs/heads/git-annex"] +[2016-01-11 22:26:02.990485] process done ExitSuccess +[2016-01-11 22:26:02.990941] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","update-ref","refs/heads/git-annex","1359dc4d9296bc4f5941e7299376319899ea222c"] +[2016-01-11 22:26:02.995434] process done ExitSuccess + +[2] - 1084 done cat /tmp/pipe | + 1085 done git annex --debug addurl --batch --with-files + +$> git status +On branch master + +Initial commit + +Changes to be committed: + (use "git rm --cached ..." to unstage) + + new file: banner.png + +$> git annex version +git-annex version: 6.20160104+gitg0cf96be-1~ndall+1 +... + +"""]] + +[[!meta author=yoh]] From 9bc2b9ad574e56d009af2cf899c43aec31f72255 Mon Sep 17 00:00:00 2001 From: "https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" Date: Tue, 12 Jan 2016 03:35:15 +0000 Subject: [PATCH 6/9] Added a comment --- .../comment_1_79211a11eaf733da209664a31726b3ea._comment | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/bugs/addurl_--batch__--with-files_doesn__39__t_add_file_into_git_until_pipe_is_closed/comment_1_79211a11eaf733da209664a31726b3ea._comment diff --git a/doc/bugs/addurl_--batch__--with-files_doesn__39__t_add_file_into_git_until_pipe_is_closed/comment_1_79211a11eaf733da209664a31726b3ea._comment b/doc/bugs/addurl_--batch__--with-files_doesn__39__t_add_file_into_git_until_pipe_is_closed/comment_1_79211a11eaf733da209664a31726b3ea._comment new file mode 100644 index 0000000000..fbc6924d44 --- /dev/null +++ b/doc/bugs/addurl_--batch__--with-files_doesn__39__t_add_file_into_git_until_pipe_is_closed/comment_1_79211a11eaf733da209664a31726b3ea._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4" + subject="comment 1" + date="2016-01-12T03:35:15Z" + content=""" +actually tried now the similar way in bash with pre-downloaded file, the same situation. I guess my test in python for this scenario didn't care to try to commit it without explicit 'git add' ;-) +"""]] From 4b075509f2d216e022bf7b78d8000ec3cb1e49cf Mon Sep 17 00:00:00 2001 From: "jhannwong@c9c7a67b5632a4bbc0c959cfeb3d340e02f28565" Date: Tue, 12 Jan 2016 03:52:58 +0000 Subject: [PATCH 7/9] Added a comment: Does not work --- ..._53b60f39e8ba93522eb90688237cef17._comment | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/submodules/comment_1_53b60f39e8ba93522eb90688237cef17._comment diff --git a/doc/submodules/comment_1_53b60f39e8ba93522eb90688237cef17._comment b/doc/submodules/comment_1_53b60f39e8ba93522eb90688237cef17._comment new file mode 100644 index 0000000000..cc8e3c83eb --- /dev/null +++ b/doc/submodules/comment_1_53b60f39e8ba93522eb90688237cef17._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="jhannwong@c9c7a67b5632a4bbc0c959cfeb3d340e02f28565" + nickname="jhannwong" + subject="Does not work" + date="2016-01-12T03:52:58Z" + content=""" +Truly experimental. + +1. Create a git repo, and add some git-annexed files to it. +2. Create another git repo, and make above git repo a submodule under it. +3. In the git repo in step 2, add some git-annexed files to it. +4. Create yet another git repo, and add to it the git repo in step 2 as a submodule. +5. In the git repo in step 4, do `git-annex init '. + +The symlink that is created will be wrong. +Even if you manually create a correct symlink, git-annex still fails to work. + +Truly experimental. For now, I have to avoid using git-annex in submodules. +"""]] From 99fdd14f9f830d48980fe1aeae0acaf4f94e1908 Mon Sep 17 00:00:00 2001 From: sameerds Date: Tue, 12 Jan 2016 04:07:40 +0000 Subject: [PATCH 8/9] Added a comment: cygwin/msys-independent fix? --- .../comment_4_958e887dd83b8b636df2d81ade0dbbf1._comment | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_4_958e887dd83b8b636df2d81ade0dbbf1._comment diff --git a/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_4_958e887dd83b8b636df2d81ade0dbbf1._comment b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_4_958e887dd83b8b636df2d81ade0dbbf1._comment new file mode 100644 index 0000000000..a345023215 --- /dev/null +++ b/doc/forum/__34__git_annex_get__34___on_windows_fails_with_rsync_error/comment_4_958e887dd83b8b636df2d81ade0dbbf1._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="sameerds" + subject="cygwin/msys-independent fix?" + date="2016-01-12T04:07:39Z" + content=""" +Thank you, pkitslaar, for working this out! I will try it out ASAP and provide feedback. But generally, why can we not create Windows-style paths, and then rely on the underlying platform to do the right thing? I would expect that \"rsync\" or any other binary will accept \"C:\scratch\a\" as a valid path, and then translate it appropriately for its own host such as cygwin or msys. \"Clients\" of these binaries should not have to worry about such things. +"""]] From e6e4407336cfa244835296da950ae7997f69b558 Mon Sep 17 00:00:00 2001 From: "jhannwong@c9c7a67b5632a4bbc0c959cfeb3d340e02f28565" Date: Tue, 12 Jan 2016 04:19:36 +0000 Subject: [PATCH 9/9] removed --- ..._53b60f39e8ba93522eb90688237cef17._comment | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 doc/submodules/comment_1_53b60f39e8ba93522eb90688237cef17._comment diff --git a/doc/submodules/comment_1_53b60f39e8ba93522eb90688237cef17._comment b/doc/submodules/comment_1_53b60f39e8ba93522eb90688237cef17._comment deleted file mode 100644 index cc8e3c83eb..0000000000 --- a/doc/submodules/comment_1_53b60f39e8ba93522eb90688237cef17._comment +++ /dev/null @@ -1,19 +0,0 @@ -[[!comment format=mdwn - username="jhannwong@c9c7a67b5632a4bbc0c959cfeb3d340e02f28565" - nickname="jhannwong" - subject="Does not work" - date="2016-01-12T03:52:58Z" - content=""" -Truly experimental. - -1. Create a git repo, and add some git-annexed files to it. -2. Create another git repo, and make above git repo a submodule under it. -3. In the git repo in step 2, add some git-annexed files to it. -4. Create yet another git repo, and add to it the git repo in step 2 as a submodule. -5. In the git repo in step 4, do `git-annex init '. - -The symlink that is created will be wrong. -Even if you manually create a correct symlink, git-annex still fails to work. - -Truly experimental. For now, I have to avoid using git-annex in submodules. -"""]]