From f818da2f851db8c55d152c297a022b34c0d4edbb Mon Sep 17 00:00:00 2001 From: kyle Date: Thu, 16 Jul 2020 21:53:14 +0000 Subject: [PATCH 01/12] bug: find --batch regression --- ...ch_reports_files_that_have_no_content.mdwn | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 doc/bugs/find_--batch_reports_files_that_have_no_content.mdwn diff --git a/doc/bugs/find_--batch_reports_files_that_have_no_content.mdwn b/doc/bugs/find_--batch_reports_files_that_have_no_content.mdwn new file mode 100644 index 0000000000..8c481401a0 --- /dev/null +++ b/doc/bugs/find_--batch_reports_files_that_have_no_content.mdwn @@ -0,0 +1,63 @@ +As of 75aab72d2 (mostly done with location log precaching, +2020-07-13), feeding `git annex find --batch` (no matching options) +the name of a file whose content is not present locally outputs its +name rather than an empty string. + +[[!format sh """ +cd "$(mktemp -d ${TMPDIR:-/tmp}/-XXXXXXX)" +git init +git annex init +echo . >f1 +echo .. >f2 +git annex add f1 f2 +git commit -mfiles +git annex drop --force f1 +echo "--- find, no batch" +git annex find f1 f2 +echo "--- find, batch" +echo "f1\nf2" | git annex find --batch +git annex version +"""]] + +Here's the output on the current tip of master (77299ae6e): + +``` +[...] +--- find, no batch +f2 +--- find, batch +f1 +f2 +git-annex version: 8.20200618-g77299ae6e +build flags: Assistant Webapp Pairing S3 WebDAV Inotify TorrentParser Feeds Testsuite +dependency versions: aws-0.21.1 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.4 feed-1.2.0.1 ghc-8.6.5 http-client-0.5.14 persistent-sqlite-2.9.3 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0.1 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs hook external +operating system: linux x86_64 +supported repository versions: 8 +upgrade supported from repository versions: 0 1 2 3 4 5 6 7 +local repository version: 8 +``` + +And here's the expected output on the parent of 75aab72d2: + +``` +[...] +--- find, no batch +f2 +--- find, batch + +f2 +git-annex version: 8.20200618-gdf5860980 +build flags: Assistant Webapp Pairing S3 WebDAV Inotify TorrentParser Feeds Testsuite +dependency versions: aws-0.21.1 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.4 feed-1.2.0.1 ghc-8.6.5 http-client-0.5.14 persistent-sqlite-2.9.3 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0.1 +key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL +remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs hook external +operating system: linux x86_64 +supported repository versions: 8 +upgrade supported from repository versions: 0 1 2 3 4 5 6 7 +local repository version: 8 +``` + +[[!meta author=kyle]] +[[!tag projects/datalad]] From 4f152089eb4af77e11ca05af3949887e9f344e6f Mon Sep 17 00:00:00 2001 From: yarikoptic Date: Thu, 16 Jul 2020 22:01:17 +0000 Subject: [PATCH 02/12] Added a comment: Windows build of file (which includes libmagic) --- ...mment_6_5542a421c6c7c878394d7dd7a3bce9c1._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/todo/provide_windows_build_with_MagicMime/comment_6_5542a421c6c7c878394d7dd7a3bce9c1._comment diff --git a/doc/todo/provide_windows_build_with_MagicMime/comment_6_5542a421c6c7c878394d7dd7a3bce9c1._comment b/doc/todo/provide_windows_build_with_MagicMime/comment_6_5542a421c6c7c878394d7dd7a3bce9c1._comment new file mode 100644 index 0000000000..6afc07dc34 --- /dev/null +++ b/doc/todo/provide_windows_build_with_MagicMime/comment_6_5542a421c6c7c878394d7dd7a3bce9c1._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="Windows build of file (which includes libmagic)" + date="2020-07-16T22:01:17Z" + content=""" +- I have successfully cross-built (well -- it produced binaries, didn't test if they are usable) built `file` using/as shown in https://github.com/nscaife/file-windows/pull/9 +- There is https://github.com/julian-r/file-windows with a visual studio built `file`. releases provide builds using difference VS versions, not sure if it needs any VS runtime +- In conda apparently there is already `m2-file` package (a bit dated from 3 years ago) + +so may be one of those ways could assist in establishing a built with MAGIC support on windows? +"""]] From aa554e1adbaf22d384fc2756037f3a3f67feedd2 Mon Sep 17 00:00:00 2001 From: erewhon Date: Fri, 17 Jul 2020 16:38:22 +0000 Subject: [PATCH 03/12] --- ...special_remote__44___even_if_not_in_preferred_contents.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/forum/Exporting_files_to_special_remote__44___even_if_not_in_preferred_contents.mdwn diff --git a/doc/forum/Exporting_files_to_special_remote__44___even_if_not_in_preferred_contents.mdwn b/doc/forum/Exporting_files_to_special_remote__44___even_if_not_in_preferred_contents.mdwn new file mode 100644 index 0000000000..62988f2c5c --- /dev/null +++ b/doc/forum/Exporting_files_to_special_remote__44___even_if_not_in_preferred_contents.mdwn @@ -0,0 +1,3 @@ +I use my ereader as a special remote since its file system does not support symbolic links. My preferred contents expression for this remote includes everything in my library of books which is under 6 mb. However, I do occasionally want to export specific books above this size limit. Is there a good way of doing this besides explicitly adding the file to the preferred content expression? + +Many thanks From c46b9ac4aeb1f284fa1983c5285797507ec0d0e1 Mon Sep 17 00:00:00 2001 From: yarikoptic Date: Sat, 18 Jul 2020 03:54:34 +0000 Subject: [PATCH 04/12] initial 2nd wave of whining about startup time and to consider prelink or alike --- ...how_avoid_all_the_failing_opens__63__.mdwn | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__.mdwn diff --git a/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__.mdwn b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__.mdwn new file mode 100644 index 0000000000..6da8dc6d88 --- /dev/null +++ b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__.mdwn @@ -0,0 +1,63 @@ +Original TODO issue was [reincarnated](https://git-annex.branchable.com/todo/could_standalone___39__fixed__39___git-annex_binaries_be_prelinked__63__/#comment-0a78eeecf21ae7778436e233f9df22c2) and (thank you `git log -S`) and originally marked fixed by [6.20160613-81-gc4229be9a AKA 6.20160619~60](https://git.kitenet.net/index.cgi/git-annex.git/commit/?id=c4229be9a7a2318ef71b9ae433bc14bf604c9caf). Using standalone build of annex is notably (~30%) slower than any other. I was stracing running of a sample datalad test and looked inside to see e.g. + +``` +4188353 1595043333.316032 openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3 +4188353 1595043333.316089 openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3 +4188353 1595043333.316524 openat(AT_FDCWD, "/usr/lib/git-annex.linux/bin/git", O_RDONLY) = 3 +4188353 1595043333.316818 openat(AT_FDCWD, "/usr/lib/git-annex.linux/shimmed/git/git", O_RDONLY|O_CLOEXEC) = 3 +4188353 1595043333.316992 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/tls/haswell/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317022 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/tls/haswell/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317049 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/tls/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317079 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/tls/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317106 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/haswell/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317133 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/haswell/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317160 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317187 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317216 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/tls/haswell/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317248 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/tls/haswell/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317278 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/tls/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317306 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/tls/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317336 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/haswell/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317367 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/haswell/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317395 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317426 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317455 openat(AT_FDCWD, "/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/haswell/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317482 openat(AT_FDCWD, "/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/haswell/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317509 openat(AT_FDCWD, "/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317536 openat(AT_FDCWD, "/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317562 openat(AT_FDCWD, "/usr/lib/git-annex.linux//etc/ld.so.conf.d/haswell/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317589 openat(AT_FDCWD, "/usr/lib/git-annex.linux//etc/ld.so.conf.d/haswell/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317615 openat(AT_FDCWD, "/usr/lib/git-annex.linux//etc/ld.so.conf.d/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317642 openat(AT_FDCWD, "/usr/lib/git-annex.linux//etc/ld.so.conf.d/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317670 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib64/tls/haswell/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317697 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib64/tls/haswell/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317724 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib64/tls/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317750 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib64/tls/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317778 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib64/haswell/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317805 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib64/haswell/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317832 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib64/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317858 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317889 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib/x86_64-linux-gnu/tls/haswell/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317917 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib/x86_64-linux-gnu/tls/haswell/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317943 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib/x86_64-linux-gnu/tls/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317970 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib/x86_64-linux-gnu/tls/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.317997 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib/x86_64-linux-gnu/haswell/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.318024 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib/x86_64-linux-gnu/haswell/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.318051 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib/x86_64-linux-gnu/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.318077 openat(AT_FDCWD, "/usr/lib/git-annex.linux//lib/x86_64-linux-gnu/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.318105 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/tls/haswell/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.318135 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/tls/haswell/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.318162 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/tls/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.318189 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/tls/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.318216 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/haswell/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.318243 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/haswell/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.318270 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/x86_64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +4188353 1595043333.318299 openat(AT_FDCWD, "/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = 3 + +``` +so there is about 40 unsuccessful attempts to find/open libpcre2-8.so.0 . Other libraries might be less or more likely -- I didn't check... this one alone starts at 1595043333.316992 and finally succeeds at 1595043333.318299 -- so ~1.3 msec to just find this single library in a single invocation of git-annex (and we do many of them)... may be prelink or some other trick could be tried again? or if that is the lost effect of the original patch -- could there be some regression test added ? + +git annex build: 8.20200617+git192-g5849bd634-1~ndall+1 + +[[!meta author=yoh]] +[[!tag projects/datalad]] From 360de9446ea395a965e795b593e31d0348c87c76 Mon Sep 17 00:00:00 2001 From: yarikoptic Date: Sat, 18 Jul 2020 03:57:20 +0000 Subject: [PATCH 05/12] Added a comment --- ...ment_1_c67d403cb5bb67c3486ff9ade9607b2f._comment | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_1_c67d403cb5bb67c3486ff9ade9607b2f._comment diff --git a/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_1_c67d403cb5bb67c3486ff9ade9607b2f._comment b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_1_c67d403cb5bb67c3486ff9ade9607b2f._comment new file mode 100644 index 0000000000..89be386df6 --- /dev/null +++ b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_1_c67d403cb5bb67c3486ff9ade9607b2f._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 1" + date="2020-07-18T03:57:20Z" + content=""" +a test could be I guess as simple as + +``` +[ `strace -f git annex version 2>&1 1>/dev/null | grep open.*libpcre.*ENOENT | wc -l` -lt 2 ] +``` +or some other number than the ideal `2` (i.e. a direct hit right away ;)) but something less than current 47 +"""]] From 526a549766b578aba112e90990005b2aaf41b39b Mon Sep 17 00:00:00 2001 From: Ilya_Shlyakhter Date: Sat, 18 Jul 2020 03:58:18 +0000 Subject: [PATCH 06/12] Added a comment --- .../comment_1_8276c225867733e7a79ba47be2447b1e._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Exporting_files_to_special_remote__44___even_if_not_in_preferred_contents/comment_1_8276c225867733e7a79ba47be2447b1e._comment diff --git a/doc/forum/Exporting_files_to_special_remote__44___even_if_not_in_preferred_contents/comment_1_8276c225867733e7a79ba47be2447b1e._comment b/doc/forum/Exporting_files_to_special_remote__44___even_if_not_in_preferred_contents/comment_1_8276c225867733e7a79ba47be2447b1e._comment new file mode 100644 index 0000000000..e0d473d5bc --- /dev/null +++ b/doc/forum/Exporting_files_to_special_remote__44___even_if_not_in_preferred_contents/comment_1_8276c225867733e7a79ba47be2447b1e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="Ilya_Shlyakhter" + avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0" + subject="comment 1" + date="2020-07-18T03:58:18Z" + content=""" +You could use a [[metadata|git-annex-metadata]] field to mark the content you want, then use a metadata-based preferred content expression. +"""]] From 5215fe92b992243b33efb4853e67f88bef135a96 Mon Sep 17 00:00:00 2001 From: yarikoptic Date: Sat, 18 Jul 2020 04:34:05 +0000 Subject: [PATCH 07/12] Added a comment --- ..._c67fe4be932b8e7e365ed336e612b526._comment | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_2_c67fe4be932b8e7e365ed336e612b526._comment diff --git a/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_2_c67fe4be932b8e7e365ed336e612b526._comment b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_2_c67fe4be932b8e7e365ed336e612b526._comment new file mode 100644 index 0000000000..f04472d8f5 --- /dev/null +++ b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_2_c67fe4be932b8e7e365ed336e612b526._comment @@ -0,0 +1,72 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 2" + date="2020-07-18T04:34:05Z" + content=""" + +
+also for some reason paths like `ld.so.conf.d` are consulted + +```shell +$> strace -f git annex version 2>&1 1>/dev/null | grep ld.so +access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5629] access(\"/etc/ld.so.preload\", R_OK +[pid 5629] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5631] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5631] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5632] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5632] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5629] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5636] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5636] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5637] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5637] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5641] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5641] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5642] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5642] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5643] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5643] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5629] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5629] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/haswell/x86_64/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/haswell/x86_64\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/haswell/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/haswell\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/x86_64/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/x86_64\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/haswell/x86_64/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/haswell/x86_64\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/haswell/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/haswell\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/x86_64/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/x86_64\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d\", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0 +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libsqlite3.so.0\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libpthread.so.0\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libmagic.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libxml2.so.2\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libz.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +.... +``` +
+ +I thought that somehow may be ld.so.conf could indeed be used to actually configure paths for libraries to index at debian package installation level... + +interestingly, if I do directly what you are doing in the ld.so symlink shim, I get a \"direct hit\": + +``` +$> ( GIT_ANNEX_PROGRAMPATH=/usr/lib/git-annex.linux/bin/git-annex strace -f /usr/lib/git-annex.linux/exe/git-annex --library-path /usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv:/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit:/usr/lib/git-annex.linux//etc/ld.so.conf.d:/usr/lib/git-annex.linux//lib64:/usr/lib/git-annex.linux//lib/x86_64-linux-gnu:/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu: /usr/lib/git-annex.linux/shimmed/git-annex/git-annex version 2>&1 ; ) | grep pcre +[pid 8539] openat(AT_FDCWD, \"/usr/lib/git-annex.linux/usr/lib/x86_64-linux-gnu/libpcre2-8.so.0\", O_RDONLY|O_CLOEXEC + +``` +but something I guess throws it off in the \"top level\" runshell (sorry -- dunno a better word) since I do observe all those numerous attempts stracing the `/usr/lib/git-annex.linux/git-annex` +"""]] From d7b4df85e45b361c4369ebad5adc3dfae9d5c6e0 Mon Sep 17 00:00:00 2001 From: yarikoptic Date: Sat, 18 Jul 2020 04:34:26 +0000 Subject: [PATCH 08/12] Added a comment --- ..._6ae9e999b6af6b6bfe078dd187f7ff22._comment | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_3_6ae9e999b6af6b6bfe078dd187f7ff22._comment diff --git a/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_3_6ae9e999b6af6b6bfe078dd187f7ff22._comment b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_3_6ae9e999b6af6b6bfe078dd187f7ff22._comment new file mode 100644 index 0000000000..ea25cff3f8 --- /dev/null +++ b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_3_6ae9e999b6af6b6bfe078dd187f7ff22._comment @@ -0,0 +1,72 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 3" + date="2020-07-18T04:34:26Z" + content=""" + +
+also for some reason paths like `ld.so.conf.d` are consulted + +```shell +$> strace -f git annex version 2>&1 1>/dev/null | grep ld.so +access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5629] access(\"/etc/ld.so.preload\", R_OK +[pid 5629] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5631] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5631] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5632] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5632] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5629] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5636] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5636] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5637] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5637] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5641] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5641] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5642] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5642] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5643] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5643] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5629] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/etc/ld.so.cache\", O_RDONLY|O_CLOEXEC) = 3 +[pid 5629] access(\"/etc/ld.so.preload\", R_OK) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/haswell/x86_64/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/haswell/x86_64\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/haswell/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/haswell\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/x86_64/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/x86_64\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/tls\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/haswell/x86_64/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/haswell/x86_64\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/haswell/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/haswell\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/x86_64/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d/x86_64\", 0x7ffd99c76b60) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libm.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] stat(\"/usr/lib/git-annex.linux//etc/ld.so.conf.d\", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0 +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libsqlite3.so.0\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libpthread.so.0\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libmagic.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libxml2.so.2\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libz.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +[pid 5629] openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +.... +``` +
+ +I thought that somehow may be ld.so.conf could indeed be used to actually configure paths for libraries to index at debian package installation level... + +interestingly, if I do directly what you are doing in the ld.so symlink shim, I get a \"direct hit\": + +``` +$> ( GIT_ANNEX_PROGRAMPATH=/usr/lib/git-annex.linux/bin/git-annex strace -f /usr/lib/git-annex.linux/exe/git-annex --library-path /usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv:/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit:/usr/lib/git-annex.linux//etc/ld.so.conf.d:/usr/lib/git-annex.linux//lib64:/usr/lib/git-annex.linux//lib/x86_64-linux-gnu:/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu: /usr/lib/git-annex.linux/shimmed/git-annex/git-annex version 2>&1 ; ) | grep pcre +[pid 8539] openat(AT_FDCWD, \"/usr/lib/git-annex.linux/usr/lib/x86_64-linux-gnu/libpcre2-8.so.0\", O_RDONLY|O_CLOEXEC + +``` +but something I guess throws it off in the \"top level\" runshell (sorry -- dunno a better word) since I do observe all those numerous attempts stracing the `/usr/lib/git-annex.linux/git-annex` +"""]] From a6b0147b7f462ac67894f2979a6365ce5d675739 Mon Sep 17 00:00:00 2001 From: yarikoptic Date: Sat, 18 Jul 2020 04:49:49 +0000 Subject: [PATCH 09/12] Added a comment --- ..._bfb9b94048fc239f29351a9d38c8d4e3._comment | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_4_bfb9b94048fc239f29351a9d38c8d4e3._comment diff --git a/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_4_bfb9b94048fc239f29351a9d38c8d4e3._comment b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_4_bfb9b94048fc239f29351a9d38c8d4e3._comment new file mode 100644 index 0000000000..fa4bc12558 --- /dev/null +++ b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_4_bfb9b94048fc239f29351a9d38c8d4e3._comment @@ -0,0 +1,38 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 4" + date="2020-07-18T04:49:49Z" + content=""" +and since debian standalone is a \"static\" installation, most likely all the dance in `runshell` could be avoided upon every invocation and paths could be computed/recorded at installation time and then shim could be as lean as what I get with + +```shell +$> rm -rf /tmp/fastannex; mkdir /tmp/fastannex; for f in /usr/lib/git-annex.linux/bin/git-annex{,-shell}; do echo -e \"#\!/bin/sh\nGIT_ANNEX_DIR=/usr/lib/git-annex.linux/ GIT_ANNEX_LD_LIBRARY_PATH=/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv:/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit:/usr/lib/git-annex.linux//etc/ld.so.conf.d:/usr/lib/git-annex.linux//lib64:/usr/lib/git-annex.linux//lib/x86_64-linux-gnu:/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu: $f \\"\$@\\"\" > /tmp/fastannex/$(basename $f); done; chmod a+x /tmp/fastannex/* +``` + +and speed up on a random sample test is already obvious: + + +``` +$> time PATH=/tmp/fastannex:$PATH python -m nose datalad/support/tests/test_annexrepo.py:test_annex_version_handling_bad_git_annex +. +---------------------------------------------------------------------- +Ran 1 test in 0.280s + +OK +PATH=/tmp/fastannex:$PATH python -m nose 0.78s user 0.25s system 100% cpu 1.029 total +``` + +vs + +``` +$> time python -m nose datalad/support/tests/test_annexrepo.py:test_annex_version_handling_bad_git_annex +. +---------------------------------------------------------------------- +Ran 1 test in 0.498s + +OK +python -m nose 0.98s user 0.29s system 101% cpu 1.254 total +``` + +"""]] From 4ab711e15323056874eb1aae91b80359fea9990e Mon Sep 17 00:00:00 2001 From: yarikoptic Date: Sat, 18 Jul 2020 04:50:12 +0000 Subject: [PATCH 10/12] Added a comment --- ..._7a57b6713788778f6a6e8144cf3931b2._comment | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_5_7a57b6713788778f6a6e8144cf3931b2._comment diff --git a/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_5_7a57b6713788778f6a6e8144cf3931b2._comment b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_5_7a57b6713788778f6a6e8144cf3931b2._comment new file mode 100644 index 0000000000..b57fcab42d --- /dev/null +++ b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_5_7a57b6713788778f6a6e8144cf3931b2._comment @@ -0,0 +1,38 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 5" + date="2020-07-18T04:50:12Z" + content=""" +and since debian standalone is a \"static\" installation, most likely all the dance in `runshell` could be avoided upon every invocation and paths could be computed/recorded at installation time and then shim could be as lean as what I get with + +```shell +$> rm -rf /tmp/fastannex; mkdir /tmp/fastannex; for f in /usr/lib/git-annex.linux/bin/git-annex{,-shell}; do echo -e \"#\!/bin/sh\nGIT_ANNEX_DIR=/usr/lib/git-annex.linux/ GIT_ANNEX_LD_LIBRARY_PATH=/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv:/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit:/usr/lib/git-annex.linux//etc/ld.so.conf.d:/usr/lib/git-annex.linux//lib64:/usr/lib/git-annex.linux//lib/x86_64-linux-gnu:/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu: $f \\"\$@\\"\" > /tmp/fastannex/$(basename $f); done; chmod a+x /tmp/fastannex/* +``` + +and speed up on a random sample test is already obvious: + + +``` +$> time PATH=/tmp/fastannex:$PATH python -m nose datalad/support/tests/test_annexrepo.py:test_annex_version_handling_bad_git_annex +. +---------------------------------------------------------------------- +Ran 1 test in 0.280s + +OK +PATH=/tmp/fastannex:$PATH python -m nose 0.78s user 0.25s system 100% cpu 1.029 total +``` + +vs + +``` +$> time python -m nose datalad/support/tests/test_annexrepo.py:test_annex_version_handling_bad_git_annex +. +---------------------------------------------------------------------- +Ran 1 test in 0.498s + +OK +python -m nose 0.98s user 0.29s system 101% cpu 1.254 total +``` + +"""]] From 7ee0bcbee7ae176b56a35d8d3457a17d88b7f7fe Mon Sep 17 00:00:00 2001 From: yarikoptic Date: Sat, 18 Jul 2020 05:09:32 +0000 Subject: [PATCH 11/12] Added a comment --- ..._3a849428b335964f80a8503d44d51c5a._comment | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_6_3a849428b335964f80a8503d44d51c5a._comment diff --git a/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_6_3a849428b335964f80a8503d44d51c5a._comment b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_6_3a849428b335964f80a8503d44d51c5a._comment new file mode 100644 index 0000000000..0542a19a4f --- /dev/null +++ b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_6_3a849428b335964f80a8503d44d51c5a._comment @@ -0,0 +1,39 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 6" + date="2020-07-18T05:09:32Z" + content=""" +
+bundled git is also a victim on many counts (bringing locale question back as well) + +```shell +$> strace -f /usr/lib/git-annex.linux/git --version 2>&1 | grep open.*ENOE | nl | tail + 155 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 156 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//lib64/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 157 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 158 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 159 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 160 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//lib64/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 161 openat(AT_FDCWD, \"/usr/lib/locale/locale-archive\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 162 openat(AT_FDCWD, \"/usr/lib/locale/en_US.UTF-8/LC_MESSAGES\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 163 openat(AT_FDCWD, \"/usr/lib/locale/en_US.UTF-8/LC_TIME\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 164 openat(AT_FDCWD, \"/usr/lib/locale/en_US.UTF-8/LC_CTYPE\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +1 47553.....................................:Sat 18 Jul 2020 01:07:51 AM EDT:. +(git)lena:~datalad/datalad-maint[maint]git +$> strace -f /usr/lib/git-annex.linux/git --version 2>&1 | grep open.*ENOE | grep -v locale | nl | tail + 54 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libpthread.so.0\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 55 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//lib64/libpthread.so.0\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 56 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 57 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 58 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 59 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//lib64/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 60 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 61 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 62 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 63 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//lib64/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + +``` +
+ +"""]] From 6a053888772a86a7f91d5fa3ff4da5642aa7aa3d Mon Sep 17 00:00:00 2001 From: yarikoptic Date: Sat, 18 Jul 2020 05:09:54 +0000 Subject: [PATCH 12/12] Added a comment --- ..._5f5d9af7ed807b8ef203426f717822b2._comment | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_7_5f5d9af7ed807b8ef203426f717822b2._comment diff --git a/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_7_5f5d9af7ed807b8ef203426f717822b2._comment b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_7_5f5d9af7ed807b8ef203426f717822b2._comment new file mode 100644 index 0000000000..78e5a70663 --- /dev/null +++ b/doc/todo/may_be___40__again__41___to_prelink_or_somehow_avoid_all_the_failing_opens__63__/comment_7_5f5d9af7ed807b8ef203426f717822b2._comment @@ -0,0 +1,39 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 7" + date="2020-07-18T05:09:54Z" + content=""" +
+bundled git is also a victim on many counts (bringing locale question back as well) + +```shell +$> strace -f /usr/lib/git-annex.linux/git --version 2>&1 | grep open.*ENOE | nl | tail + 155 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 156 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//lib64/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 157 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 158 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 159 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 160 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//lib64/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 161 openat(AT_FDCWD, \"/usr/lib/locale/locale-archive\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 162 openat(AT_FDCWD, \"/usr/lib/locale/en_US.UTF-8/LC_MESSAGES\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 163 openat(AT_FDCWD, \"/usr/lib/locale/en_US.UTF-8/LC_TIME\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 164 openat(AT_FDCWD, \"/usr/lib/locale/en_US.UTF-8/LC_CTYPE\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) +1 47553.....................................:Sat 18 Jul 2020 01:07:51 AM EDT:. +(git)lena:~datalad/datalad-maint[maint]git +$> strace -f /usr/lib/git-annex.linux/git --version 2>&1 | grep open.*ENOE | grep -v locale | nl | tail + 54 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libpthread.so.0\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 55 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//lib64/libpthread.so.0\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 56 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 57 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 58 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 59 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//lib64/librt.so.1\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 60 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/gconv/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 61 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//usr/lib/x86_64-linux-gnu/audit/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 62 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//etc/ld.so.conf.d/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + 63 openat(AT_FDCWD, \"/usr/lib/git-annex.linux//lib64/libc.so.6\", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) + +``` +
+ +"""]]