Linux standalone build now includes its own glibc and forces the linker to use it, to remove dependence on the host glibc.

Makefile

@@ -122,7 +122,7 @@ linuxstandalone: Build/Standalone
 	install -d "$(LINUXSTANDALONE_DEST)/templates"
 	
 	touch "$(LINUXSTANDALONE_DEST)/libdirs.tmp"
-	for lib in $$(ldd "$(LINUXSTANDALONE_DEST)"/bin/* $$(find "$(LINUXSTANDALONE_DEST)"/git-core/ -type f) | grep -v -f standalone/linux/glibc-libs | grep -v "not a dynamic executable" | egrep '^	' | sed 's/^\t//' | sed 's/.*=> //' | cut -d ' ' -f 1 | sort | uniq); do \
+	for lib in $$(ldd "$(LINUXSTANDALONE_DEST)"/bin/* $$(find "$(LINUXSTANDALONE_DEST)"/git-core/ -type f) | grep -v "not a dynamic executable" | egrep '^	' | sed 's/^\t//' | sed 's/.*=> //' | cut -d ' ' -f 1 | sort | uniq); do \
 		dir=$$(dirname "$$lib"); \
 		install -d "$(LINUXSTANDALONE_DEST)/$$dir"; \
 		echo "$$dir" >> "$(LINUXSTANDALONE_DEST)/libdirs.tmp"; \
@@ -135,6 +135,25 @@ linuxstandalone: Build/Standalone
 	sort "$(LINUXSTANDALONE_DEST)/libdirs.tmp" | uniq > "$(LINUXSTANDALONE_DEST)/libdirs"
 	rm -f "$(LINUXSTANDALONE_DEST)/libdirs.tmp"
 
+	# Ensure bundle includes all glibc libs, and other support
+	# files it loads.
+	# XXX Debian specific.
+	cd $(LINUXSTANDALONE_DEST) && dpkg -L libc6 | grep \.so|tar c --files-from=- | tar x
+
+	find $(LINUXSTANDALONE_DEST) -type d -name gconv | head -n 1 > $(LINUXSTANDALONE_DEST)/gconvdir
+	find $(LINUXSTANDALONE_DEST) | grep ld-linux | head -n 1 | sed 's!$(LINUXSTANDALONE_DEST)/*!!' > $(LINUXSTANDALONE_DEST)/linker
+	
+	# Install linker shim for each binary.
+	for file in $$(find "$(LINUXSTANDALONE_DEST)" -type f); do \
+		if file "$$file" | grep ELF | grep -q executable; then \
+			mkdir -p "$(LINUXSTANDALONE_DEST)/shimmed"; \
+			mv "$$file" "$(LINUXSTANDALONE_DEST)/shimmed"; \
+			echo "#!/bin/sh" > "$$file"; \
+			echo "exec \"\$$GIT_ANNEX_LINKER\" --library-path \"\$$GIT_ANNEX_LD_LIBRARY_PATH\" \"\$$GIT_ANNEX_SHIMMED/$$(basename "$$file")\"" >> "$$file"; \
+			chmod +x "$$file"; \
+		fi; \
+	done
+
 	$(MAKE) install-mans DESTDIR="$(LINUXSTANDALONE_DEST)"
 
 	cd tmp/git-annex.linux && find . -type f > git-annex.MANIFEST

debian/changelog

@@ -8,6 +8,8 @@ git-annex (5.20131214) UNRELEASED; urgency=low
     into a directory in PATH as an alternative installation method, and will
     use readlink to find where the build was unpacked.
   * Include man pages in Linux and OSX standalone builds.
+  * Linux standalone build now includes its own glibc and forces the linker to
+    use it, to remove dependence on the host glibc.
 
  -- Joey Hess <joeyh@debian.org>  Sun, 15 Dec 2013 13:32:49 -0400
 

doc/install/Linux_standalone.mdwn

@@ -2,8 +2,8 @@ If your Linux distribution does not have git-annex packaged up for you,
 you can either build it [[fromscratch]], or you can use a handy
 prebuilt tarball of the most recent release.
 
-This tarball should work on most Linux systems. It does not depend
-on anything except for glibc.
+This tarball should work on most Linux systems. It has basically no
+dependencies and is self-contained.
 
 [download tarball](https://downloads.kitenet.net/git-annex/linux/current/)
 

standalone/linux/README

@@ -1,4 +1,5 @@
-You can put this directory into your PATH, and use git-annex the same
+You can put this directory into your PATH, or symlink the programs in this
+directory to anyplace already in your PATH, and use git-annex the same
 as if you'd installed it using a package manager.
 
 Or, you can use the runshell script in this directory to start a shell
@@ -6,20 +7,14 @@ that is configured to use git-annex and the other utilities included in
 this bundle, including git, gpg, rsync, ssh, etc.
 
 This should work on any Linux system of the appropriate architecture.
-More or less. There are no external dependencies, except for glibc.
-Any recent-ish version of glibc should work (2.13 is ok; so is 2.11).
+More or less.
 
 
 How it works: This directory tree contains a lot of libraries and programs
-that git-annex needs. But it's not a chroot. Instead, runshell sets PATH
-and LD_LIBRARY_PATH to point to the stuff in here.
+that git-annex needs. But it's not a chroot. Instead, runshell sets a lot
+of environment variables to cause files from here to be used, and a shim
+around the binaries arranges for them to be run with the libraries in here.
 
-The glibc libs are not included. Instead, it runs with the host system's
-glibc. We trust that glibc's excellent backwards and forward compatability
-is good enough to run binaries that were linked for a newer or older
-version. Of course, this could fail. Particularly if the binaries try to
-use some new glibc feature. But hopefully not.
-
-Why not bundle glibc too? I've not gotten it to work! The host system's 
-ld-linux.so will be used for sure, as that's hardcoded into the binaries.
-When I tried including libraries from glibc in here, everything segfaulted.
+It shouldn't even be dependent on the host system's glibc libraries.
+All that's needed is a kernel that supports the glibc included in this
+bundle.

standalone/linux/glibc-libs

@@ -1,45 +0,0 @@
-libanl-.*.so
-libutil-.*.so
-libnss_hesiod-.*.so
-libcrypt-.*.so
-libnss_compat-.*.so
-libm-.*.so
-libr.so
-libpcprofile.so
-libnss_nis-.*.so
-libSegFault.so
-libpthread-.*.so
-librt-.*.so
-libnss_dns-.*.so
-libdl-.*.so
-libBrokenLocale-.*.so
-libnss_nisplus-.*.so
-libthread_db-1.0.so
-libmemusage.so
-libcidn-.*.so
-libnss_files-.*.so
-libnsl-.*.so
-libc-.*.so
-ld-.*.so
-libnss_nis.so
-libthread_db.so
-libanl.so
-libr.so
-libnss_compat.so
-libm.so
-libnss_dns.so
-libpthread.so
-libc.so
-librt.so
-libcidn.so
-libnss_nisplus.so
-libnsl.so
-libutil.so
-libBrokenLocale.so
-ld-linux.so
-libnss_files.so
-libdl.so
-libnss_hesiod.so
-libcrypt.so
-linux-vdso.so
-linux-gate.so

standalone/linux/runshell

@@ -46,12 +46,20 @@ export ORIG_PATH
 PATH=$base/bin:$PATH
 export PATH
 
-ORIG_LD_LIBRARY_PATH="$LD_LIBRARY_PATH"
-export ORIG_LD_LIBRARY_PATH
+# This is used by the shim wrapper around each binary.
 for lib in $(cat $base/libdirs); do
-	LD_LIBRARY_PATH="$base/$lib:$LD_LIBRARY_PATH"
+	GIT_ANNEX_LD_LIBRARY_PATH="$base/$lib:$GIT_ANNEX_LD_LIBRARY_PATH"
 done
-export LD_LIBRARY_PATH
+export GIT_ANNEX_LD_LIBRARY_PATH
+GIT_ANNEX_LINKER=$(cat $base/linker)
+export GIT_ANNEX_LINKER
+GIT_ANNEX_SHIMMED="$base/shimmed"
+export GIT_ANNEX_SHIMMED
+
+ORIG_GCONV_PATH="$GCONV_PATH"
+export ORIG_GCONV_PATH
+GCONV_PATH=$base/$(cat $base/gconvdir)
+export GCONV_PATH
 
 ORIG_GIT_EXEC_PATH="$GIT_EXEC_PATH"
 export ORIG_GIT_EXEC_PATH
@@ -68,8 +76,9 @@ export ORIG_MANPATH
 MANPATH="$base/usr/share/man:$MANPATH"
 export MANPATH
 
-# Indicate which variables were exported above.
-GIT_ANNEX_STANDLONE_ENV="PATH LD_LIBRARY_PATH GIT_EXEC_PATH GIT_TEMPLATE_DIR MANPATH"
+# Indicate which variables were exported above and should be cleaned
+# when running non-bundled programs.
+GIT_ANNEX_STANDLONE_ENV="PATH GCONV_PATH GIT_EXEC_PATH GIT_TEMPLATE_DIR MANPATH"
 export GIT_ANNEX_STANDLONE_ENV
 
 if [ "$1" ]; then