diff --git a/doc/internals/git-remote-annex.mdwn b/doc/internals/git-remote-annex.mdwn index 0334e53edb..0e916c99ff 100644 --- a/doc/internals/git-remote-annex.mdwn +++ b/doc/internals/git-remote-annex.mdwn @@ -1,6 +1,6 @@ This adds two new object types to git-annex, GITMANIFEST and a GITBUNDLE. -GITMANIFEST-$UUID is the manifest for a git repository stored in the +GITMANIFEST--$UUID is the manifest for a git repository stored in the git-annex repository with that UUID. GITBUNDLE--sha256 is a git bundle. @@ -9,16 +9,21 @@ GITBUNDLE--sha256 is a git bundle. An ordered list of bundle keys, one per line. +The last bundle in the list provides all refs that are currently stored in +the repository. The bundles before it in the list can incrementally provide +objects, but not refs. + # fetching 1. download GITMANIFEST for the uuid of the special remote 2. download each listed GITBUNDLE object that we don't have -3. git fetch from bundles in timestamp order +3. `git bundle unpack` each bundle in order +4. `git fetch` from the last bundle listed in the manifest # pushing (incrementally) -1. create git bundle containing refs to push, and objects since - the previously pushed refs +1. create git bundle all refs that will be stored in the repository, + and objects since the previously pushed refs 2. hash to calculate GITBUNDLE key 3. upload GITBUNDLE object 4. download current manifest @@ -26,7 +31,8 @@ An ordered list of bundle keys, one per line. # pushing (replacing incrementals with single bundle) -1. create git bundle containing refs to push and all objects +1. create git bundle containing all refs stored in the repository, and all + objects 2. hash to calculate GITBUNDLE object name 3. upload GITBUNDLE object 4. download current manifest diff --git a/git-remote-annex b/git-remote-annex new file mode 100755 index 0000000000..6171e9d7b2 --- /dev/null +++ b/git-remote-annex @@ -0,0 +1,152 @@ +#!/bin/sh + +set -x + +# remember the refs that were uploaded already +git for-each-ref refs/namespaces/mine/ > .git/old-refs + +# Unfortunately, git bundle omits prerequisites that are omitted once, +# even if they are used by a later ref. +# For example, where x is a ref that points at A, and y is a ref +# that points at B (which has A as its parent), git bundle x A..y +# will omit inclding the x ref in the bundle at all. +check_prereq () { + # So, if a sha is one of the other refs that will be included in the + # bundle, it cannot be treated as a prerequisite. + if git for-each-ref refs/namespaces/mine/ | grep -Pv "\t$2$" | awk '{print $1}' | grep -q "$1"; then + echo "$2" + else + # And, if one of the other refs that will be included in the bundle + # is an ancestor of the sha, it cannot be treated as a prerequisite. + if [ -n "$(for x in $(git for-each-ref refs/namespaces/mine/ | grep -Pv "\t$2$" | awk '{print $1}'); do git log --oneline -n1 $x..$1; done)" ]; then + echo "$2" + else + echo "$1..$2" + fi + fi +} + +while read foo; do + case "$foo" in + capabilities) + echo fetch + echo push + echo + ;; + list*) + if [ -e "MANIFEST" ]; then + # Only list the refs in the last bundle + # listed in the manifest. Each push + # includes all refs in its bundle. + f=$(tail -n 1 MANIFEST) + if [ -n "$f" ]; then + # refs in the bundle may end up prefixed with refs/namespaces/mine/ + # when the intent is for the bundle to include a + # ref with the name that comes after that. + git bundle list-heads $f | sed 's/refs\/namespaces\/mine\///' + fi + fi + echo + ;; + fetch*) + dofetch=1 + ;; + push*) + set -- $foo + x="$2" + # src ref if prefixed with a + in a forced push + srcref="$(echo "$x" | cut -d : -f 1 | sed 's/^\+//')" + dstref="$(echo "$x" | cut -d : -f 2)" + if [ -z "$srcref" ]; then + git update-ref -d refs/namespaces/mine/"$dstref" + else + # Need to create a bundle containing $dstref, but + # don't want to overwrite that ref in the local + # repo. Unfortunately, git bundle does not support + # GIT_NAMESPACE, so it's not possible to do that + # without making a clone of the whole git repo. + # Instead, just create a ref under the namespace + # refs/namespaces/mine/ that will be put in the + # bundle. + git update-ref refs/namespaces/mine/"$dstref" "$srcref" + fi + dopush=1 + ;; + # docs say a blank line ends communication, but that's not + # accurate, actually a blank line comes after a series of + # fetch or push commands, and also according to the docs, + # another series of commands could follow + "") + if [ "$dofetch" ]; then + if [ -e "MANIFEST" ]; then + for f in $(cat MANIFEST); do + git bundle unbundle "$f" >/dev/null 2>&1 + done + fi + echo + dofetch="" + fi + if [ "$dopush" ]; then + if [ -z "$(git for-each-ref refs/namespaces/mine/)" ]; then + # deleted all refs + if [ -e "MANIFEST" ]; then + for f in $(cat MANIFEST); do + rm "$f" + done + rm MANIFEST + touch MANIFEST + fi + else + # set REPUSH=1 to do a full push + # rather than incremental + if [ "$REPUSH" ]; then + rm MANIFEST + rm *.bundle + git for-each-ref refs/namespaces/mine/ | awk '{print $3}' | \ + git bundle create --quiet new.bundle --stdin + else + # incremental bundle + IFS=" +" + (for l in $(git for-each-ref refs/namespaces/mine/); do + r=$(echo "$l" | awk '{print $3}') + newsha=$(echo "$l" | awk '{print $1}') + oldsha=$(grep -P "\t$r$" .git/old-refs | awk '{print $1}') + if [ -n "$oldsha" ]; then + # include changes from $oldsha to $r when there are some + if [ -n "$(git log --oneline $oldsha..$r)" ]; then + check_prereq "$oldsha" "$r" + else + if [ "$oldsha" = "$newsha" ]; then + # $r is unchanged from last push, so include + # the minimum data to make the bundle contain $r + rparentsha=$(git log -n 2 "$r" --format='%H' | tail -n+2) + if [ -n "$rparentsha" ]; then + check_prereq "$rparentsha" "$r" + else + # $r has no parent so include it as is + echo "$r" + fi + else + # $oldsha is not a parent of $r, so + # include $r and all its parents + echo "$r" + fi + fi + else + # no old version was pushed so include $r and all its parents + echo "$r" + fi + done) \ + | git bundle create --quiet new.bundle --stdin + fi + sha1=$(sha1sum new.bundle | awk '{print $1}') + mv new.bundle "$sha1.bundle" + echo "$sha1.bundle" >> MANIFEST + fi + echo + dopush="" + fi + ;; + esac +done