Add source information
This commit is contained in:
commit
0f51180097
GPG key ID:
D62A472A4AA7D541
2 changed files with 242 additions and 0 deletions
32
README.md
Normal file
32
README.md
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
# annexgetfile
|
||||
Upstream: https://ayakael.net/forge/annexgetfile
|
||||
|
||||
## Description
|
||||
|
||||
Script to extract name and location information out of git-annex repo
|
||||
|
||||
Based on: https://gist.github.com/eruffaldi/924f6b53a63dede6e59f
|
||||
|
||||
## How to use
|
||||
|
||||
|
||||
```shell
|
||||
usage: annexgetfile.py [-h] [--annex ANNEX] [--all] [--verbose] [--tar TAR] [--abs] [path ...]
|
||||
|
||||
Retrieve file from git-annex, even barebone
|
||||
|
||||
positional arguments:
|
||||
path file to be looked at
|
||||
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
--annex ANNEX path to annex repository
|
||||
--all list all
|
||||
--verbose verbose dump
|
||||
--tar TAR produces a tar file with given path cotaining the symbolic links
|
||||
--abs makes abs files
|
||||
|
||||
```
|
||||
|
||||
## License
|
||||
This readme, abuilds and support scripts are licensed under MIT License.
|
||||
210
annexgetfile.py
Normal file
210
annexgetfile.py
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
#! /usr/bin/env python3
|
||||
|
||||
#
|
||||
# Extract files from Bare git-annex repositories without git-annex
|
||||
# Supports version v6
|
||||
#
|
||||
# See internals: http://git-annex.branchable.com/internals/
|
||||
#
|
||||
# Modified: added non-bare repos, added tar file (of symlinks) output for use with archivemount
|
||||
#
|
||||
# TODO: improve output
|
||||
# TODO: use cat-files instead of archive
|
||||
# TODO: export to tar WITH relative links
|
||||
#
|
||||
# Emanuele Ruffaldi (C) 2016
|
||||
import sys,argparse,os,subprocess
|
||||
import tarfile,hashlib,struct
|
||||
import tarfile,io,hashlib,struct
|
||||
from hashlib import md5
|
||||
|
||||
def gitgetpathinfo(branch,path,recurse=False):
|
||||
"""uses ls-tree to extract information about a path in the branch or in general tree-ish"""
|
||||
if recurse:
|
||||
r = "-r"
|
||||
else:
|
||||
r = ""
|
||||
w = subprocess.check_output(["git", "ls-tree",r,branch,"--",path])
|
||||
w = w.decode("utf-8")
|
||||
return [pa.split("\t") for pa in w.split("\n") if pa != ""] # meta TAB filename ==> meta is: ?? SPACE type
|
||||
|
||||
def tarextraclink(content):
|
||||
"""extracts the path of a link in a Tar expressed by content"""
|
||||
t = tarfile.open(mode="r",fileobj=io.BytesIO(content))
|
||||
ti = t.getmembers()[0]
|
||||
return ti.linkname
|
||||
|
||||
def gitgetfile(branch,path):
|
||||
"""uses archive for extracing the path. This is better than the git show solution because it deals with diff automatically. But does not work with symbolic links"""
|
||||
xpath,n = os.path.split(path)
|
||||
xx = "git archive --format=tar --prefix= \"%s:%s\" \"%s\" | tar -xO \"%s\"" % (branch,xpath,n,n)
|
||||
return subprocess.check_output(xx,shell=True)
|
||||
|
||||
def gitgetfile_tar(branch,path):
|
||||
"""returns the content of a file in tar format"""
|
||||
try:
|
||||
xpath,n = os.path.split(path)
|
||||
xx = "git archive --format=tar --prefix= \"%s:%s\" \"%s\"" % (branch,xpath,n)
|
||||
return(subprocess.check_output(xx,shell="True"))
|
||||
except:
|
||||
return None
|
||||
|
||||
def gitgetfile_show(branch,path):
|
||||
"""retrieve path content: first getting the hash and then the content via git show"""
|
||||
found = gitgetpathinfo(branch,path)
|
||||
if len(found) == 1:
|
||||
return subprocess.check_output(["git", "show",found[0][0].split(" ")[2]])
|
||||
else:
|
||||
return None
|
||||
|
||||
def annexgetremotes(useshow):
|
||||
"""list of remotes AKA uuid.log"""
|
||||
if useshow:
|
||||
return gitgetfile_show("git-annex","uuid.log")
|
||||
else: # slow with bare
|
||||
return gitgetfile("git-annex","uuid.log")
|
||||
|
||||
#https://gist.github.com/giomasce/a7802bda1417521c5b30
|
||||
def hashdirlower(key):
|
||||
hasher = hashlib.md5()
|
||||
hasher.update(key.encode("utf-8"))
|
||||
digest = hasher.hexdigest()
|
||||
return "%s/%s/" % (digest[:3], digest[3:6])
|
||||
|
||||
#https://gist.github.com/giomasce/a7802bda1417521c5b30
|
||||
def hashdirmixed(key):
|
||||
hasher = hashlib.md5()
|
||||
hasher.update(key)
|
||||
digest = hasher.digest()
|
||||
first_word = struct.unpack('<I', digest[:4])[0]
|
||||
nums = [first_word >> (6 * x) & 31 for x in range(4)]
|
||||
letters = ["0123456789zqjxkmvwgpfZQJXKMVWGPF"[i] for i in nums]
|
||||
return "%s%s/%s%s/" % (letters[1], letters[0], letters[3], letters[2])
|
||||
|
||||
def annexwhereis_bare(key):
|
||||
"""returns the location of the key object of git-annex"""
|
||||
#hashdirlower is used for bare git repositories, the git-annex branch, and on special remotes as well.
|
||||
#m = md5.new()
|
||||
#m.update(key)
|
||||
#h = m.hexdigest()
|
||||
#pre = h[0:3]
|
||||
#post = h[3:6]
|
||||
#print key,pre,post
|
||||
papa = hashdirlower(key)
|
||||
return gitgetfile("git-annex",os.path.join(papa,key+".log")),os.path.join("annex","objects",papa,key,key)
|
||||
|
||||
def annexwhereis(key):
|
||||
"""returns the location of the key object of git-annex"""
|
||||
#non bare uses hashdirmixed
|
||||
#It takes the md5sum of the key, but rather than a string, represents it as 4 32bit words. Only the first word is used. It is converted into a string by the same mechanism that would be used to encode a normal md5sum value into a string, but where that would normally encode the bits using the 16 characters 0-9a-f, this instead uses the 32 characters "0123456789zqjxkmvwgpfZQJXKMVWGPF". The first 2 letters of the resulting string are the first directory, and the second 2 are the second directory.
|
||||
papaM = hashdirmixed(key)
|
||||
papaL = hashdirlower(key)
|
||||
return gitgetfile("git-annex",os.path.join(papaL,key+".log")),os.path.join("annex","objects",papaM,key,key)
|
||||
|
||||
def checkbare(args):
|
||||
"""checks if the repo is a bare"""
|
||||
gitdir = os.path.join(args.annex,".git")
|
||||
if os.path.isdir(gitdir):
|
||||
if not os.path.isdir(os.path.join(gitdir,"annex")):
|
||||
return None
|
||||
else:
|
||||
return False,gitdir
|
||||
elif os.path.isdir(os.path.join(args.annex,"annex")):
|
||||
gitdir = args.annex
|
||||
return True,gitdir
|
||||
else:
|
||||
return None
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(description='Retrieve file from git-annex, even barebone')
|
||||
parser.add_argument('--annex', help="path to annex repository",default=".")
|
||||
parser.add_argument('path', help="file to be looked at",nargs="*")
|
||||
parser.add_argument('--all', help="list all",action="store_true")
|
||||
parser.add_argument('--verbose', help="verbose dump",action="store_true")
|
||||
parser.add_argument('--tar', help="produces a tar file with given path cotaining the symbolic links")
|
||||
parser.add_argument('--abs',help="makes abs files",action="store_true")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# check if bare repository
|
||||
isbare = checkbare(args)
|
||||
if isbare is None:
|
||||
print("not a git-annex repisitory")
|
||||
isbare,gitdir = isbare
|
||||
print("isbare?",isbare,gitdir)
|
||||
|
||||
if not isbare:
|
||||
workdir = args.annex
|
||||
else:
|
||||
workdir = None
|
||||
|
||||
os.environ["GIT_DIR"] = gitdir
|
||||
print("list annexes\n",annexgetremotes(useshow=False))
|
||||
|
||||
if args.tar:
|
||||
ot = tarfile.open(args.tar,"w")
|
||||
|
||||
if args.all:
|
||||
args.path = [x[1] for x in gitgetpathinfo("edge",".",recurse=True)]
|
||||
|
||||
for p in args.path:
|
||||
# we cannot use
|
||||
ww = gitgetfile_tar("edge",p) # tarred 1 file
|
||||
if ww is None:
|
||||
print("not found",p)
|
||||
continue
|
||||
link = tarextraclink(ww) # extract the link from the single file
|
||||
if args.verbose:
|
||||
print("aslink",link)
|
||||
#w = gitgetfile("edge",p) -- not working using tar because it is a link
|
||||
#ref = gitgetfile_show("edge",p) -- not working in theory
|
||||
ref = link
|
||||
if ref == "":
|
||||
print("not found",p)
|
||||
else:
|
||||
key = os.path.split(ref)[1] # the link contains the annex key
|
||||
if args.verbose:
|
||||
print("key is",key)
|
||||
if isbare:
|
||||
locations,path = annexwhereis_bare(key) # extract
|
||||
else:
|
||||
locations,path = annexwhereis(key)
|
||||
path = os.path.join(gitdir,path)
|
||||
|
||||
if args.verbose:
|
||||
print(p,"located in\n",locations)
|
||||
if not os.path.isfile(path):
|
||||
if not isbare:
|
||||
if os.path.isfile(path+".map"):
|
||||
mpath = os.path.join(workdir,open(path+".map","r").read().strip())
|
||||
if os.path.isfile(mpath):
|
||||
path = mpath
|
||||
else:
|
||||
print("mapped file not found",mpath," for ",path) # or direct mode not supported
|
||||
path = None
|
||||
else:
|
||||
print("non bare file not found",path) # or direct mode not supported
|
||||
path = None
|
||||
else:
|
||||
print("file not found",path) # or direct mode not supported
|
||||
path = None
|
||||
if path is not None:
|
||||
ss = os.stat(path)
|
||||
if args.verbose:
|
||||
print(path,ss)
|
||||
ti = tarfile.TarInfo(p)
|
||||
ti.size = 0 # zero for links: ss.st_size
|
||||
ti.mode = ss.st_mode
|
||||
ti.mtime = ss.st_mtime
|
||||
ti.type = tarfile.SYMTYPE
|
||||
ti.uid = ss.st_uid
|
||||
ti.gid = ss.st_gid
|
||||
if args.abs:
|
||||
ti.linkname = os.path.abspath(path)
|
||||
else:
|
||||
ti.linkname = path
|
||||
ot.addfile(ti)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in a new issue