Merge branch 'master' into v7-default
This commit is contained in:
Joey Hess
commit
061231621e
43 changed files with 788 additions and 17 deletions
|
|
@ -155,7 +155,7 @@ isInitialized = maybe Annex.Branch.hasSibling (const $ return True) =<< getVersi
|
|||
{- A crippled filesystem is one that does not allow making symlinks,
|
||||
- or removing write access from files. -}
|
||||
probeCrippledFileSystem :: Annex Bool
|
||||
probeCrippledFileSystem = withOtherTmp $ \tmp -> do
|
||||
probeCrippledFileSystem = withEventuallyCleanedOtherTmp $ \tmp -> do
|
||||
(r, warnings) <- liftIO $ probeCrippledFileSystem' tmp
|
||||
mapM_ warning warnings
|
||||
return r
|
||||
|
|
@ -212,7 +212,7 @@ probeLockSupport = do
|
|||
#ifdef mingw32_HOST_OS
|
||||
return True
|
||||
#else
|
||||
withOtherTmp $ \tmp -> do
|
||||
withEventuallyCleanedOtherTmp $ \tmp -> do
|
||||
let f = tmp </> "lockprobe"
|
||||
mode <- annexFileMode
|
||||
liftIO $ do
|
||||
|
|
@ -230,7 +230,7 @@ probeFifoSupport = do
|
|||
#ifdef mingw32_HOST_OS
|
||||
return False
|
||||
#else
|
||||
withOtherTmp $ \tmp -> do
|
||||
withEventuallyCleanedOtherTmp $ \tmp -> do
|
||||
let f = tmp </> "gaprobe"
|
||||
let f2 = tmp </> "gaprobe2"
|
||||
liftIO $ do
|
||||
|
|
|
|||
|
|
@ -266,7 +266,8 @@ gitAnnexTmpOtherDir r = addTrailingPathSeparator $ gitAnnexDir r </> "othertmp"
|
|||
gitAnnexTmpOtherLock :: Git.Repo -> FilePath
|
||||
gitAnnexTmpOtherLock r = gitAnnexDir r </> "othertmp.lck"
|
||||
|
||||
{- Tmp directory used by old versions of git-annex. -}
|
||||
{- .git/annex/misctmp/ was used by old versions of git-annex and is still
|
||||
- used during initialization -}
|
||||
gitAnnexTmpOtherDirOld :: Git.Repo -> FilePath
|
||||
gitAnnexTmpOtherDirOld r = addTrailingPathSeparator $ gitAnnexDir r </> "misctmp"
|
||||
|
||||
|
|
|
|||
26
Annex/Tmp.hs
26
Annex/Tmp.hs
|
|
@ -7,9 +7,8 @@
|
|||
|
||||
module Annex.Tmp where
|
||||
|
||||
import Common
|
||||
import Annex
|
||||
import Annex.Locations
|
||||
import Annex.Common
|
||||
import qualified Annex
|
||||
import Annex.LockFile
|
||||
import Annex.Perms
|
||||
import Types.CleanupActions
|
||||
|
|
@ -24,13 +23,30 @@ import Data.Time.Clock.POSIX
|
|||
-- any time.
|
||||
withOtherTmp :: (FilePath -> Annex a) -> Annex a
|
||||
withOtherTmp a = do
|
||||
addCleanup OtherTmpCleanup cleanupOtherTmp
|
||||
Annex.addCleanup OtherTmpCleanup cleanupOtherTmp
|
||||
tmpdir <- fromRepo gitAnnexTmpOtherDir
|
||||
tmplck <- fromRepo gitAnnexTmpOtherLock
|
||||
withSharedLock (const tmplck) $ do
|
||||
void $ createAnnexDirectory tmpdir
|
||||
a tmpdir
|
||||
|
||||
-- | This uses an alternate temp directory. The action should normally
|
||||
-- clean up whatever files it writes there, but if it leaves files
|
||||
-- there (perhaps due to being interrupted), the files will be eventually
|
||||
-- cleaned up by another git-annex process (after they're a week old).
|
||||
--
|
||||
-- Unlike withOtherTmp, this does not rely on locking working.
|
||||
-- Its main use is in situations where the state of lockfile is not
|
||||
-- determined yet, eg during initialization.
|
||||
withEventuallyCleanedOtherTmp :: (FilePath -> Annex a) -> Annex a
|
||||
withEventuallyCleanedOtherTmp = bracket setup cleanup
|
||||
where
|
||||
setup = do
|
||||
tmpdir <- fromRepo gitAnnexTmpOtherDirOld
|
||||
void $ createAnnexDirectory tmpdir
|
||||
return tmpdir
|
||||
cleanup = liftIO . void . tryIO . removeDirectory
|
||||
|
||||
-- | Cleans up any tmp files that were left by a previous
|
||||
-- git-annex process that got interrupted or failed to clean up after
|
||||
-- itself for some other reason.
|
||||
|
|
@ -42,8 +58,6 @@ cleanupOtherTmp = do
|
|||
void $ tryIO $ tryExclusiveLock (const tmplck) $ do
|
||||
tmpdir <- fromRepo gitAnnexTmpOtherDir
|
||||
void $ liftIO $ tryIO $ removeDirectoryRecursive tmpdir
|
||||
-- This is only to clean up cruft left by old versions of
|
||||
-- git-annex; it can be removed eventually.
|
||||
oldtmp <- fromRepo gitAnnexTmpOtherDirOld
|
||||
liftIO $ mapM_ cleanold =<< dirContentsRecursive oldtmp
|
||||
liftIO $ void $ tryIO $ removeDirectory oldtmp -- when empty
|
||||
|
|
|
|||
|
|
@ -23,11 +23,14 @@ git-annex (7.20190826) UNRELEASED; urgency=medium
|
|||
* info: When file matching options are specified when getting
|
||||
info of something other than a directory, they won't have any effect,
|
||||
so error out to avoid confusion.
|
||||
* info: Display trust level when getting info on a uuid, same as on a remote.
|
||||
* When upgrading a direct mode repo to v7 with adjusted unlocked branches,
|
||||
fix a bug that prevented annex.thin from taking effect for the files
|
||||
in working tree.
|
||||
* Avoid making a commit when upgrading from direct mode to v7.
|
||||
* init: Catch more exceptions when testing locking.
|
||||
* init: Fix a reversion that broke initialization on systems that
|
||||
need to use pid locking.
|
||||
|
||||
-- Joey Hess <id@joeyh.name> Sat, 24 Aug 2019 12:54:35 -0400
|
||||
|
||||
|
|
|
|||
|
|
@ -271,7 +271,6 @@ file_stats f k =
|
|||
remote_fast_stats :: Remote -> [Stat]
|
||||
remote_fast_stats r = map (\s -> s r)
|
||||
[ remote_name
|
||||
, remote_trust
|
||||
, remote_cost
|
||||
, remote_type
|
||||
]
|
||||
|
|
@ -280,6 +279,7 @@ uuid_fast_stats :: UUID -> [Stat]
|
|||
uuid_fast_stats u = map (\s -> s u)
|
||||
[ repo_uuid
|
||||
, repo_description
|
||||
, repo_trust
|
||||
]
|
||||
|
||||
uuid_slow_stats :: UUID -> [Stat]
|
||||
|
|
@ -347,9 +347,8 @@ repo_description = simpleStat "description" . lift . Remote.prettyUUID
|
|||
repo_uuid :: UUID -> Stat
|
||||
repo_uuid = simpleStat "uuid" . pure . fromUUID
|
||||
|
||||
remote_trust :: Remote -> Stat
|
||||
remote_trust r = simpleStat "trust" $ lift $
|
||||
showTrustLevel <$> lookupTrust (Remote.uuid r)
|
||||
repo_trust :: UUID -> Stat
|
||||
repo_trust u = simpleStat "trust" $ lift $ showTrustLevel <$> lookupTrust u
|
||||
|
||||
remote_cost :: Remote -> Stat
|
||||
remote_cost r = simpleStat "cost" $ pure $
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
I'm running three repositories (laptop, usb connected to laptop, local server) that are connected via ssh access.
|
||||
Running the webapp on both laptop and server, I'm quite frequently seeing in both webapps:
|
||||
|
||||
warning
|
||||
Pusher crashed: fd:56: hPutStr: illegal operation (handle is closed)
|
||||
|
||||
Pushing the "Restart thread" button sometimes dismisses the warning, sometimes it immediatly comes back even when trying several times.
|
||||
A similar warning sometimes shows up for a Netwatcher thread.
|
||||
|
||||
Not sure but when that crash occurs tranfers between the repos seem to get stalled.
|
||||
|
||||
Server: git-annex 7.20190129-2~bpo9+1 on debian stretch
|
||||
Laptop: git-annex 7.20190129-3 on debian testing
|
||||
|
||||
|
||||
Log snippet:
|
||||
|
||||
[[!format sh """
|
||||
[2019-09-07 14:10:30.330041206] Committer: Adding jmg_0578.jpg5895 jmg_0578.jpg
|
||||
add incoming/5d/jmg_0578.jpg [2019-09-07 14:10:30.358484399] Committer: Committing changes to git
|
||||
(recording state in git...)
|
||||
[2019-09-07 14:13:01.986780732] Pusher: Syncing with usbimg, cube_media_srv_img
|
||||
Pusher crashed: fd:56: hPutStr: illegal operation (handle is closed)
|
||||
[2019-09-07 14:13:02.009823288] Pusher: warning Pusher crashed: fd:56: hPutStr: illegal operation (handle is closed)
|
||||
[2019-09-07 14:13:49.112731437] RemoteControl: Syncing with cube_media_srv_img
|
||||
(merging synced/git-annex into git-annex...)
|
||||
fd:56: hPutStr: illegal operation (handle is closed)
|
||||
fd:56: hPutStr: illegal operation (handle is closed)
|
||||
From ssh://git-annex-cube-flixh_22_.2Fmedia.2Fsrv.2Fimg/media/srv/img
|
||||
3c9104b770..cea5c2f058 annex/direct/master -> cube_media_srv_img/annex/direct/master
|
||||
71006e08a7..536584bdfa git-annex -> cube_media_srv_img/git-annex
|
||||
fd:56: hPutStr: illegal operation (handle is closed)
|
||||
[2019-09-07 14:15:58.002820518] Committer: Adding jmg_0592...ftool_tmp
|
||||
[2019-09-07 14:15:59.526375687] Committer: Adding jmg_0592...ftool_tmp jmg_0592.cr2
|
||||
add incoming/5d/jmg_0592.cr2 [2019-09-07 14:15:59.599057025] Committer: Committing changes to git
|
||||
(recording state in git...)
|
||||
[2019-09-07 14:15:59.670107852] Pusher: Syncing with usbimg, cube_media_srv_img
|
||||
Pusher crashed: fd:56: hPutStr: illegal operation (handle is closed)
|
||||
[2019-09-07 14:15:59.683904185] Pusher: warning Pusher crashed: fd:56: hPutStr: illegal operation (handle is closed)
|
||||
[2019-09-07 14:18:09.282924907] Committer: Adding CI1026+1375.JPG5895
|
||||
|
||||
"""]]
|
||||
|
|
@ -0,0 +1 @@
|
|||
I can't addurl a file because there is a special remote which is not available. I can see that git annex is asking it if it wants to claim the url which it doesn't. I don't feel addurl should depend on the remote after CLAIMURL-FAILURE. Or do I miss something?
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="Ilya_Shlyakhter"
|
||||
avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
|
||||
subject="comment 1"
|
||||
date="2019-09-02T15:49:57Z"
|
||||
content="""
|
||||
Post the trace (with `git annex --debug --verbose`) and your config files (`git config --list`)? (Redact any sensitive info). Maybe, `annex.security.allowed-ip-addresses` excludes the IP address of the URL, so the built-in web remote never gets a chance to claim the URL? I recall running into something like you describe, and the issue was with some config setting I had.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
[[!comment format=mdwn
|
||||
username="Ilya_Shlyakhter"
|
||||
avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
|
||||
subject="another trace of concurrent copy failure"
|
||||
date="2019-09-05T06:09:13Z"
|
||||
content="""
|
||||
Some more concurrent copy failures when non-concurrent works:
|
||||
|
||||
[[!format sh \"\"\"
|
||||
(master_env_v156_py36) 23:35 [viral-ngs-benchmarks] $ git annex copy -J16 --to viral-ngs-benchmarks-s3 --not --in=viral-ngs-benchmark\
|
||||
s-s3 --and --not --in=mygs --and --not --in=dnanexus
|
||||
copy benchmarks/intermed/analysis-FBGGXX00YJ80114YGQ0yKzFG/benchmark_variants/v1.23.0_spades-3.13.0/files/calls/assemble_denovo/assemb\
|
||||
le/0/outputs/subsampBam/LASV_NGA_2016_0014.ll2.subsamp.bam (checking viral-ngs-benchmarks-s3...) (to viral-ngs-benchmarks-s3...) ok
|
||||
copy benchmarks/intermed/analysis-FBGGXX00YJ80114YGQ0yKzFG/benchmark_variants/v1.23.0_spades-3.13.0/files/calls/assemble_denovo/refine\
|
||||
_2x_and_plot/0/outputs/aligned_bam_idx/LASV_NGA_2016_0014.ll2.all.bai (checking viral-ngs-benchmarks-s3...) (to viral-ngs-benchmarks-s\
|
||||
3...) ok
|
||||
copy benchmarks/intermed/analysis-FBGGXX00YJ80114YGQ0yKzFG/benchmark_variants/v1.23.0_spades-3.13.0/files/calls/assemble_denovo/refine\
|
||||
_2x_and_plot/0/outputs/aligned_only_reads_bam/LASV_NGA_2016_0014.ll2.mapped.bam (checking viral-ngs-benchmarks-s3...) (to viral-ngs-be\
|
||||
nchmarks-s3...) ok
|
||||
...
|
||||
|
||||
copy benchmarks/intermed/analysis-FBGGXX00YJ80114YGQ0yKzFG/benchmark_variants/v1.23.0_spades-3.13.0/cromwell_submit_output.txt (checki\
|
||||
ng viral-ngs-benchmarks-s3...) (to viral-ngs-benchmarks-s3...)
|
||||
S3Error {s3StatusCode = Status {statusCode = 403, statusMessage = \"Forbidden\"}, s3ErrorCode = \"SignatureDoesNotMatch\", s3ErrorMessag\
|
||||
e = \"The request signature we calculated does not match the signature you provided. Check your key and signing method.\", s3ErrorResour\
|
||||
ce = Nothing, s3ErrorHostId = Just \"7uk2qBwWawhwYhROH7SzY+i3Y/49OQ9OiIT81eRkoEvZowo56xVkATE9qPU4Zs78x4C7gPsu744=\", s3ErrorAccessKeyId \
|
||||
= Just \"AKIAXXXXXXXXXXXXXXX\", s3ErrorStringToSign = Just \"PUT\n\nin\nThu, 05 Sep 2019 03:35:47 GMT\nx-amz-storage-class:STANDARD\n/vi\
|
||||
ral-ngs-benchmarks/MD5E-s628-S16777216-C1--065d8c01e5f7bd3b997c6d24109005c7.txt\", s3ErrorBucket = Nothing, s3ErrorEndpointRaw = Nothin\
|
||||
g, s3ErrorEndpoint = Nothing}
|
||||
ok
|
||||
|
||||
...
|
||||
copy benchmarks/intermed/analysis-FBGGXX00YJ80114YGQ0yKzFG/benchmark_variants/v1.23.0_spades/files/calls/assemble_denovo/refine_2x_and\
|
||||
_plot/0/outputs/aligned_only_reads_bam/LASV_NGA_2016_0014.ll2.mapped.bam (checking viral-ngs-benchmarks-s3...) (to viral-ngs-benchmark\
|
||||
s-s3...) ok
|
||||
copy benchmarks/intermed/analysis-FBGGXX00YJ80114YGQ0yKzFG/benchmark_variants/v1.23.0_spades/files/calls/assemble_denovo/assemble/0/st\
|
||||
dout/stdout (checking viral-ngs-benchmarks-s3...) (to viral-ngs-benchmarks-s3...) ok
|
||||
double free or corruption (out)
|
||||
error: git-annex died of signal 6
|
||||
(master_env_v156_py36) 23:35 [viral-ngs-benchmarks] $ git annex copy --to viral-ngs-benchmarks-s3 --not --in=viral-ngs-benchmarks-s3\
|
||||
--and --not --in=mygs --and --not --in=dnanexus
|
||||
...
|
||||
(recording state in git...)
|
||||
(master_env_v156_py36) 01:17 [viral-ngs-benchmarks] $ echo $?
|
||||
0
|
||||
|
||||
|
||||
|
||||
\"\"\"]]
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
### Please describe the problem.
|
||||
|
||||
It is probably not a regression (since happens with now elderly git-annex in debian sid), it could be something either changed on debian sid or on datalad side (originally ran into it while trying to build 0.11.7 package for debian sid) to make test directory name even more "tricky" than before.
|
||||
|
||||
Summary: `git annex init` fails even though `git init` (and other commands, not shown here) works just fine:
|
||||
|
||||
|
||||
[[!format sh """
|
||||
root@smaug:/tmp/datalad_temp_test_create_with_obscure_name18qdmqwg/ "';a&b&cΔЙקم๗あ `| 1# pwd
|
||||
/tmp/datalad_temp_test_create_with_obscure_name18qdmqwg/ "';a&b&cΔЙקم๗あ `| 1
|
||||
|
||||
# git init was ran before already
|
||||
root@smaug:/tmp/datalad_temp_test_create_with_obscure_name18qdmqwg/ "';a&b&cΔЙקم๗あ `| 1# git init
|
||||
Reinitialized existing Git repository in /tmp/datalad_temp_test_create_with_obscure_name18qdmqwg/ "';a&b&cΔЙקم๗あ `| 1/.git/
|
||||
|
||||
root@smaug:/tmp/datalad_temp_test_create_with_obscure_name18qdmqwg/ "';a&b&cΔЙקم๗あ `| 1# git annex init
|
||||
init fatal: Unable to create '/tmp/datalad_temp_test_create_with_obscure_name18qdmqwg/ "';a&b&c `| 1/.git/annex/index.lock': No such file or directory
|
||||
fatal: Unable to create '/tmp/datalad_temp_test_create_with_obscure_name18qdmqwg/ "';a&b&c `| 1/.git/annex/index.lock': No such file or directory
|
||||
|
||||
git-annex: failed to read sha from git write-tree
|
||||
CallStack (from HasCallStack):
|
||||
error, called at ./Git/Sha.hs:18:15 in main:Git.Sha
|
||||
failed
|
||||
git-annex: init: 1 failed
|
||||
|
||||
root@smaug:/tmp/datalad_temp_test_create_with_obscure_name18qdmqwg/ "';a&b&cΔЙקم๗あ `| 1# git annex version
|
||||
git-annex version: 7.20190129
|
||||
build flags: Assistant Webapp Pairing S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite
|
||||
dependency versions: aws-0.20 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.3 feed-1.0.0.0 ghc-8.4.4 http-client-0.5.13.1 persistent-sqlite-2.8.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0
|
||||
key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
|
||||
remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar hook external
|
||||
operating system: linux x86_64
|
||||
supported repository versions: 5 7
|
||||
upgrade supported from repository versions: 0 1 2 3 4 5 6
|
||||
|
||||
"""]]
|
||||
|
||||
so above version is the one in Debian proper (apparently quite dated!). I have tried with 7.20190819+git2-g908476a9b-1~ndall+1 and 7.20190819+git60-gcdb679818-1~ndall+1
|
||||
|
||||
You can see that `git-annex` got unicode portion of the directory name stripped (reports ``` "';a&b&c `| 1``` instead of ``` "';a&b&cΔЙקم๗あ `| 1```).
|
||||
|
||||
My wild guess is that full path (instead of a relative path) is used somewhere, encoded/decoded with losses, and then kaboom comes.
|
||||
|
||||
<details>
|
||||
<summary>locale is just C</summary>
|
||||
|
||||
[[!format sh """
|
||||
root@smaug:/tmp/datalad_temp_test_create_with_obscure_name18qdmqwg/ "';a&b&cΔЙקم๗あ `| 1# locale
|
||||
LANG=C
|
||||
LANGUAGE=
|
||||
LC_CTYPE="C"
|
||||
LC_NUMERIC="C"
|
||||
LC_TIME="C"
|
||||
LC_COLLATE="C"
|
||||
LC_MONETARY="C"
|
||||
LC_MESSAGES="C"
|
||||
LC_PAPER="C"
|
||||
LC_NAME="C"
|
||||
LC_ADDRESS="C"
|
||||
LC_TELEPHONE="C"
|
||||
LC_MEASUREMENT="C"
|
||||
LC_IDENTIFICATION="C"
|
||||
LC_ALL=C
|
||||
"""]]
|
||||
</details>
|
||||
|
||||
[[!meta author=yoh]]
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[[!comment format=mdwn
|
||||
username="yarikoptic"
|
||||
avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
|
||||
subject="archaeological expedition summary"
|
||||
date="2019-09-09T12:53:16Z"
|
||||
content="""
|
||||
just notes on the mystery on why didn't trigger before:
|
||||
- example shown above was triggered by a `test_create_with_obscure_name` which was added only recently in 0.11.7~13^2~2
|
||||
- (here I was going to provide analysis for other tests, but ran out of time... ;))
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 2"""
|
||||
date="2019-09-09T13:04:29Z"
|
||||
content="""
|
||||
To reproduce this, set LANG=C. In a unicode locale, it does not have the
|
||||
problem.
|
||||
|
||||
"קم๗あ" is a sufficient amount of unicode to cause the failure (so is "¡").
|
||||
The ascii punctuation is not involved, eg it works in a directory named
|
||||
"';a&b&c `| 1
|
||||
|
||||
joey@darkstar:/tmp/datalad_temp_test_create_with_obscure_name18qdmqwg/קم๗あ>LANG=C git-annex init
|
||||
init fatal: Unable to create '/tmp/datalad_temp_test_create_with_obscure_name18qdmqwg//.git/annex/index.lock': No such file or directory
|
||||
|
||||
Notice that all the unicode has been stripped out of the directory name
|
||||
somehow.
|
||||
|
||||
This is not happening to git-annex generally; .git/annex/ get created and
|
||||
populated with several files; it's the call to git update-index to create the
|
||||
annex index file that is failing.
|
||||
|
||||
Dumping the env that git-annex sets when running that command, it includes eg
|
||||
`("GIT_INDEX_FILE","/tmp/\56514\56481/.git/annex/index")`; the "¡"
|
||||
has been decomposed into what I think are two unicode surrigates; that's
|
||||
produced when getting the working directory.
|
||||
|
||||
joey@darkstar:/tmp/¡>LANG=C ghci
|
||||
GHCi, version 8.6.5: http://www.haskell.org/ghc/ :? for help
|
||||
Loaded GHCi configuration from /home/joey/.etc/.ghci
|
||||
ghci> import System.Posix.Directory
|
||||
ghci> getWorkingDirectory
|
||||
"/tmp/\56514\56481"
|
||||
|
||||
This shell command does not exhibit the problem:
|
||||
|
||||
printf '100644 8a1218a1024a212bb3db30becd860315f9f3ac52 1\tfrotz' | GIT_INDEX_FILE=`pwd`/.git/annex/index LANG=C git update-index --index-info
|
||||
|
||||
So, the problem is probably in the process library's passing
|
||||
of unicode environment variables to exec. I've verified the problem in ghci,
|
||||
running createProcess with this:
|
||||
|
||||
CreateProcess {cmdspec = RawCommand "git" ["update-index","--index-info"], cwd = Nothing, env = Just [("GIT_INDEX_FILE","/tmp/\56514\56481/.git/annex/index")], std_in = Inherit, std_out = Inherit, std_err = Inherit, close_fds = False, create_group = False, delegate_ctlc = False, detach_console = False, create_new_console = False, new_session = False, child_group = Nothing, child_user = Nothing, use_process_jobs = False}
|
||||
|
||||
This bug needs to be forwarded to process.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 3"""
|
||||
date="2019-09-09T18:41:05Z"
|
||||
content="""
|
||||
<https://github.com/haskell/process/issues/152> forwarded with a
|
||||
prospective patch.
|
||||
|
||||
----
|
||||
|
||||
Normally git-annex tries to use relative paths, and if it used a relative
|
||||
path to .git/annex/index, it would avoid this problem. Except perhaps
|
||||
if `GIT_DIR` pointed to a different directory with unicode in its path.
|
||||
A relative `GIT_INDEX_FILE` would not suffice to close this bug, but would
|
||||
fix your test suite. But, git has some very strange behavior with a relative
|
||||
`GIT_INDEX_FILE`, it's not necessarily treated as relative to pwd.
|
||||
So I'd rather not open that can of worms.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
### Please describe the problem.
|
||||
|
||||
`git-annex-copy --json --json-error-messages` writes errors messages to stdout, but not to stderr. This can be confusing in that the stderr output is empty, even though there were errors. It would be more intuitive to (also) output error messages as json records to stderr.
|
||||
|
||||
Also, the error messages that *are* written to stderr are not always in json format, e.g. the `git-annex: thread blocked indefinitely in an STM transaction` messages in [[bugs/git-annex-copy_fails_with___34__thread_blocked_indefinitely_in_an_STM_transaction__34__]]. Adding `--debug --verbose` also causes some non-json output.
|
||||
|
||||
### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
|
||||
|
||||
git-annex has been an extremely flexible and versatile tool. So far I've been able to adapt to most usage scenarios I've encountered.
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
### Please describe the problem.
|
||||
`git-annex-copy` fails after a short time with the message "thread blocked indefinitely in an STM transaction". Re-running it brings the same problem.
|
||||
|
||||
### What steps will reproduce the problem?
|
||||
|
||||
See below
|
||||
|
||||
### What version of git-annex are you using? On what operating system?
|
||||
|
||||
[[!format sh """
|
||||
(master_env_v156_py36) 14:22 [viral-ngs-benchmarks] $ git annex version
|
||||
git-annex version: 7.20190819-ge4cecf8
|
||||
build flags: Assistant Webapp Pairing S3 WebDAV Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite
|
||||
dependency versions: aws-0.21.1 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.3 feed-1.0.1.0 ghc-8.6.5 http-client-0.5.14 persistent-sql\
|
||||
ite-2.9.3 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0
|
||||
key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_2\
|
||||
24 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE\
|
||||
2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224\
|
||||
BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
|
||||
remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs hook external
|
||||
operating system: linux x86_64
|
||||
supported repository versions: 5 7
|
||||
upgrade supported from repository versions: 0 1 2 3 4 5 6
|
||||
local repository version: 5
|
||||
(master_env_v156_py36) 14:24 [viral-ngs-benchmarks] $ uname -a
|
||||
Linux ip-172-31-81-103.ec2.internal 4.14.138-114.102.amzn2.x86_64 #1 SMP Thu Aug 15 15:29:58 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux
|
||||
(master_env_v156_py36) 14:24 [viral-ngs-benchmarks] $
|
||||
|
||||
"""]]
|
||||
|
||||
|
||||
|
||||
### Please provide any additional information below.
|
||||
|
||||
[[!format sh """
|
||||
# If you can, paste a complete transcript of the problem occurring here.
|
||||
# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
|
||||
|
||||
(master_env_v156_py36) 14:18 [viral-ngs-benchmarks] $ git annex --debug --verbose copy -J1 --not --in dnanexus --and --not --in mygs \
|
||||
--to viral-ngs-benchmarks-s3 --json --json-error-messages > tmp/copy.out.txt 2> tmp/copy.err.txt
|
||||
C-c C-c
|
||||
(master_env_v156_py36) 14:22 [viral-ngs-benchmarks] $ git annex copy -J1 --not --in dnanexus --and --not --in mygs --to viral-ngs-ben\
|
||||
chmarks-s3 --json --json-error-messages > tmp/copy.out.txt 2> tmp/copy.err.txt
|
||||
(master_env_v156_py36) 14:22 [viral-ngs-benchmarks] $ echo $?
|
||||
1
|
||||
(master_env_v156_py36) 14:22 [viral-ngs-benchmarks] $ cat tmp/copy.err.txt
|
||||
git-annex: thread blocked indefinitely in an STM transaction
|
||||
git-annex: thread blocked indefinitely in an STM transaction
|
||||
(
|
||||
|
||||
# End of transcript or log.
|
||||
"""]]
|
||||
|
||||
### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 4"""
|
||||
date="2019-09-06T16:08:19Z"
|
||||
content="""
|
||||
The second transcript seems to show that the first sync never
|
||||
tries to push to origin. Then the second sync does push to origin. Then the
|
||||
third sync doesn't push to origin. Since sync always pushes to origin
|
||||
unless remote.name.annex-push or remote.name.annex-readonly are set,
|
||||
I remain confused why successive runs would sometimes push and other times
|
||||
not push.
|
||||
|
||||
(It also appears as if stderr output is sometimes appearing before
|
||||
stdout output that actually comes first. Which makes me not trust this
|
||||
transcript's paste accuracy either, or maybe something to do with the
|
||||
terminal is resulting in this unusual output ordering.)
|
||||
|
||||
Also, the second sync found one modified file to commit, so something must
|
||||
have modified that file in between the two sync runs. If a file gets
|
||||
modified in between two sync runs, they will of course not do the same
|
||||
thing.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
[[!comment format=mdwn
|
||||
username="Ilya_Shlyakhter"
|
||||
avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
|
||||
subject="git and submodule configs"
|
||||
date="2019-09-09T18:47:41Z"
|
||||
content="""
|
||||
Here is the git config:
|
||||
[[!format sh \"\"\"
|
||||
(master_env_v156_py36) 14:35 [viral-ngs-benchmarks] $ git config -l | grep annex- | grep -v uuid
|
||||
remote.origin.annex-ignore=true
|
||||
remote.ilya-work.annex-speculate-present=true
|
||||
remote.ilya-work.annex-readonly=true
|
||||
remote.ilya-work.annex-sync=false
|
||||
remote.dnanexus.annex-externaltype=dnanexus
|
||||
remote.dnanexus.annex-cost=100.0
|
||||
remote.dnanexus.annex-availability=GloballyAvailable
|
||||
remote.dnanexus.annex-security-allow-unverified-downloads=ACKTHPPT
|
||||
remote.dnanexus.annex-sync=false
|
||||
remote.dnanexus.annex-ignore=false
|
||||
remote.viral-ngs-benchmarks-s3.annex-s3=true
|
||||
remote.my-ldir.annex-externaltype=ldir
|
||||
remote.my-ldir.annex-cost=200.0
|
||||
remote.my-ldir.annex-availability=LocallyAvailable
|
||||
remote.from-ilya-work.annex-externaltype=canget
|
||||
remote.from-ilya-work.annex-readonly=true
|
||||
remote.from-ilya-work.annex-sync=false
|
||||
remote.from-ilya-work.annex-ignore=true
|
||||
remote.from-ilya-work-01.annex-externaltype=canget
|
||||
remote.from-ilya-work-01.annex-ignore=true
|
||||
remote.from-ilya-work-01.annex-cost=200.0
|
||||
remote.from-ilya-work-01.annex-availability=LocallyAvailable
|
||||
remote.from-ilya-work-01.annex-readonly=true
|
||||
remote.from-ilya-work-01.annex-sync=false
|
||||
remote.from-ilya-work-02.annex-externaltype=canget
|
||||
remote.from-ilya-work-02.annex-readonly=true
|
||||
remote.from-ilya-work-02.annex-sync=false
|
||||
remote.from-ilya-work-02.annex-ignore=true
|
||||
remote.from-ilya-work-03.annex-externaltype=canget
|
||||
remote.from-ilya-work-03.annex-cost=200.0
|
||||
remote.from-ilya-work-03.annex-availability=GloballyAvailable
|
||||
remote.from-ilya-work-03.annex-speculate-present=true
|
||||
remote.mygs.annex-externaltype=gs_uri
|
||||
remote.mygs.annex-cost=400.0
|
||||
remote.mygs.annex-availability=GloballyAvailable
|
||||
remote.s3-viral-ngs-benchmarks-web.annex-s3=true
|
||||
\"\"\"]]
|
||||
|
||||
\"stderr output is sometimes appearing before stdout output that actually comes first\" -- probably from running in an Emacs shell terminal. But the `git-annex-sync` commands were entered by hand, not in a script, so earlier commands' output is earlier.
|
||||
|
||||
\"the second sync found one modified file to commit, so something must have modified that file in between the two sync runs\" -- could the first sync run have somehow modified it?
|
||||
|
||||
If relevant, this git-annex repo has one submodule, configured as
|
||||
|
||||
[[!format sh \"\"\"
|
||||
[submodule \"viral-ngs\"]
|
||||
path = viral-ngs
|
||||
url = git@github.com:broadinstitute/viral-ngs.git
|
||||
branch = is-dx-benchmarks
|
||||
|
||||
\"\"\"]]
|
||||
|
||||
Maybe, add test cases for `git-annex-sync` on repos with submodules?
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
[[!comment format=mdwn
|
||||
username="https://tribut.de/"
|
||||
nickname="Felix"
|
||||
avatar="http://cdn.libravatar.org/avatar/d7508a030fd9902a76b02f7feff1327e80400a1317ee98e463c68ef4c9c40bb8"
|
||||
subject="comment 2"
|
||||
date="2019-09-09T09:01:26Z"
|
||||
content="""
|
||||
Just encountered the same error. I can reproduce like this:
|
||||
|
||||
$ git init foobar
|
||||
$ cd foobar/
|
||||
$ git annex init
|
||||
$ git annex upgrade
|
||||
$ touch baz
|
||||
$ git add baz
|
||||
$ git commit -m \"baz\"
|
||||
$ git annex adjust --unlock
|
||||
$ git submodule add https://github.com/tribut/homeassistant-docker-venv.git qux
|
||||
$ git add qux
|
||||
$ git commit -m \"qux\"
|
||||
$ git annex sync
|
||||
commit
|
||||
On branch adjusted/master(unlocked)
|
||||
nothing to commit, working tree clean
|
||||
ok
|
||||
fatal: entry 'qux' object type (blob) doesn't match mode type (commit)
|
||||
git-annex: user error (git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"mktree\",\"--batch\",\"-z\"] exited 128)
|
||||
|
||||
May it is related to gpg signatures? When I ran the exact same thing on my main machine, where `commit.gpgSign` is `true` I got instead:
|
||||
|
||||
$ git annex sync
|
||||
commit
|
||||
On branch adjusted/master(unlocked)
|
||||
nothing to commit, working tree clean
|
||||
ok
|
||||
git-annex: bad srcsha
|
||||
CallStack (from HasCallStack):
|
||||
error, called at ./Git/DiffTree.hs:113:39 in main:Git.DiffTree
|
||||
|
||||
I'm running git-annex 7.20190731-gbb16a2610 using the standalone version.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
[[!comment format=mdwn
|
||||
username="flixh"
|
||||
avatar="http://cdn.libravatar.org/avatar/1f7e89860de517a494f35ebfb385288e"
|
||||
subject="comment 4"
|
||||
date="2019-09-07T11:44:47Z"
|
||||
content="""
|
||||
You are right, git is at version 2.11.0-3+deb9u4 on this machine.
|
||||
OOM killer might be, the machine has 2GB RAM. During the upgrade I observed the memory usage increasing steadily but at some point decreasing again.
|
||||
|
||||
Unfortunately I've nuked the repo in the meantime, so I cannot try your suggestion of retrying after upgrading git. Also it looks like I need to go from stretch to testing to get a git >= 2.22. Which will probably not happen too soon.
|
||||
|
||||
Thanks,
|
||||
Felix
|
||||
"""]]
|
||||
|
|
@ -62,3 +62,5 @@ type nfs (rw,relatime,vers=3,rsize=131072,wsize=131072,namlen=255,hard,proto=tcp
|
|||
```
|
||||
|
||||
[[!meta author=yoh]]
|
||||
|
||||
> [[fixed|done]] --[[Joey]]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
[[!comment format=mdwn
|
||||
username="yarikoptic"
|
||||
avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
|
||||
subject="comment 2"
|
||||
date="2019-08-31T13:00:57Z"
|
||||
content="""
|
||||
I have ran git bisect, didn't double check but it points to
|
||||
|
||||
[[!format sh \"\"\"
|
||||
...
|
||||
running /home/bids/git-annexes/datalad/tools/bisect-git-annex /home/bids/datalad-datalad-master-git-annex-dev.simg /home/bids/git-annexes/datalad/tools/bisect-git-annex.scripts/bisect-nfs-lock.sh
|
||||
I: cleaning /home/bids/git-annexes/git-annex
|
||||
I: building 7.20181211-248-gd5f246370
|
||||
I: running the script
|
||||
Initialized empty Git repository in /inbox/BIDS/.git/tmp/repo/.git/
|
||||
init
|
||||
git-annex: waitToSetLock: resource exhausted (No locks available)
|
||||
failed
|
||||
git-annex: init: 1 failed
|
||||
d5f2463702a7cd8e56ea000e21bfe5d4ba9cd386 is the first bad commit
|
||||
commit d5f2463702a7cd8e56ea000e21bfe5d4ba9cd386
|
||||
Author: Joey Hess <joeyh@joeyh.name>
|
||||
Date: Thu Jan 17 15:40:44 2019 -0400
|
||||
|
||||
misctmp cleanup
|
||||
|
||||
* Switch to using .git/annex/othertmp for tmp files other than partial
|
||||
downloads, and make stale files left in that directory when git-annex
|
||||
is interrupted be cleaned up promptly by subsequent git-annex processes.
|
||||
* The .git/annex/misctmp directory is no longer used and git-annex will
|
||||
delete anything lingering in there after it's 1 week old.
|
||||
|
||||
Also, in Annex.Ingest, made the filename it uses in the tmp dir be
|
||||
prefixed with \"ingest-\" to avoid potentially using a filename used by
|
||||
some other code.
|
||||
|
||||
:040000 040000 3a3f3af1c5222bb96684c79b9bccc20b3ff266a2 24efba74a2b6e4a361e300f32325fafba76b8243 M Annex
|
||||
:040000 040000 193fa3b2bc663031a0e387e921c720e022daabbc 6a0081f3e1df14a182416e734eedb0a2d8bdd46d M Assistant
|
||||
:100644 100644 d313f9ee315a095e992b729f0d7001e14f413aee f5c5f511d85ffb52c303d1b718fc62e5179b66f9 M CHANGELOG
|
||||
:040000 040000 432613892dd51d2ebb804f11c96d1d2c7956dab7 9bcbedc855d35faeeaf9b6d74cc2f48c48d8d764 M Command
|
||||
:040000 040000 0d4841d00ee7ec5b59037ac2c37ab10482359671 0b12fe02a9028e3a7a083fd3dfdc01bf38a8626e M Remote
|
||||
:100644 100644 555166e29905dca5984403500528d4594b4b7bb6 8f9c460306db4c9f3abed23d9181a10f20b42b77 M Test.hs
|
||||
:040000 040000 2eb286c09f87da53c2377cc60b75c2c49fcd049a 33adb5b5e5b8a33b2012345b73f00eb094afd42c M Types
|
||||
:040000 040000 0406fc6fff2205a48d6a215cf18dba2c682de99e a688ce72ede0bbb71b7e2f1f6a811602ee333a16 M doc
|
||||
:100644 100644 839b469b5cea11a65f20652963fea316be367c93 ff2db6e39cee114d8bd386873b0dba6f89e4d4d1 M git-annex.cabal
|
||||
bisect run success
|
||||
\"\"\"]]
|
||||
may be that gives ideas? (build env was the same, freshly regenerated buster singularity image available via `singularity pull shub://datalad/datalad:git-annex-dev`
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
[[!comment format=mdwn
|
||||
username="yarikoptic"
|
||||
avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
|
||||
subject="the issue persists in master"
|
||||
date="2019-09-02T14:17:14Z"
|
||||
content="""
|
||||
FWIW, the issue persists with current version in master (which comes after comment from 3 days ago where wider exception catching was told to be implemented):
|
||||
|
||||
[[!format sh \"\"\"
|
||||
[bids@rolando git-annex] > ~/git-annexes/datalad/tools/bisect-git-annex.scripts/bisect-nfs-lock.sh
|
||||
Initialized empty Git repository in /inbox/BIDS/.git/tmp/repo/.git/
|
||||
init
|
||||
git-annex: waitToSetLock: resource exhausted (No locks available)
|
||||
failed
|
||||
git-annex: init: 1 failed
|
||||
[bids@rolando git-annex] > echo $?
|
||||
1
|
||||
[bids@rolando git-annex] > git annex version
|
||||
git-annex version: 7.20190826-gf951a6221
|
||||
build flags: Assistant Webapp Pairing S3 WebDAV Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite
|
||||
dependency versions: aws-0.20 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.3 feed-1.0.0.0 ghc-8.4.4 http-client-0.5.13.1 persistent-sqlite-2.8.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0
|
||||
key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
|
||||
remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs hook external
|
||||
operating system: linux x86_64
|
||||
supported repository versions: 5 7
|
||||
upgrade supported from repository versions: 0 1 2 3 4 5 6
|
||||
|
||||
\"\"\"]]
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 4"""
|
||||
date="2019-09-10T16:28:12Z"
|
||||
content="""
|
||||
Thanks for the bisection. It's not the init code that's
|
||||
trying and failing to set a lock, but the misctmp cleanup code. Which is
|
||||
ironically now used in setting up the temp directory that init uses to
|
||||
probe for locking problems. Chicken and egg problem.
|
||||
|
||||
Committed a fix.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="Ilya_Shlyakhter"
|
||||
avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
|
||||
subject="buffering chunks in memory when uploading to remote"
|
||||
date="2019-09-04T19:50:42Z"
|
||||
content="""
|
||||
\"git-annex has to buffer chunks in memory before they are sent to a remote\" -- would it be hard to remove this restriction? I want to have a non-chunked S3 remote, so that files in it can be accessed through a URL. But if some files are 50GB, then git-annex running on a 16GB machine would fail when talking to this repo?
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 4"""
|
||||
date="2019-09-10T18:36:06Z"
|
||||
content="""
|
||||
It seems that the file stored in the remote must have either been deleted
|
||||
by something, or possibly gotten corrupted. But probably deleted I think,
|
||||
because a corrupted file would have a different error message.
|
||||
|
||||
Lot of possible ways a file could get deleted. Perhaps StorageBox is
|
||||
storing the files as file on the filesystem, and had a disk problem that
|
||||
caused fsck to move the file to lost+found. Could happen, disks fail.
|
||||
|
||||
Another way would be, if you had made a clone of your git-annex repository,
|
||||
and enabled StorageBox in there, and done a `git annex move --from
|
||||
StorageBox`. Then the clone would know it had the copy and StorageBox no
|
||||
longer did, but until that clone's git-annex branch is pushed out to other
|
||||
repositories, they will think StorageBox still has the copy, and fail like
|
||||
you showed.
|
||||
|
||||
git-annex fsck exists because of these kinds of situations, and gets things
|
||||
back to a correct state. But it makes sense to investigate what happened.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 1"""
|
||||
date="2019-09-10T18:23:24Z"
|
||||
content="""
|
||||
I guess you moved the files using something other than `git mv`. So the
|
||||
files in `_archive` are not being tracked by git, the same as if you'd
|
||||
added new files to the repo.
|
||||
|
||||
If so, need to `git annex add _archive`
|
||||
(or `git add _archive` since you're using v7.)
|
||||
|
||||
The lack of output from git-annex indicates it's not found any files to
|
||||
work on.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="SophiaKelly"
|
||||
avatar="http://cdn.libravatar.org/avatar/6a04ff0f05dce6f672799f84d794f3fa"
|
||||
subject="Change AOL Password"
|
||||
date="2019-09-10T09:59:03Z"
|
||||
content="""
|
||||
The pre-built git-annex bundle for OS X includes a bunch of git and related tools within the bundle, it seemed like it should be possible -- but actually doing so, for \"server\" usage (eg, git annex sync and git annex copy . initiated from another system) proved more subtle than I expected. Users can Read:- Users can Read: <a href=\"http://www.aolhelp247.com/aol-change-password/\">Change AOL Password</a>
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
Hi,
|
||||
|
||||
I am trying to recover files from Hubic that were at some point uploaded with a leading slash in the filename. This made it impossible to get them with the git annex rclone extension because rclone interprets the leading slash as a directory move. So I wrote a script using the swiftclient python lib to download the files anyway. Then I am decrypting them with the ga_decrypt shell script and I am trying to reinject them into my local repo. For some files, the --known flag just works and injects the contents to the symlink that is broken. For some cases, I have to submit the src and dst filename and for some cases even then git-annex will tell me that the content does not match the expected content of the file.
|
||||
|
||||
This is the process that I follow:
|
||||
|
||||
For the file that is only on hubic I do a git-annex get file. The error message (since hubic integration is not working for that repo) gives me the key that I need to look that file up in hubic (what would be a cleaner way?):
|
||||
|
||||
git annex get 2012-06-23\ 15\:17\:23.mp4
|
||||
get 2012-06-23 15:17:23.mp4 (from hubic...)
|
||||
mv: cannot stat '/tmp/rclone-annex-tmp.diesZi1ZL/GPGHMACSHA1--b22130c742403960d80c0e9c78314e3d7034ba79': No such file or directory
|
||||
|
||||
Unable to access these remotes: hubic
|
||||
|
||||
Try making some of these repositories available:
|
||||
396e481c-5fc9-4933-b6a4-b1398fb7c61b -- [hubic]
|
||||
failed
|
||||
git-annex: get: 1 failed
|
||||
|
||||
|
||||
I prepend the leading slash and download that file manually from hubic. I use ga_decrypt and get files that actually are readable (in this case a video).
|
||||
However, when I do git-annex reinject src dst I get (I removed the colons because I wanted to rule out a bug):
|
||||
|
||||
git-annex: 2012-06-23_151723.mp4 does not have expected content of ../Pictures/Marek Handy/2012/2012-06-23_151723.mp4
|
||||
failed
|
||||
git-annex: reinject: 1 failed
|
||||
|
||||
Ok so apparently, git annex does not like that content for that file. Also if I do git-annex calckey on the downloaded and decrypted file I get a different key than the original key.
|
||||
|
||||
git-annex calckey 2012-06-23_151723.mp4
|
||||
SHA256E-s19520577--116c8b3101916fb1d81bf6816cc1b616a8eb30b5fe7b293053cfdd41fa178b0a.mp4
|
||||
|
||||
but ls -l on the file gives me:
|
||||
|
||||
ls -larth ../Pictures/Marek\ Handy/2012/2012-06-23\ 15\:17\:23.mp4
|
||||
lrwxrwxrwx 1 marek marek 201 Aug 6 17:28 '../Pictures/Marek Handy/2012/2012-06-23 15:17:23.mp4' -> ../../../.git/annex/objects/jQ/mk/SHA256E-s19120318--2631ff42deb7419e97dc46e7423937694c40d09b336d9b8e7aa2c5fe94e73cc7 /SHA256E-s19120318--2631ff42deb7419e97dc46e7423937694c40d09b336d9b8e7aa2c5fe94e73cc7
|
||||
|
||||
|
||||
I appreciate any hints to what went wrong or where maybe my thinking is wrong?
|
||||
|
||||
Thank you,
|
||||
Marek
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
Hi,
|
||||
|
||||
Is it possible to use hashdirlower layout with S3 special remote, instead of storing all the files in the root of the bucket?
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="TroisSinges"
|
||||
avatar="http://cdn.libravatar.org/avatar/f4c62f9ebe09016b8852b04f1dd6612f"
|
||||
subject="comment 1"
|
||||
date="2019-09-02T22:19:04Z"
|
||||
content="""
|
||||
Hmmm, this is strange, because according to <http://git-annex.branchable.com/special_remotes/S3/#comment-e9d48ba37aa5b01c236ab894057660b7>, files should be \"stored in a hashed directory structure with the names of their key used\". This isn't the case for me, using S3 special remote for Wasabi.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 2"""
|
||||
date="2019-09-05T15:29:57Z"
|
||||
content="""
|
||||
That comment was just bad wording or possibly I conflated S3 with how other
|
||||
remotes that do use hash directories work. I've corrected the comment.
|
||||
|
||||
According to Amazon's documentation, S3 does not have a concept of
|
||||
directories; "foo/bar" and "foo_bar" and "foo\\bar" are all just opaque
|
||||
strings as far as it's concerned. So I don't see any point in using hash
|
||||
directories with S3.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="Ilya_Shlyakhter"
|
||||
avatar="http://cdn.libravatar.org/avatar/1647044369aa7747829c38b9dcc84df0"
|
||||
subject="directories on S3"
|
||||
date="2019-09-05T18:40:20Z"
|
||||
content="""
|
||||
\"S3 does not have a concept of directories; \"foo/bar\" and \"foo_bar\" and \"foo\bar\" are all just opaque strings as far as it's concerned\" -- just to note: (1) AWS [CLI](https://aws.amazon.com/cli/) commands like `aws s3 ls` and `aws s3 cp` do have the concept of directories; (2) An [S3 tips page](https://www.sumologic.com/insight/10-things-might-not-know-using-s3/) says that \"latency on S3 operations depends on key names since prefix similarities become a bottleneck at more than about 100 requests per second. If you have need for high volumes of operations, it is essential to consider naming schemes with more variability at the beginning of the key names, like alphanumeric or hex hash codes in the first 6 to 8 characters, to avoid internal “hot spots” within S3 infrastructure.\"
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[[!comment format=mdwn
|
||||
username="TroisSinges"
|
||||
avatar="http://cdn.libravatar.org/avatar/f4c62f9ebe09016b8852b04f1dd6612f"
|
||||
subject="comment 4"
|
||||
date="2019-09-05T19:12:30Z"
|
||||
content="""
|
||||
Thanks Joey for your answer. However, even if directories are indeed totally virtual in S3, file paths are split using \"/\" characters when using S3 web console, for example (it's the same for Wasabi, for example). It's quite convenient!
|
||||
|
||||
Moreover, some S3-compatible services do create directories using \"/\" delimiters.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 5"""
|
||||
date="2019-09-06T17:53:00Z"
|
||||
content="""
|
||||
The stuff @Ilya found about prefix similaries causing bottlenecks in S3
|
||||
infra is interesting. git-annex keys have a common prefix, and have
|
||||
the hash at the end. So it could have a performance impact.
|
||||
|
||||
But that info also seems out of date when it talks about 6-8 characters
|
||||
prefix length. And the rate limit has also been raised significantly, to
|
||||
3000-5000 ops/sec per prefix. See
|
||||
<https://stackoverflow.com/questions/52443839/s3-what-exactly-is-a-prefix-and-what-ratelimits-apply>
|
||||
|
||||
From that, it seems S3 does actually treat '/' as a prefix delimiter.
|
||||
(Based on a single not very clear email from Amazon support and not
|
||||
documented anywhere else..) So a single level of hash "directories"
|
||||
could increase the rate limit accordingly.
|
||||
|
||||
If a git-annex exceeded those rate limits, it would start getting 503
|
||||
responses from S3, so it wouldn't slow down but would instead fail whatever
|
||||
operation it was doing. I can't recall anyone complaining of 503's from
|
||||
S3.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1 @@
|
|||
Have others experienced frequent failures of parallel/concurrent git-annex operations, like ones described in [[bugs/concurrent_git-annex-copy_to_s3_special_remote_fails/]] and [[bugs/parallel_copy_fails/]]? I'm trying to understand if there is something specific to my system that makes these failures more likely. Thanks!
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
[[!comment format=mdwn
|
||||
username="johnmario.itec19@69a7b742534851b36216e0f951f1a00dbb9067cd"
|
||||
nickname="johnmario.itec19"
|
||||
avatar="http://cdn.libravatar.org/avatar/2f07ffce1656bdcd6aa19aaab7517975"
|
||||
subject="commenting on git-annex-add"
|
||||
date="2019-09-02T06:21:27Z"
|
||||
content="""
|
||||
Yes you can do that. Simplest way is to git add the files you want to directly be in the git repo (e.g. the source code) and git annex add the large files.
|
||||
|
||||
You can then check in any changes to the source code files (or anything else you added with git add) to github as normal.
|
||||
|
||||
You can manage the storage and versioning of the large files using git annex commands. Git annex supports using AWS S3 and/or glacier for backing up the files. It can also back them up to a server you control over ssh or to an external drive (or any combination of the above). http://git-annex.branchable.com/special_remotes/
|
||||
|
||||
With the latest version of git annex, you can also set up automatically filters that decide which types/sizes of files to check in directly to git vs which ones to store as links in the annex. https://git-annex.branchable.com/tips/largefiles/
|
||||
For more tech related assistance or support <a href=\"https://uaedatarecovery.com/data-recovery-dubai/\">Data Recovery Dubai</a>
|
||||
"""]]
|
||||
|
|
@ -4,7 +4,7 @@
|
|||
subject="comment 8"
|
||||
date="2013-01-20T20:37:09Z"
|
||||
content="""
|
||||
If encryption is not used, the files are stored in S3 as-is, and can be accessed directly. They are stored in a hashed directory structure with the names of their key used, rather than the original filename. To get back to the original filename, a copy of the git repo would also be needed.
|
||||
If encryption is not used, the files are stored in S3 as-is, and can be accessed directly. The S3 bucket contains object named using the git-annex key, rather than the original filename. To get back to the original filename, a copy of the git repo would also be needed.
|
||||
|
||||
With encryption, you need the gpg key used in the encryption, or, for shared encryption, a symmetric key which is stored in the git repo.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 4"""
|
||||
date="2019-09-06T15:57:45Z"
|
||||
content="""
|
||||
annex.security.allowed-url-schemes can be configured to allow using file:
|
||||
urls.
|
||||
|
||||
However, adding a temp file as an url does not seem very useful. Seems
|
||||
that a much more typical bittorrent way to go would be to find or
|
||||
generate a magnet: url for the torrent.
|
||||
"""]]
|
||||
|
|
@ -66,3 +66,4 @@ EVAN HENSHAWPLATH,
|
|||
James Read,
|
||||
Luke Shumaker,
|
||||
Marius Konitzer,
|
||||
Ryan Rix,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
[[!comment format=mdwn
|
||||
username="ginquistador@86f226616ead98d2733e249429918f241f928064"
|
||||
nickname="ginquistador"
|
||||
avatar="http://cdn.libravatar.org/avatar/f0ef7d68c0ff5d4948a9b0d282987195"
|
||||
subject="Disappointed with `git add`"
|
||||
date="2019-09-03T07:30:28Z"
|
||||
content="""
|
||||
I first have to say, I have been following and using git annex for ages (5+ years at least), and is my trusted source for all my data. However, for the first time in all these years, I'm seeing a decision that I do not agree with or understand.
|
||||
|
||||
Specifically, using `git add .` to add a file to git annex as the default pattern just seems a fundamentally wrong design to me (at least for my usage pattern). I want to be able to use git normally, and have git-annex only get involved when I explicitly request it to, and not for all files. AFAIK, git-lfs does do it right. I understand [annex.largefiles: configuring mixed content repositories](http://git-annex.branchable.com/tips/largefiles/) can be configured to get the behavior I want. However, the default behavior should add it to vanilla git, and any other desired behavior can be obtained by the user via annex attributes, or extra command line flags to `git annex add`
|
||||
|
||||
Knowing Joey, I assume there's a strong rationale as always, and would love to hear it, but I would still like to STRONGLY REQUEST changing the default behavior.
|
||||
|
||||
|
||||
"""]]
|
||||
3
doc/todo/S3_export_redirecting_to_key-value_store.mdwn
Normal file
3
doc/todo/S3_export_redirecting_to_key-value_store.mdwn
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
S3 lets you [redirect](https://docs.aws.amazon.com/AmazonS3/latest/dev/how-to-page-redirect.html) requests for an object to another object, or to a URL. This could be used to export a git branch, in the manner of [[`git-annex-export`|git-annex-export]], but with annexed objects redirecting to a key-value S3 remote in the same bucket.
|
||||
|
||||
Related: [[todo/simpler__44___trusted_export_remotes]] ; [[forum/Using_hashdirlower_layout_for_S3_special_remote]].
|
||||
|
|
@ -18,8 +18,9 @@ a new empty directory in its place and start putting files in there.
|
|||
|
||||
* Odd behavior has been reported if the repository is deleted while
|
||||
git-annex is running it it. Including a git-annex seeming to hang or
|
||||
spin. I have not reproduced that, but I did see git-annex re-create the
|
||||
directories and get fairly confused. The sooner git-annex gives up in
|
||||
spin. <https://github.com/datalad/datalad/issues/3613>
|
||||
|
||||
The sooner git-annex gives up in
|
||||
such a situation, the less likely it is do get into an unusual state.
|
||||
|
||||
What's needed is an action that creates directories only up to a given
|
||||
|
|
|
|||
|
|
@ -42,6 +42,8 @@ avoids the problems discussed in step 5.
|
|||
|
||||
## step 5: automatic v5 to v7 upgrades
|
||||
|
||||
`v7-default` branch in git has this.
|
||||
|
||||
Since v5 repos and v7 repos not using unlocked files are functionally
|
||||
almost identical, this is unlikely to break much. Unlocking files will of
|
||||
course change behavior though.
|
||||
|
|
|
|||
Loading…
Reference in a new issue