Merge branch 'master' into streamproxy
This commit is contained in:
Joey Hess
commit
8baccda98f
37 changed files with 499 additions and 54 deletions
|
|
@ -11,7 +11,7 @@ module Annex.AdjustedBranch (
|
|||
Adjustment(..),
|
||||
LinkAdjustment(..),
|
||||
PresenceAdjustment(..),
|
||||
LinkPresentAdjustment(..),
|
||||
LockUnlockPresentAdjustment(..),
|
||||
adjustmentHidesFiles,
|
||||
adjustmentIsStable,
|
||||
OrigBranch,
|
||||
|
|
@ -88,11 +88,11 @@ instance AdjustTreeItem Adjustment where
|
|||
adjustTreeItem p t >>= \case
|
||||
Nothing -> return Nothing
|
||||
Just t' -> adjustTreeItem l t'
|
||||
adjustTreeItem (LinkPresentAdjustment l) t = adjustTreeItem l t
|
||||
adjustTreeItem (LockUnlockPresentAdjustment l) t = adjustTreeItem l t
|
||||
|
||||
adjustmentIsStable (LinkAdjustment l) = adjustmentIsStable l
|
||||
adjustmentIsStable (PresenceAdjustment p _) = adjustmentIsStable p
|
||||
adjustmentIsStable (LinkPresentAdjustment l) = adjustmentIsStable l
|
||||
adjustmentIsStable (LockUnlockPresentAdjustment l) = adjustmentIsStable l
|
||||
|
||||
instance AdjustTreeItem LinkAdjustment where
|
||||
adjustTreeItem UnlockAdjustment =
|
||||
|
|
@ -115,7 +115,7 @@ instance AdjustTreeItem PresenceAdjustment where
|
|||
adjustmentIsStable HideMissingAdjustment = False
|
||||
adjustmentIsStable ShowMissingAdjustment = True
|
||||
|
||||
instance AdjustTreeItem LinkPresentAdjustment where
|
||||
instance AdjustTreeItem LockUnlockPresentAdjustment where
|
||||
adjustTreeItem UnlockPresentAdjustment =
|
||||
ifPresent adjustToPointer adjustToSymlink
|
||||
adjustTreeItem LockPresentAdjustment =
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ instance SerializeAdjustment Adjustment where
|
|||
serializeAdjustment p
|
||||
serializeAdjustment (PresenceAdjustment p (Just l)) =
|
||||
serializeAdjustment p <> "-" <> serializeAdjustment l
|
||||
serializeAdjustment (LinkPresentAdjustment l) =
|
||||
serializeAdjustment (LockUnlockPresentAdjustment l) =
|
||||
serializeAdjustment l
|
||||
deserializeAdjustment s =
|
||||
(LinkAdjustment <$> deserializeAdjustment s)
|
||||
|
|
@ -46,7 +46,7 @@ instance SerializeAdjustment Adjustment where
|
|||
<|>
|
||||
(PresenceAdjustment <$> deserializeAdjustment s <*> pure Nothing)
|
||||
<|>
|
||||
(LinkPresentAdjustment <$> deserializeAdjustment s)
|
||||
(LockUnlockPresentAdjustment <$> deserializeAdjustment s)
|
||||
where
|
||||
(s1, s2) = separate' (== (fromIntegral (ord '-'))) s
|
||||
|
||||
|
|
@ -68,7 +68,7 @@ instance SerializeAdjustment PresenceAdjustment where
|
|||
deserializeAdjustment "showmissing" = Just ShowMissingAdjustment
|
||||
deserializeAdjustment _ = Nothing
|
||||
|
||||
instance SerializeAdjustment LinkPresentAdjustment where
|
||||
instance SerializeAdjustment LockUnlockPresentAdjustment where
|
||||
serializeAdjustment UnlockPresentAdjustment = "unlockpresent"
|
||||
serializeAdjustment LockPresentAdjustment = "lockpresent"
|
||||
deserializeAdjustment "unlockpresent" = Just UnlockPresentAdjustment
|
||||
|
|
|
|||
|
|
@ -359,8 +359,8 @@ addUnlocked matcher mi contentpresent =
|
|||
go (LinkAdjustment UnFixAdjustment) = False
|
||||
go (PresenceAdjustment _ (Just la)) = go (LinkAdjustment la)
|
||||
go (PresenceAdjustment _ Nothing) = False
|
||||
go (LinkPresentAdjustment UnlockPresentAdjustment) = contentpresent
|
||||
go (LinkPresentAdjustment LockPresentAdjustment) = False
|
||||
go (LockUnlockPresentAdjustment UnlockPresentAdjustment) = contentpresent
|
||||
go (LockUnlockPresentAdjustment LockPresentAdjustment) = False
|
||||
|
||||
{- Adds a file to the work tree for the key, and stages it in the index.
|
||||
- The content of the key may be provided in a temp file, which will be
|
||||
|
|
|
|||
|
|
@ -513,7 +513,7 @@ applyView'' mkviewedfile getfilemetadata view madj l clean conv = do
|
|||
Nothing -> stagesymlink uh f k
|
||||
Just (LinkAdjustment UnlockAdjustment) ->
|
||||
stagepointerfile uh f k mtreeitemtype
|
||||
Just (LinkPresentAdjustment UnlockPresentAdjustment) ->
|
||||
Just (LockUnlockPresentAdjustment UnlockPresentAdjustment) ->
|
||||
ifM (inAnnex k)
|
||||
( stagepointerfile uh f k mtreeitemtype
|
||||
, stagesymlink uh f k
|
||||
|
|
|
|||
|
|
@ -7,8 +7,15 @@ git-annex (10.20240928) UNRELEASED; urgency=medium
|
|||
* p2phttp: Support serving unauthenticated users while requesting
|
||||
authentication for operations that need it. Eg, --unauth-readonly
|
||||
can be combined with --authenv.
|
||||
* p2phttp: Allow unauthenticated users to lock content by default.
|
||||
* p2phttp: Added --unauth-nolocking option to prevent unauthenticated
|
||||
users from locking content.
|
||||
* Allow enabling the servant build flag with older versions of stm,
|
||||
allowing building with ghc 9.0.2.
|
||||
* git-remote-annex: Fix bug that prevented using it with external special
|
||||
remotes, leading to protocol error messages involving "GITMANIFEST".
|
||||
* adjust: Allow any order of options when combining --hide-missing with
|
||||
options like --unlock.
|
||||
|
||||
-- Joey Hess <id@joeyh.name> Thu, 17 Oct 2024 11:02:17 -0400
|
||||
|
||||
|
|
|
|||
|
|
@ -17,9 +17,24 @@ cmd = notBareRepo $ noDaemonRunning $
|
|||
|
||||
optParser :: CmdParamsDesc -> Parser Adjustment
|
||||
optParser _ =
|
||||
(LinkAdjustment <$> linkAdjustmentParser)
|
||||
<|> (PresenceAdjustment <$> presenceAdjustmentParser <*> maybeLinkAdjustmentParser)
|
||||
<|> (LinkPresentAdjustment <$> linkPresentAdjustmentParser)
|
||||
linkPresentAdjustmentParser
|
||||
<|> (LockUnlockPresentAdjustment <$> lockUnlockPresentAdjustmentParser)
|
||||
|
||||
linkPresentAdjustmentParser :: Parser Adjustment
|
||||
linkPresentAdjustmentParser = comb <$> some ps
|
||||
where
|
||||
ps = (LinkAdjustment <$> linkAdjustmentParser)
|
||||
<|> (PresenceAdjustment <$> presenceAdjustmentParser <*> pure Nothing)
|
||||
comb (LinkAdjustment _ : LinkAdjustment b : c) =
|
||||
comb (LinkAdjustment b : c)
|
||||
comb (PresenceAdjustment _a1 a2 : PresenceAdjustment b1 b2 : c) =
|
||||
comb (PresenceAdjustment b1 (b2 <|> a2) : c)
|
||||
comb (LinkAdjustment a : PresenceAdjustment b1 b2 : c) =
|
||||
comb (PresenceAdjustment b1 (b2 <|> Just a) : c)
|
||||
comb (PresenceAdjustment a1 _a2 : LinkAdjustment b : c) =
|
||||
comb (PresenceAdjustment a1 (Just b) : c)
|
||||
comb (a : _) = a
|
||||
comb [] = error "internal"
|
||||
|
||||
linkAdjustmentParser :: Parser LinkAdjustment
|
||||
linkAdjustmentParser =
|
||||
|
|
@ -36,9 +51,6 @@ linkAdjustmentParser =
|
|||
<> help "fix symlinks to annnexed files"
|
||||
)
|
||||
|
||||
maybeLinkAdjustmentParser :: Parser (Maybe LinkAdjustment)
|
||||
maybeLinkAdjustmentParser = Just <$> linkAdjustmentParser <|> pure Nothing
|
||||
|
||||
presenceAdjustmentParser :: Parser PresenceAdjustment
|
||||
presenceAdjustmentParser =
|
||||
flag' HideMissingAdjustment
|
||||
|
|
@ -46,8 +58,8 @@ presenceAdjustmentParser =
|
|||
<> help "hide annexed files whose content is not present"
|
||||
)
|
||||
|
||||
linkPresentAdjustmentParser :: Parser LinkPresentAdjustment
|
||||
linkPresentAdjustmentParser =
|
||||
lockUnlockPresentAdjustmentParser :: Parser LockUnlockPresentAdjustment
|
||||
lockUnlockPresentAdjustmentParser =
|
||||
flag' UnlockPresentAdjustment
|
||||
( long "unlock-present"
|
||||
<> help "unlock files whose content is present; lock rest"
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ data Options = Options
|
|||
, authEnvHttpOption :: Bool
|
||||
, unauthReadOnlyOption :: Bool
|
||||
, unauthAppendOnlyOption :: Bool
|
||||
, unauthNoLockingOption :: Bool
|
||||
, wideOpenOption :: Bool
|
||||
, proxyConnectionsOption :: Maybe Integer
|
||||
, clusterJobsOption :: Maybe Int
|
||||
|
|
@ -83,6 +84,10 @@ optParser _ = Options
|
|||
( long "unauth-appendonly"
|
||||
<> help "allow unauthenticated users to read and append to the repository"
|
||||
)
|
||||
<*> switch
|
||||
( long "unauth-nolocking"
|
||||
<> help "prevent unauthenticated users from locking content in the repository"
|
||||
)
|
||||
<*> switch
|
||||
( long "wideopen"
|
||||
<> help "give unauthenticated users full read+write access"
|
||||
|
|
@ -128,15 +133,25 @@ seek o = getAnnexWorkerPool $ \workerpool ->
|
|||
|
||||
mkGetServerMode :: M.Map Auth P2P.ServerMode -> Options -> GetServerMode
|
||||
mkGetServerMode _ o _ Nothing
|
||||
| wideOpenOption o =
|
||||
ServerMode P2P.ServeReadWrite False
|
||||
| unauthAppendOnlyOption o =
|
||||
ServerMode P2P.ServeAppendOnly canauth
|
||||
| unauthReadOnlyOption o =
|
||||
ServerMode P2P.ServeReadOnly canauth
|
||||
| wideOpenOption o = ServerMode
|
||||
{ serverMode = P2P.ServeReadWrite
|
||||
, unauthenticatedLockingAllowed = unauthlock
|
||||
, authenticationAllowed = False
|
||||
}
|
||||
| unauthAppendOnlyOption o = ServerMode
|
||||
{ serverMode = P2P.ServeAppendOnly
|
||||
, unauthenticatedLockingAllowed = unauthlock
|
||||
, authenticationAllowed = canauth
|
||||
}
|
||||
| unauthReadOnlyOption o = ServerMode
|
||||
{ serverMode = P2P.ServeReadOnly
|
||||
, unauthenticatedLockingAllowed = unauthlock
|
||||
, authenticationAllowed = canauth
|
||||
}
|
||||
| otherwise = CannotServeRequests
|
||||
where
|
||||
canauth = authEnvOption o || authEnvHttpOption o
|
||||
unauthlock = not (unauthNoLockingOption o)
|
||||
mkGetServerMode authenv o issecure (Just auth) =
|
||||
case (issecure, authEnvOption o, authEnvHttpOption o) of
|
||||
(Secure, True, _) -> checkauth
|
||||
|
|
@ -144,7 +159,11 @@ mkGetServerMode authenv o issecure (Just auth) =
|
|||
_ -> noauth
|
||||
where
|
||||
checkauth = case M.lookup auth authenv of
|
||||
Just servermode -> ServerMode servermode False
|
||||
Just servermode -> ServerMode
|
||||
{ serverMode = servermode
|
||||
, authenticationAllowed = False
|
||||
, unauthenticatedLockingAllowed = False
|
||||
}
|
||||
Nothing -> noauth
|
||||
noauth = mkGetServerMode authenv noautho issecure Nothing
|
||||
noautho = o { authEnvOption = False, authEnvHttpOption = False }
|
||||
|
|
|
|||
|
|
@ -425,7 +425,7 @@ serveLockContent
|
|||
-> Maybe Auth
|
||||
-> Handler LockResult
|
||||
serveLockContent st su apiver (B64Key k) cu bypass sec auth = do
|
||||
conn <- getP2PConnection apiver st cu su bypass sec auth WriteAction id
|
||||
conn <- getP2PConnection apiver st cu su bypass sec auth LockAction id
|
||||
let lock = do
|
||||
lockresv <- newEmptyTMVarIO
|
||||
unlockv <- newEmptyTMVarIO
|
||||
|
|
@ -465,7 +465,7 @@ serveKeepLocked
|
|||
-> S.SourceT IO UnlockRequest
|
||||
-> Handler LockResult
|
||||
serveKeepLocked st _su _apiver lckid _cu _bypass sec auth _ _ unlockrequeststream = do
|
||||
checkAuthActionClass st sec auth WriteAction $ \_ -> do
|
||||
checkAuthActionClass st sec auth LockAction $ \_ -> do
|
||||
liftIO $ keepingLocked lckid st
|
||||
_ <- liftIO $ S.unSourceT unlockrequeststream go
|
||||
return (LockResult False Nothing)
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ type GetServerMode = IsSecure -> Maybe Auth -> ServerMode
|
|||
data ServerMode
|
||||
= ServerMode
|
||||
{ serverMode :: P2P.ServerMode
|
||||
, unauthenticatedLockingAllowed :: Bool
|
||||
, authenticationAllowed :: Bool
|
||||
}
|
||||
| CannotServeRequests
|
||||
|
|
@ -68,7 +69,7 @@ mkP2PHttpServerState acquireconn annexworkerpool getservermode = P2PHttpServerSt
|
|||
<*> pure getservermode
|
||||
<*> newTMVarIO mempty
|
||||
|
||||
data ActionClass = ReadAction | WriteAction | RemoveAction
|
||||
data ActionClass = ReadAction | WriteAction | RemoveAction | LockAction
|
||||
deriving (Eq)
|
||||
|
||||
withP2PConnection
|
||||
|
|
@ -152,6 +153,8 @@ checkAuthActionClass st sec auth actionclass go =
|
|||
case (sm, actionclass) of
|
||||
(ServerMode { serverMode = P2P.ServeReadWrite }, _) ->
|
||||
go P2P.ServeReadWrite
|
||||
(ServerMode { unauthenticatedLockingAllowed = True }, LockAction) ->
|
||||
go P2P.ServeReadOnly
|
||||
(ServerMode { serverMode = P2P.ServeAppendOnly }, RemoveAction) ->
|
||||
throwError $ forbiddenWithoutAuth sm
|
||||
(ServerMode { serverMode = P2P.ServeAppendOnly }, _) ->
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ module Types.AdjustedBranch where
|
|||
data Adjustment
|
||||
= LinkAdjustment LinkAdjustment
|
||||
| PresenceAdjustment PresenceAdjustment (Maybe LinkAdjustment)
|
||||
| LinkPresentAdjustment LinkPresentAdjustment
|
||||
| LockUnlockPresentAdjustment LockUnlockPresentAdjustment
|
||||
deriving (Show, Eq)
|
||||
|
||||
data LinkAdjustment
|
||||
|
|
@ -25,7 +25,7 @@ data PresenceAdjustment
|
|||
| ShowMissingAdjustment
|
||||
deriving (Show, Eq)
|
||||
|
||||
data LinkPresentAdjustment
|
||||
data LockUnlockPresentAdjustment
|
||||
= UnlockPresentAdjustment
|
||||
| LockPresentAdjustment
|
||||
deriving (Show, Eq)
|
||||
|
|
@ -41,8 +41,8 @@ instance ReversableAdjustment Adjustment where
|
|||
LinkAdjustment (reverseAdjustment l)
|
||||
reverseAdjustment (PresenceAdjustment p ml) =
|
||||
PresenceAdjustment (reverseAdjustment p) (fmap reverseAdjustment ml)
|
||||
reverseAdjustment (LinkPresentAdjustment l) =
|
||||
LinkPresentAdjustment (reverseAdjustment l)
|
||||
reverseAdjustment (LockUnlockPresentAdjustment l) =
|
||||
LockUnlockPresentAdjustment (reverseAdjustment l)
|
||||
|
||||
instance ReversableAdjustment LinkAdjustment where
|
||||
reverseAdjustment UnlockAdjustment = LockAdjustment
|
||||
|
|
@ -55,7 +55,7 @@ instance ReversableAdjustment PresenceAdjustment where
|
|||
reverseAdjustment HideMissingAdjustment = ShowMissingAdjustment
|
||||
reverseAdjustment ShowMissingAdjustment = HideMissingAdjustment
|
||||
|
||||
instance ReversableAdjustment LinkPresentAdjustment where
|
||||
instance ReversableAdjustment LockUnlockPresentAdjustment where
|
||||
reverseAdjustment UnlockPresentAdjustment = LockPresentAdjustment
|
||||
reverseAdjustment LockPresentAdjustment = UnlockPresentAdjustment
|
||||
|
||||
|
|
|
|||
|
|
@ -354,6 +354,7 @@ parseKeyVariety "WORM" = WORMKey
|
|||
parseKeyVariety "URL" = URLKey
|
||||
parseKeyVariety "VURL" = VURLKey
|
||||
parseKeyVariety "GITBUNDLE" = GitBundleKey
|
||||
parseKeyVariety "GITMANIFEST" = GitManifestKey
|
||||
parseKeyVariety b
|
||||
| "X" `S.isPrefixOf` b =
|
||||
let b' = S.tail b
|
||||
|
|
|
|||
|
|
@ -40,3 +40,5 @@ This is admittedly arguably consistent with https://git-annex.branchable.com/git
|
|||
### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
|
||||
|
||||
Yes, I've been really happy using it to manage a bunch of videos, where I only need some on my laptop at any given time. Way better than my previous "manually scp things around" strategy.
|
||||
|
||||
> [[fixed|done]] --[[Joey]]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 1"""
|
||||
date="2024-10-21T19:06:47Z"
|
||||
content="""
|
||||
Surprising and unintentional! Fixed this.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 1"""
|
||||
date="2024-10-19T20:23:21Z"
|
||||
content="""
|
||||
Here I see the error after `git-annex get`, but not after `git-annex add`.
|
||||
|
||||
git version 2.45.2
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 2"""
|
||||
date="2024-10-19T20:24:21Z"
|
||||
content="""
|
||||
The git command that git-annex is running that outputs this message,
|
||||
in the `git-annex get` case, is:
|
||||
|
||||
git diff 4b825dc642cb6eb9a060e54bf8d69288fbee4904 335cd01e749bcfecc96b9d6d1f0bc3edb3ef2aec042da9a40379414a0f5d2243
|
||||
|
||||
Here 4b825dc642cb6eb9a060e54bf8d69288fbee4904 is the empty tree sha1, which is
|
||||
hardcoded in git-annex as emptyTree. It must need to instead use the
|
||||
corresponding sha256
|
||||
(6ef19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321) when diffing
|
||||
against a sha256.
|
||||
|
||||
A related problem is that deleteSha is the all 0 sha1, and it turns out that
|
||||
using that rather than the sha256 version, in a sha256 repository, fails at
|
||||
least when `git update-index --index-info` does it. This is a bit surprising
|
||||
given the comment in deleteSha that says git will accept the sha1 in that
|
||||
situation. deleteSha is used in a couple of other places too, which will need to be checked.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 1"""
|
||||
date="2024-10-21T14:49:49Z"
|
||||
content="""
|
||||
Given the number of different ways freezeContent is called, it would be
|
||||
very difficult to implement error unwinding for it. So a failure would
|
||||
leave things in a potentially unusual state. For example a failure during
|
||||
`git-annex add` at one point would leave the file moved into the
|
||||
repository, but without an annex symlink pointing to it.
|
||||
|
||||
Or in some cases, freezeContent is the last thing that is done, so failing
|
||||
would leave it in the same state as not failing.
|
||||
|
||||
Looking at this another way, there are two ways a freezecontent hook can
|
||||
fail:
|
||||
|
||||
1. It exists successfully, but has not actually managed to freeze the
|
||||
content.
|
||||
2. It exits nonzero.
|
||||
|
||||
git-annex cannot detect the first case. So there does not seem to be much
|
||||
benefit to complicating git-annex to detect the second case and handle it
|
||||
specially.
|
||||
|
||||
Regular lock down of files by removing permissions can also fail, in an
|
||||
expected way, in a shared repository when the "wrong" user is running
|
||||
git-annex . `git-annex fsck` deals with that by trying again to freeze the
|
||||
content and checking if the file has the desired permission, and warning if
|
||||
not.
|
||||
|
||||
Since fsck does run the freeze hook, it will display those error messages
|
||||
again, which at least will remind the user that the hook is broken. And if
|
||||
they fix it, fsck will freeze the content.
|
||||
"""]]
|
||||
|
|
@ -46,3 +46,7 @@ I'm slowly integrating it into everything I do because it's so cool. Right now,
|
|||
- git-remote-annex - Seems incredibly promising despite issues I've had getting it to install (see below) let alone work quite yet
|
||||
|
||||
Also, if anyone is having issue using git-remote-annex, like I did running on OSX installed via homebrew, the solution is to add a symlink manually from (e.g.) `$(realpath /opt/homebrew/bin/git-annex[-shell]) <- /opt/homebrew/bin/git-remote-annex`. In other words, the current homebrew recipe simply doesn't symlink the git-annex command to git-remote-annex on your path, which git annex automagically act like a git remote helper if called like this so luckily it's an easy fix.
|
||||
|
||||
> [[fixed|done]] I *think*, please comment if not --[[Joey]]
|
||||
|
||||
[[!tag projects/INM7]]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 1"""
|
||||
date="2024-10-19T20:48:11Z"
|
||||
content="""
|
||||
I got this protocol level dump:
|
||||
|
||||
[2024-10-19 16:59:54.229607011] (Annex.ExternalAddonProcess) /home/joey/bin/rclone[1] <-- CHECKPRESENT GITMANIFEST--1eb4e389-9aea-4968-a947-498fd52645e0
|
||||
[2024-10-19 16:59:54.230079229] (Annex.ExternalAddonProcess) /home/joey/bin/rclone[1] --> CHECKPRESENT-FAILURE GITMANIFEST--1eb4e389-9aea-4968-a947-498fd52645e0
|
||||
|
||||
That protocol looks fine, problem is parseKeyVariety does not have a parser for the
|
||||
GITMANIFEST key type. Once I added that, it got to a different failure that
|
||||
looked due to my lack of setup of rclone, presumably it would work with a
|
||||
properly setup rclone.
|
||||
|
||||
I think this probably affects using any special remote with git-remote-annex.
|
||||
This is still a fairly new and not much used feature, and I may not have tested
|
||||
it with external special remotes when developing it.
|
||||
|
||||
Re your questions, git-remote-annex doesn't care about the details of how
|
||||
the special remote is configured, it's up to you to provide an url that
|
||||
provides all necessary configuration options for the special remote,
|
||||
whatever those are. Spaces in a path should work if properly url-encoded.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 2"""
|
||||
date="2024-10-19T21:51:48Z"
|
||||
content="""
|
||||
About using rclone with git-remote-annex, I wonder how practical it will
|
||||
be, since rclone does need its own configuration. To clone from it you will
|
||||
need to have rclone configured already, it seems.
|
||||
|
||||
If the rclone special remote could include the complete rclone
|
||||
configuration in its configuration, it would be possible to make a
|
||||
git-remote-annex url that could be cloned with no additional configuration.
|
||||
"""]]
|
||||
|
|
@ -41,3 +41,5 @@ local repository version: 10
|
|||
### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
|
||||
|
||||
[[!tag projects/INM7]]
|
||||
|
||||
> [[fixed|done]] --[[Joey]]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 10"""
|
||||
date="2024-10-21T13:46:00Z"
|
||||
content="""
|
||||
Ok I decided to allow locking for unauthenticated users by default.
|
||||
|
||||
In case that gets abused there is a --unauth-nolocking option which will
|
||||
result in a 401 when --authenv is used, or a 403 otherwise.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 7"""
|
||||
date="2024-10-18T14:56:10Z"
|
||||
content="""
|
||||
In a sense the underlying problem here is that git-annex as a client to
|
||||
p2phttp does not know if the user wants it to be read-only or prompt for a
|
||||
password as necessary to perform write operations.
|
||||
|
||||
Another way that would be a problem is if a p2phttp server supports both
|
||||
readonly and authenticated operation, but the user does not have an
|
||||
account, and is using say, `git-annex assist`, which wants to store content
|
||||
on the server if possible. So it will prompt repeatedly for a login and
|
||||
password which the user does not have.
|
||||
|
||||
In this sense, the server is fine in sending a 401, the problem is that the
|
||||
client doesn't know when the user doesn't want that to result in a
|
||||
password prompt. If the client did know it could treat that 401 the same as
|
||||
a 403.
|
||||
|
||||
Looking at drop locking through this lens, if the client wants to avoid
|
||||
password prompts and the server requires authentication for lockcontent,
|
||||
it's reasonable for the client to fall back to checkpresent for dropping.
|
||||
The same as it's reasonable for checkpresent to be used when the remote is
|
||||
a dumb http git remote.
|
||||
|
||||
The url to the p2phttp remote seems like the natural way for the user to
|
||||
tell git-annex if they want an anonymous or an authenticated connection.
|
||||
|
||||
It already works to use `annex+http://joey@example.com/git-annex/`,
|
||||
which will make it prompt for my password when operations need
|
||||
authentication.
|
||||
|
||||
So it would make sense to support "anonymous@" and special case that
|
||||
to treat 401 the same as 403.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 8"""
|
||||
date="2024-10-18T15:50:06Z"
|
||||
content="""
|
||||
@matrss would there be a way for forgejo-aneksajo to draw this distinction
|
||||
between anonymous and authenticated urls in its user interface?
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
[[!comment format=mdwn
|
||||
username="matrss"
|
||||
avatar="http://cdn.libravatar.org/avatar/59541f50d845e5f81aff06e88a38b9de"
|
||||
subject="comment 9"
|
||||
date="2024-10-18T16:56:46Z"
|
||||
content="""
|
||||
I agree, this would be easy to deal with if the user intent was somehow made clear.
|
||||
|
||||
_Technically_ anonymous is a valid username in Forgejo.
|
||||
|
||||
The way I've implemented it so far the user interface only exposes the `http(s)://` URL for the normal git remote and the `annex+http(s)://` URL is then taken from the server-side git config (on `git annex init`). Since I believe that git-annex will always try to read the config without authentication first I don't yet see a way to distinguish user intent. It _could_ show the user different URLs for the git remote depending on if the user is logged in or not, but I don't think there is way to make it dependent on the plain-git remote URL? That also doesn't sound like the best idea, and I am not sure if it really is a good proxy for user intent; I would expect people to not be authenticated in the web interface and still want to push files to the server after cloning (e.g. maybe they got logged out from a timeout, don't notice, and now go to a public repository of their own to clone it and make changes).
|
||||
|
||||
To mark a remote as read-only there is the already existing `remote.<remote>.annex-readonly` config option that could be used. But when to set it...
|
||||
|
||||
I still think the most practical option would be to special-case lockcontent... I'd expect that any user who seriously uses http will have some kind of credential helper configured. If they have used copy/move/sync/assist/etc then they will already have their credentials in the helper and the lockcontent request can just use them, and if they have just gotten some files and now drop them and don't have credentials for the remote in their helper then I think it is safe to assume they don't want to specify them just for dropping, i.e. fallback to checkpresent. In a situation where a lock is really necessary for dropping and it wouldn't succeed without git-annex could still ask for credentials. Is there a situation in which that wouldn't do \"the right thing\"?
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 1"""
|
||||
date="2024-10-21T15:50:26Z"
|
||||
content="""
|
||||
I suspect that one file has a "yt:" url recorded and the other one does not.
|
||||
That's not displayed by `whereis` but you can see it if you look at the
|
||||
$key.log.web file in the git-annex branch.
|
||||
|
||||
That typically would happen if you used `git-annex addurl --raw`,
|
||||
or added the url when yt-dlp was not working, although that doesn't explain
|
||||
how the youtube title got into the filename. (Maybe importfeed?)
|
||||
|
||||
(I seemed to remember you having a similar problem before, and
|
||||
found <https://git-annex.branchable.com/todo/yt-dlp__58___parse__47__handle___40__error__41_____34__Video_unavailable__34__/>
|
||||
which seems possibly relevant to how this could have happened.)
|
||||
"""]]
|
||||
|
|
@ -39,7 +39,8 @@ Authentication is done using HTTP basic auth. The realm to use when
|
|||
authenticating is "git-annex". The charset is UTF-8.
|
||||
|
||||
When authentication is successful but does not allow a request to be
|
||||
performed, it will fail with 403 Forbidden.
|
||||
performed, it will fail with 403 Forbidden. (This also is sent when the
|
||||
server does not support authentication.)
|
||||
|
||||
Note that HTTP basic auth is not encrypted so is only secure when used
|
||||
over HTTPS.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 1"""
|
||||
date="2024-10-21T23:11:11Z"
|
||||
content="""
|
||||
The name of the cluster ends up in git-annex:proxy.log. So probably the
|
||||
easiest thing to do is to `git-annex initcluster` a new cluster with the new name,
|
||||
change the git configs remote.foo.annex-cluster-node to be set to the new
|
||||
name, and run `git-annex updatecluster`.
|
||||
|
||||
That will remove all the nodes from the old cluster name and add them to the new
|
||||
cluster name, and without any nodes left the old cluster name will never be
|
||||
displayed anywhere.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
I'm trying to set up git-annex on my mac (running macOS Sonoma 14.7). I want it to sync a directory to a linux server so that the linux server always has a full copy of the directory.
|
||||
|
||||
The linux server runs Arch Linux. All I did there was installing the git-annex package from the official package repository. (sshd, rsync etc. were already installed)
|
||||
|
||||
On the mac I used `brew install git-annex` to install and `brew services start git-annex` to start the service, then issued `git annex webapp` to start the web app for configuration. In the web app I followed the setup wizard to create a local repo and added the linux server as a remote (basically what's shown in the 10 minute assistant intro screencast but I chose group "backup" instead of "transfer" for the remote).
|
||||
|
||||
When I now copy files into the repository, they get auto-added to the repository – but not auto-transferred to the remote. For that to happen I have to restart the service (`brew services restart git-annex`) or do a `git annex push` / `git annex sync --content`.
|
||||
|
||||
After copying a file to the repo:
|
||||
|
||||
```sh
|
||||
~/annex $ git annex list
|
||||
here
|
||||
|linux-server
|
||||
||web
|
||||
|||bittorrent
|
||||
||||
|
||||
X___ file.ext
|
||||
```
|
||||
|
||||
After restarting etc.:
|
||||
|
||||
```sh
|
||||
~/annex $ git annex list
|
||||
here
|
||||
|linux-server
|
||||
||web
|
||||
|||bittorrent
|
||||
||||
|
||||
XX__ file.ext
|
||||
```
|
||||
|
||||
Is there a way to have added files automatically transfer to the remote?
|
||||
|
||||
I also noticed that the web app hangs for 30s every 4-5 clicks. Something seems to be wrong with my setup and configuration. What can it be?
|
||||
|
||||
Thanks, this seems to be a really great a versatile tool!
|
||||
|
||||
---
|
||||
|
||||
Update:
|
||||
|
||||
Looks like the git-annex service doesn't really run:
|
||||
|
||||
```sh
|
||||
$ brew services
|
||||
Name Status User File
|
||||
git-annex stopped username ~/Library/LaunchAgents/homebrew.mxcl.git-annex.plist
|
||||
```
|
||||
|
||||
Trying to start it yields:
|
||||
|
||||
```sh
|
||||
$ brew services start git-annex
|
||||
Bootstrap failed: 5: Input/output error
|
||||
Try re-running the command as root for richer errors.
|
||||
Error: Failure while executing; `/bin/launchctl bootstrap gui/501 /Users/username/Library/LaunchAgents/homebrew.mxcl.git-annex.plist` exited with 5.
|
||||
```
|
||||
|
||||
Restarting does not show any errors
|
||||
|
||||
```sh
|
||||
$ brew services restart git-annex
|
||||
Stopping `git-annex`... (might take a while)
|
||||
==> Successfully stopped `git-annex` (label: homebrew.mxcl.git-annex)
|
||||
==> Successfully started `git-annex` (label: homebrew.mxcl.git-annex)
|
||||
```
|
||||
|
||||
yet it makes no difference:
|
||||
|
||||
```sh
|
||||
$ brew services
|
||||
Name Status User File
|
||||
git-annex stopped username ~/Library/LaunchAgents/homebrew.mxcl.git-annex.plist
|
||||
```
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="yarikoptic"
|
||||
avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
|
||||
subject="comment 4"
|
||||
date="2024-10-18T20:31:21Z"
|
||||
content="""
|
||||
I agree -- absence of timestamps hinders comprehension of the sequence of events
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 10"""
|
||||
date="2024-10-18T14:28:12Z"
|
||||
content="""
|
||||
The windows build uses stack and the stack.yaml enables the flag, so yes.
|
||||
|
||||
By the way, still waiting on this change: <https://github.com/datalad/git-annex/issues/204>
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""Re: numcopies & force-trusting is ignored by fsck on readonly directory remotes?"""
|
||||
date="2024-10-21T18:54:55Z"
|
||||
content="""
|
||||
importtree=yes remotes are always untrusted. The reason is that
|
||||
something else is assumed to be writing to those remotes, which is what
|
||||
populates them with files. And that could delete or change any file at any
|
||||
time. So if git-annex didn't untrust the remote, and relied on it to hold
|
||||
the only copy of a file, such a change would cause data loss.
|
||||
|
||||
There would need to be a new config setting to add the concept of
|
||||
guaranteed readonly importtree=yes remotes.
|
||||
|
||||
git-annex does not allow --numcopies to be set to 0 as that can cause data
|
||||
loss.
|
||||
"""]]
|
||||
|
|
@ -127,6 +127,19 @@ convenient way to download the content of any key, by using the path
|
|||
This can be combined with `--authenv` or `--authenv-http` to allow
|
||||
anonymous appendonly access, and authenticated remove access.
|
||||
|
||||
* `--unauth-nolocking`
|
||||
|
||||
By default, when `--unauth-readonly` or `--unauth-appendonly` is used,
|
||||
unauthenticated users are allowed to lock content in the repository.
|
||||
This option prevents that.
|
||||
|
||||
Locking content prevents it from being dropped from the repository
|
||||
so it may be that an unauthenticated user abuses that, and this option
|
||||
can be used in such a situation.
|
||||
|
||||
Note that enabling this option will prevent unauthenticated users from
|
||||
dropping content from their other remotes in some cases.
|
||||
|
||||
* `--wideopen`
|
||||
|
||||
Gives unauthenticated users full read+write+remove access to the
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""Re: corruption using git-annex-remote-rclone"""
|
||||
date="2024-10-21T14:35:17Z"
|
||||
content="""
|
||||
This is plausible. git-annex requires that special remotes only show a file
|
||||
as present after a successful upload. If the data store doesn't work that
|
||||
way, the file needs to be uploaded to a temporary name and renamed
|
||||
atomically instead. If that's not possible, the data store is not safe for
|
||||
use by git-annex.
|
||||
|
||||
Given all the different types data stores supported by rclone, this may be
|
||||
difficult, but it's the right thing for the external special remote to do.
|
||||
I think you should file a bug.
|
||||
|
||||
(Does `rclone gitannex` also have this problem?)
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 1"""
|
||||
date="2024-10-21T15:34:26Z"
|
||||
content="""
|
||||
See [[bugs/git-annex-import_imports_outside_of_directory]] where following
|
||||
symlinks on import turned into a serious foot bomb.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
[[!comment format=mdwn
|
||||
username="matrss"
|
||||
avatar="http://cdn.libravatar.org/avatar/59541f50d845e5f81aff06e88a38b9de"
|
||||
subject="comment 2"
|
||||
date="2024-10-21T16:32:18Z"
|
||||
content="""
|
||||
I understand those concerns, but I think skipping all symbolic links is doing too much in this case.
|
||||
|
||||
I am interested in preserving symbolic links that point at other files within the same to-be-imported directory. I think this could be done safely by either resolving the symlink and importing the target file if it is a file within the importtree directory, or just importing the symlink as a symbolic link into git. I am starting to prefer the latter, and since it wouldn't import any actual file content it would even be safe to import symlinks pointing outside of the importtree directory.
|
||||
|
||||
This is what a small part of these datasets looks like:
|
||||
|
||||
```
|
||||
.
|
||||
├── 0630_1
|
||||
│ ├── falcon_avionik.nc -> ../falcon/nasa/avionik/bin/cmet_080630a.nc
|
||||
│ ├── falcon_cmet.nc -> ../falcon/nasa/cmet/bin/f080630a_TG_all_V3.nc
|
||||
│ ├── falcon_fish.nc -> ../falcon/fish/bin/ff080630_1.dat.1_s.CALC.nc
|
||||
│ ├── falcon_jno2.nc -> ../falcon/nasa/jno2/bin/JNO2_final_F080630a.nc
|
||||
<...>
|
||||
├── falcon
|
||||
│ ├── fish
|
||||
│ │ ├── bin
|
||||
│ │ │ ├── ff080630_1.dat.1_s.CALC.nc
|
||||
<...>
|
||||
│ ├── nasa
|
||||
│ │ ├── avionik
|
||||
│ │ │ ├── bin
|
||||
│ │ │ │ ├── cmet_080630a.nc
|
||||
<...>
|
||||
│ │ ├── cmet
|
||||
│ │ │ ├── bin
|
||||
│ │ │ │ ├── f080630a_TG_all_V3.nc
|
||||
<...>
|
||||
│ │ └── jno2
|
||||
│ │ ├── bin
|
||||
│ │ │ ├── JNO2_final_F080630a.nc
|
||||
<...>
|
||||
```
|
||||
|
||||
I would like to meaningfully preserve these symbolic links when using `git annex import`. Right now these paths are simply missing after the import.
|
||||
|
||||
I think @Ilya_Shlyakhter has been asking for this as well in the linked bug report.
|
||||
"""]]
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 2"""
|
||||
date="2024-10-21T15:25:22Z"
|
||||
content="""
|
||||
One simple solution is the put the script somewhere in PATH and then set
|
||||
the config to the name of the script rather than its location.
|
||||
|
||||
In the case of the freeze and thaw hooks, these are run a lot and so
|
||||
git-annex shouldn't be checking if .git/hooks/annex-freeze-content exists
|
||||
every time.
|
||||
"""]]
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
# This is temporarily present to work around this issue.
|
||||
# https://github.com/datalad/git-annex/issues/204
|
||||
flags:
|
||||
git-annex:
|
||||
production: true
|
||||
parallelbuild: true
|
||||
assistant: true
|
||||
pairing: true
|
||||
torrentparser: true
|
||||
magicmime: false
|
||||
dbus: false
|
||||
debuglocks: false
|
||||
benchmark: true
|
||||
crypton: true
|
||||
servant: true
|
||||
packages:
|
||||
- '.'
|
||||
resolver: nightly-2024-07-29
|
||||
extra-deps:
|
||||
- filepath-bytestring-1.4.100.3.2
|
||||
Loading…
Add table
Reference in a new issue