git-lfs apiurl parameter
git-lfs: Added an optional apiurl parameter. This needs version 1.2.5 of the haskell git-lfs library to be used. stack.yaml updated to use that. Note that git-annex enableremote can be used to add apiurl= to an existing git-lfs special remote. To allow unsetting the apiurl and instead use the probed url, support enableremote with apiurl set to an empty string. Sponsored-by: Luke T. Shumaker
This commit is contained in:
parent
dcf2f71696
commit
d394f0b020
6 changed files with 95 additions and 29 deletions
|
@ -4,6 +4,8 @@ git-annex (10.20250116) UNRELEASED; urgency=medium
|
|||
* Allow setting remote.foo.annex-tracking-branch to a branch name
|
||||
that contains "/", as long as it's not a remote tracking branch.
|
||||
* Added OsPath build flag, which speeds up git-annex's operations on files.
|
||||
* git-lfs: Added an optional apiurl parameter.
|
||||
(This needs version 1.2.5 of the haskell git-lfs library to be used.)
|
||||
|
||||
-- Joey Hess <id@joeyh.name> Mon, 20 Jan 2025 10:24:51 -0400
|
||||
|
||||
|
|
|
@ -7,6 +7,7 @@
|
|||
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE RankNTypes #-}
|
||||
{-# LANGUAGE CPP #-}
|
||||
|
||||
module Remote.GitLFS (remote, gen, configKnownUrl) where
|
||||
|
||||
|
@ -66,6 +67,8 @@ remote = specialRemoteType $ RemoteType
|
|||
, configParser = mkRemoteConfigParser
|
||||
[ optionalStringParser urlField
|
||||
(FieldDesc "url of git-lfs repository")
|
||||
, optionalStringParser apiUrlField
|
||||
(FieldDesc "url of LFS API endpoint")
|
||||
]
|
||||
, setup = mySetup
|
||||
, exportSupported = exportUnsupported
|
||||
|
@ -76,6 +79,9 @@ remote = specialRemoteType $ RemoteType
|
|||
urlField :: RemoteConfigField
|
||||
urlField = Accepted "url"
|
||||
|
||||
apiUrlField :: RemoteConfigField
|
||||
apiUrlField = Accepted "apiurl"
|
||||
|
||||
gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> RemoteStateHandle -> Annex (Maybe Remote)
|
||||
gen r u rc gc rs = do
|
||||
c <- parsedRemoteConfig remote rc
|
||||
|
@ -87,7 +93,7 @@ gen r u rc gc rs = do
|
|||
liftIO $ Git.GCrypt.encryptedRemote g r
|
||||
else pure r
|
||||
sem <- liftIO $ MSemN.new 1
|
||||
h <- liftIO $ newTVarIO $ LFSHandle Nothing Nothing sem r' gc
|
||||
h <- liftIO $ newTVarIO $ LFSHandle Nothing Nothing sem r' gc c
|
||||
cst <- remoteCost gc c expensiveRemoteCost
|
||||
let specialcfg = (specialRemoteCfg c)
|
||||
-- chunking would not improve git-lfs
|
||||
|
@ -219,6 +225,7 @@ data LFSHandle = LFSHandle
|
|||
, getEndPointLock :: MSemN.MSemN Int
|
||||
, remoteRepo :: Git.Repo
|
||||
, remoteGitConfig :: RemoteGitConfig
|
||||
, remoteConfigs :: ParsedRemoteConfig
|
||||
}
|
||||
|
||||
-- Only let one thread at a time do endpoint discovery.
|
||||
|
@ -230,10 +237,24 @@ withEndPointLock h = bracket_
|
|||
l = getEndPointLock h
|
||||
|
||||
discoverLFSEndpoint :: LFS.TransferRequestOperation -> LFSHandle -> Annex (Maybe LFS.Endpoint)
|
||||
discoverLFSEndpoint tro h
|
||||
| Git.repoIsSsh r = gossh
|
||||
| Git.repoIsHttp r = gohttp
|
||||
| otherwise = unsupportedurischeme
|
||||
discoverLFSEndpoint tro h =
|
||||
case fmap fromProposedAccepted $ M.lookup apiUrlField (unparsedRemoteConfig (remoteConfigs h)) of
|
||||
Just apiurl | not (null apiurl) -> case parseURIRelaxed apiurl of
|
||||
Nothing -> unsupportedurischeme
|
||||
#if MIN_VERSION_git_lfs(1,2,5)
|
||||
Just apiuri -> case LFS.mkEndpoint apiuri of
|
||||
Just endpoint -> checkhttpauth endpoint
|
||||
Nothing -> unsupportedurischeme
|
||||
#else
|
||||
#warning Building with old version of git-lfs, apiurl= will not be supported
|
||||
Just _ -> do
|
||||
warning $ "Unable to use configured apiurl because this git-annex is not built with version 1.2.5 of the haskell git-lfs library."
|
||||
return Nothing
|
||||
#endif
|
||||
_
|
||||
| Git.repoIsSsh r -> gossh
|
||||
| Git.repoIsHttp r -> gohttp
|
||||
| otherwise -> unsupportedurischeme
|
||||
where
|
||||
r = remoteRepo h
|
||||
lfsrepouri = case Git.location r of
|
||||
|
@ -278,31 +299,33 @@ discoverLFSEndpoint tro h
|
|||
warning "unexpected response from git-lfs remote when doing ssh endpoint discovery"
|
||||
return Nothing
|
||||
Just endpoint -> return (Just endpoint)
|
||||
|
||||
|
||||
gohttp = case LFS.guessEndpoint lfsrepouri of
|
||||
Nothing -> unsupportedurischeme
|
||||
Just endpoint -> checkhttpauth endpoint
|
||||
|
||||
-- The endpoint may or may not need http basic authentication,
|
||||
-- which involves using git-credential to prompt for the password.
|
||||
--
|
||||
-- To determine if it does, make a download or upload request to
|
||||
-- it, not including any objects in the request, and see if
|
||||
-- the server requests authentication.
|
||||
gohttp = case LFS.guessEndpoint lfsrepouri of
|
||||
Nothing -> unsupportedurischeme
|
||||
Just endpoint -> do
|
||||
let testreq = LFS.startTransferRequest endpoint transfernothing
|
||||
flip catchNonAsync (const (returnendpoint endpoint)) $ do
|
||||
resp <- makeSmallAPIRequest testreq
|
||||
if needauth (responseStatus resp)
|
||||
then do
|
||||
cred <- prompt $ inRepo $ Git.getUrlCredential (show lfsrepouri)
|
||||
let endpoint' = addbasicauth (Git.credentialBasicAuth cred) endpoint
|
||||
let testreq' = LFS.startTransferRequest endpoint' transfernothing
|
||||
flip catchNonAsync (const (returnendpoint endpoint')) $ do
|
||||
resp' <- makeSmallAPIRequest testreq'
|
||||
inRepo $ if needauth (responseStatus resp')
|
||||
then Git.rejectUrlCredential cred
|
||||
else Git.approveUrlCredential cred
|
||||
returnendpoint endpoint'
|
||||
else returnendpoint endpoint
|
||||
checkhttpauth endpoint = do
|
||||
let testreq = LFS.startTransferRequest endpoint transfernothing
|
||||
flip catchNonAsync (const (returnendpoint endpoint)) $ do
|
||||
resp <- makeSmallAPIRequest testreq
|
||||
if needauth (responseStatus resp)
|
||||
then do
|
||||
cred <- prompt $ inRepo $ Git.getUrlCredential (show lfsrepouri)
|
||||
let endpoint' = addbasicauth (Git.credentialBasicAuth cred) endpoint
|
||||
let testreq' = LFS.startTransferRequest endpoint' transfernothing
|
||||
flip catchNonAsync (const (returnendpoint endpoint')) $ do
|
||||
resp' <- makeSmallAPIRequest testreq'
|
||||
inRepo $ if needauth (responseStatus resp')
|
||||
then Git.rejectUrlCredential cred
|
||||
else Git.approveUrlCredential cred
|
||||
returnendpoint endpoint'
|
||||
else returnendpoint endpoint
|
||||
where
|
||||
transfernothing = LFS.TransferRequest
|
||||
{ LFS.req_operation = tro
|
||||
|
@ -314,10 +337,10 @@ discoverLFSEndpoint tro h
|
|||
|
||||
needauth status = status == unauthorized401
|
||||
|
||||
addbasicauth (Just ba) endpoint =
|
||||
LFS.modifyEndpointRequest endpoint $
|
||||
addbasicauth (Just ba) endpoint' =
|
||||
LFS.modifyEndpointRequest endpoint' $
|
||||
applyBasicAuth' ba
|
||||
addbasicauth Nothing endpoint = endpoint
|
||||
addbasicauth Nothing endpoint' = endpoint'
|
||||
|
||||
-- The endpoint is cached for later use.
|
||||
getLFSEndpoint :: LFS.TransferRequestOperation -> TVar LFSHandle -> Annex (Maybe LFS.Endpoint)
|
||||
|
|
|
@ -66,3 +66,5 @@ Nil
|
|||
### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
|
||||
|
||||
Love git-annex. Long time supporter.
|
||||
|
||||
> [[fixed|done]] --[[Joey]]
|
||||
|
|
|
@ -0,0 +1,35 @@
|
|||
[[!comment format=mdwn
|
||||
username="joey"
|
||||
subject="""comment 1"""
|
||||
date="2025-02-18T16:23:23Z"
|
||||
content="""
|
||||
LFS uses http basic auth, so using it over http probably allows
|
||||
any man in the middle to take over your storage.
|
||||
|
||||
With that rationalle, <https://hackage.haskell.org/package/git-lfs>
|
||||
hardcodes a https url at LFS server discovery time. And I don't think it
|
||||
would be secure for it to do anything else by default; people do clone
|
||||
git over http and it would be a security hole if LFS then exposed their
|
||||
password.
|
||||
|
||||
In your case, you're using a nonstandard http port, and it's continuing
|
||||
to use that same port for https. That seems unlikely to work in almost any
|
||||
situation. Perhaps a http url should only be upgraded to https when
|
||||
it's using a standard port. Or perhaps the nonstandard port should be
|
||||
replaced with the standard https port. I felt that the latter was less
|
||||
likely to result in security issues, and was more consistent, so I've gone
|
||||
with that approach. That change is in version 1.2.4 of
|
||||
<https://hackage.haskell.org/package/git-lfs>.
|
||||
|
||||
git-lfs has git configs `lfs.url` and `remote.<name>.lfsurl`
|
||||
that allow the user to specify the API endpoint to use. The special
|
||||
remote's url= parameter is the git repository url, not the API endpoint.
|
||||
So I think that to handle your use case, it makes sense to add an optional
|
||||
apiurl= parameter to the special remote, which corresponds to those git
|
||||
configs.
|
||||
|
||||
Unfortunately, adding apiurl= needed a new version 1.2.5 of
|
||||
<https://hackage.haskell.org/package/git-lfs>, so it will only
|
||||
be available in builds of git-annex that use that version of the library.
|
||||
Which will take a while to reach all builds.
|
||||
"""]]
|
|
@ -9,7 +9,7 @@ These parameters can be passed to `git annex initremote` to configure
|
|||
the git-lfs special remote:
|
||||
|
||||
* `url` - Required. The url to the git-lfs repository to use.
|
||||
Can be either a ssh url (scp-style is also accepted) or a http url.
|
||||
Can be either a ssh url (scp-style is also accepted) or a https url.
|
||||
|
||||
* `encryption` - One of "none", "hybrid", "shared", or "pubkey".
|
||||
Required. See [[encryption]]. Also see the encryption notes below.
|
||||
|
@ -18,6 +18,10 @@ the git-lfs special remote:
|
|||
git-annex stores in the repository, as well as to encrypt the git
|
||||
repository itself when using gcrypt.
|
||||
|
||||
* `apiurl` - Optional. The url to the LFS API endpoint. This can be a https
|
||||
or a http url. When this is not specified, or is not set to an url,
|
||||
the API endpoint url is guessed based on the url parameter.
|
||||
|
||||
## efficiency note
|
||||
|
||||
Since git-lfs uses SHA256 checksums, git-annex needs to keep track of the
|
||||
|
|
|
@ -18,7 +18,7 @@ resolver: nightly-2025-01-20
|
|||
extra-deps:
|
||||
- filepath-bytestring-1.5.2.0.2
|
||||
- aws-0.24.4
|
||||
- git-lfs-1.2.3
|
||||
- git-lfs-1.2.5
|
||||
- feed-1.3.2.1
|
||||
allow-newer: true
|
||||
allow-newer-deps:
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue