git-lfs apiurl parameter

git-lfs: Added an optional apiurl parameter.

This needs version 1.2.5 of the haskell git-lfs library to be used.
stack.yaml updated to use that.

Note that git-annex enableremote can be used to add apiurl= to an existing
git-lfs special remote. To allow unsetting the apiurl and instead use
the probed url, support enableremote with apiurl set to an empty string.

Sponsored-by: Luke T. Shumaker
This commit is contained in:
Joey Hess 2025-02-18 14:11:11 -04:00
parent dcf2f71696
commit d394f0b020
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
6 changed files with 95 additions and 29 deletions

View file

@ -4,6 +4,8 @@ git-annex (10.20250116) UNRELEASED; urgency=medium
* Allow setting remote.foo.annex-tracking-branch to a branch name
that contains "/", as long as it's not a remote tracking branch.
* Added OsPath build flag, which speeds up git-annex's operations on files.
* git-lfs: Added an optional apiurl parameter.
(This needs version 1.2.5 of the haskell git-lfs library to be used.)
-- Joey Hess <id@joeyh.name> Mon, 20 Jan 2025 10:24:51 -0400

View file

@ -7,6 +7,7 @@
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE RankNTypes #-}
{-# LANGUAGE CPP #-}
module Remote.GitLFS (remote, gen, configKnownUrl) where
@ -66,6 +67,8 @@ remote = specialRemoteType $ RemoteType
, configParser = mkRemoteConfigParser
[ optionalStringParser urlField
(FieldDesc "url of git-lfs repository")
, optionalStringParser apiUrlField
(FieldDesc "url of LFS API endpoint")
]
, setup = mySetup
, exportSupported = exportUnsupported
@ -76,6 +79,9 @@ remote = specialRemoteType $ RemoteType
urlField :: RemoteConfigField
urlField = Accepted "url"
apiUrlField :: RemoteConfigField
apiUrlField = Accepted "apiurl"
gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> RemoteStateHandle -> Annex (Maybe Remote)
gen r u rc gc rs = do
c <- parsedRemoteConfig remote rc
@ -87,7 +93,7 @@ gen r u rc gc rs = do
liftIO $ Git.GCrypt.encryptedRemote g r
else pure r
sem <- liftIO $ MSemN.new 1
h <- liftIO $ newTVarIO $ LFSHandle Nothing Nothing sem r' gc
h <- liftIO $ newTVarIO $ LFSHandle Nothing Nothing sem r' gc c
cst <- remoteCost gc c expensiveRemoteCost
let specialcfg = (specialRemoteCfg c)
-- chunking would not improve git-lfs
@ -219,6 +225,7 @@ data LFSHandle = LFSHandle
, getEndPointLock :: MSemN.MSemN Int
, remoteRepo :: Git.Repo
, remoteGitConfig :: RemoteGitConfig
, remoteConfigs :: ParsedRemoteConfig
}
-- Only let one thread at a time do endpoint discovery.
@ -230,10 +237,24 @@ withEndPointLock h = bracket_
l = getEndPointLock h
discoverLFSEndpoint :: LFS.TransferRequestOperation -> LFSHandle -> Annex (Maybe LFS.Endpoint)
discoverLFSEndpoint tro h
| Git.repoIsSsh r = gossh
| Git.repoIsHttp r = gohttp
| otherwise = unsupportedurischeme
discoverLFSEndpoint tro h =
case fmap fromProposedAccepted $ M.lookup apiUrlField (unparsedRemoteConfig (remoteConfigs h)) of
Just apiurl | not (null apiurl) -> case parseURIRelaxed apiurl of
Nothing -> unsupportedurischeme
#if MIN_VERSION_git_lfs(1,2,5)
Just apiuri -> case LFS.mkEndpoint apiuri of
Just endpoint -> checkhttpauth endpoint
Nothing -> unsupportedurischeme
#else
#warning Building with old version of git-lfs, apiurl= will not be supported
Just _ -> do
warning $ "Unable to use configured apiurl because this git-annex is not built with version 1.2.5 of the haskell git-lfs library."
return Nothing
#endif
_
| Git.repoIsSsh r -> gossh
| Git.repoIsHttp r -> gohttp
| otherwise -> unsupportedurischeme
where
r = remoteRepo h
lfsrepouri = case Git.location r of
@ -278,31 +299,33 @@ discoverLFSEndpoint tro h
warning "unexpected response from git-lfs remote when doing ssh endpoint discovery"
return Nothing
Just endpoint -> return (Just endpoint)
gohttp = case LFS.guessEndpoint lfsrepouri of
Nothing -> unsupportedurischeme
Just endpoint -> checkhttpauth endpoint
-- The endpoint may or may not need http basic authentication,
-- which involves using git-credential to prompt for the password.
--
-- To determine if it does, make a download or upload request to
-- it, not including any objects in the request, and see if
-- the server requests authentication.
gohttp = case LFS.guessEndpoint lfsrepouri of
Nothing -> unsupportedurischeme
Just endpoint -> do
let testreq = LFS.startTransferRequest endpoint transfernothing
flip catchNonAsync (const (returnendpoint endpoint)) $ do
resp <- makeSmallAPIRequest testreq
if needauth (responseStatus resp)
then do
cred <- prompt $ inRepo $ Git.getUrlCredential (show lfsrepouri)
let endpoint' = addbasicauth (Git.credentialBasicAuth cred) endpoint
let testreq' = LFS.startTransferRequest endpoint' transfernothing
flip catchNonAsync (const (returnendpoint endpoint')) $ do
resp' <- makeSmallAPIRequest testreq'
inRepo $ if needauth (responseStatus resp')
then Git.rejectUrlCredential cred
else Git.approveUrlCredential cred
returnendpoint endpoint'
else returnendpoint endpoint
checkhttpauth endpoint = do
let testreq = LFS.startTransferRequest endpoint transfernothing
flip catchNonAsync (const (returnendpoint endpoint)) $ do
resp <- makeSmallAPIRequest testreq
if needauth (responseStatus resp)
then do
cred <- prompt $ inRepo $ Git.getUrlCredential (show lfsrepouri)
let endpoint' = addbasicauth (Git.credentialBasicAuth cred) endpoint
let testreq' = LFS.startTransferRequest endpoint' transfernothing
flip catchNonAsync (const (returnendpoint endpoint')) $ do
resp' <- makeSmallAPIRequest testreq'
inRepo $ if needauth (responseStatus resp')
then Git.rejectUrlCredential cred
else Git.approveUrlCredential cred
returnendpoint endpoint'
else returnendpoint endpoint
where
transfernothing = LFS.TransferRequest
{ LFS.req_operation = tro
@ -314,10 +337,10 @@ discoverLFSEndpoint tro h
needauth status = status == unauthorized401
addbasicauth (Just ba) endpoint =
LFS.modifyEndpointRequest endpoint $
addbasicauth (Just ba) endpoint' =
LFS.modifyEndpointRequest endpoint' $
applyBasicAuth' ba
addbasicauth Nothing endpoint = endpoint
addbasicauth Nothing endpoint' = endpoint'
-- The endpoint is cached for later use.
getLFSEndpoint :: LFS.TransferRequestOperation -> TVar LFSHandle -> Annex (Maybe LFS.Endpoint)

View file

@ -66,3 +66,5 @@ Nil
### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
Love git-annex. Long time supporter.
> [[fixed|done]] --[[Joey]]

View file

@ -0,0 +1,35 @@
[[!comment format=mdwn
username="joey"
subject="""comment 1"""
date="2025-02-18T16:23:23Z"
content="""
LFS uses http basic auth, so using it over http probably allows
any man in the middle to take over your storage.
With that rationalle, <https://hackage.haskell.org/package/git-lfs>
hardcodes a https url at LFS server discovery time. And I don't think it
would be secure for it to do anything else by default; people do clone
git over http and it would be a security hole if LFS then exposed their
password.
In your case, you're using a nonstandard http port, and it's continuing
to use that same port for https. That seems unlikely to work in almost any
situation. Perhaps a http url should only be upgraded to https when
it's using a standard port. Or perhaps the nonstandard port should be
replaced with the standard https port. I felt that the latter was less
likely to result in security issues, and was more consistent, so I've gone
with that approach. That change is in version 1.2.4 of
<https://hackage.haskell.org/package/git-lfs>.
git-lfs has git configs `lfs.url` and `remote.<name>.lfsurl`
that allow the user to specify the API endpoint to use. The special
remote's url= parameter is the git repository url, not the API endpoint.
So I think that to handle your use case, it makes sense to add an optional
apiurl= parameter to the special remote, which corresponds to those git
configs.
Unfortunately, adding apiurl= needed a new version 1.2.5 of
<https://hackage.haskell.org/package/git-lfs>, so it will only
be available in builds of git-annex that use that version of the library.
Which will take a while to reach all builds.
"""]]

View file

@ -9,7 +9,7 @@ These parameters can be passed to `git annex initremote` to configure
the git-lfs special remote:
* `url` - Required. The url to the git-lfs repository to use.
Can be either a ssh url (scp-style is also accepted) or a http url.
Can be either a ssh url (scp-style is also accepted) or a https url.
* `encryption` - One of "none", "hybrid", "shared", or "pubkey".
Required. See [[encryption]]. Also see the encryption notes below.
@ -18,6 +18,10 @@ the git-lfs special remote:
git-annex stores in the repository, as well as to encrypt the git
repository itself when using gcrypt.
* `apiurl` - Optional. The url to the LFS API endpoint. This can be a https
or a http url. When this is not specified, or is not set to an url,
the API endpoint url is guessed based on the url parameter.
## efficiency note
Since git-lfs uses SHA256 checksums, git-annex needs to keep track of the

View file

@ -18,7 +18,7 @@ resolver: nightly-2025-01-20
extra-deps:
- filepath-bytestring-1.5.2.0.2
- aws-0.24.4
- git-lfs-1.2.3
- git-lfs-1.2.5
- feed-1.3.2.1
allow-newer: true
allow-newer-deps: