From d394f0b020488bb625a123d6ef49f1a0153437e2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Feb 2025 14:11:11 -0400 Subject: [PATCH] git-lfs apiurl parameter git-lfs: Added an optional apiurl parameter. This needs version 1.2.5 of the haskell git-lfs library to be used. stack.yaml updated to use that. Note that git-annex enableremote can be used to add apiurl= to an existing git-lfs special remote. To allow unsetting the apiurl and instead use the probed url, support enableremote with apiurl set to an empty string. Sponsored-by: Luke T. Shumaker --- CHANGELOG | 2 + Remote/GitLFS.hs | 77 ++++++++++++------- .../git-lfs_special_insists_on_https.mdwn | 2 + ..._5a80951f61874589a4df78f6d78fbfd1._comment | 35 +++++++++ doc/special_remotes/git-lfs.mdwn | 6 +- stack.yaml | 2 +- 6 files changed, 95 insertions(+), 29 deletions(-) create mode 100644 doc/bugs/git-lfs_special_insists_on_https/comment_1_5a80951f61874589a4df78f6d78fbfd1._comment diff --git a/CHANGELOG b/CHANGELOG index f720bf9850..475277f8f4 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,6 +4,8 @@ git-annex (10.20250116) UNRELEASED; urgency=medium * Allow setting remote.foo.annex-tracking-branch to a branch name that contains "/", as long as it's not a remote tracking branch. * Added OsPath build flag, which speeds up git-annex's operations on files. + * git-lfs: Added an optional apiurl parameter. + (This needs version 1.2.5 of the haskell git-lfs library to be used.) -- Joey Hess Mon, 20 Jan 2025 10:24:51 -0400 diff --git a/Remote/GitLFS.hs b/Remote/GitLFS.hs index 4103309286..372ba62a9b 100644 --- a/Remote/GitLFS.hs +++ b/Remote/GitLFS.hs @@ -7,6 +7,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-} +{-# LANGUAGE CPP #-} module Remote.GitLFS (remote, gen, configKnownUrl) where @@ -66,6 +67,8 @@ remote = specialRemoteType $ RemoteType , configParser = mkRemoteConfigParser [ optionalStringParser urlField (FieldDesc "url of git-lfs repository") + , optionalStringParser apiUrlField + (FieldDesc "url of LFS API endpoint") ] , setup = mySetup , exportSupported = exportUnsupported @@ -76,6 +79,9 @@ remote = specialRemoteType $ RemoteType urlField :: RemoteConfigField urlField = Accepted "url" +apiUrlField :: RemoteConfigField +apiUrlField = Accepted "apiurl" + gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> RemoteStateHandle -> Annex (Maybe Remote) gen r u rc gc rs = do c <- parsedRemoteConfig remote rc @@ -87,7 +93,7 @@ gen r u rc gc rs = do liftIO $ Git.GCrypt.encryptedRemote g r else pure r sem <- liftIO $ MSemN.new 1 - h <- liftIO $ newTVarIO $ LFSHandle Nothing Nothing sem r' gc + h <- liftIO $ newTVarIO $ LFSHandle Nothing Nothing sem r' gc c cst <- remoteCost gc c expensiveRemoteCost let specialcfg = (specialRemoteCfg c) -- chunking would not improve git-lfs @@ -219,6 +225,7 @@ data LFSHandle = LFSHandle , getEndPointLock :: MSemN.MSemN Int , remoteRepo :: Git.Repo , remoteGitConfig :: RemoteGitConfig + , remoteConfigs :: ParsedRemoteConfig } -- Only let one thread at a time do endpoint discovery. @@ -230,10 +237,24 @@ withEndPointLock h = bracket_ l = getEndPointLock h discoverLFSEndpoint :: LFS.TransferRequestOperation -> LFSHandle -> Annex (Maybe LFS.Endpoint) -discoverLFSEndpoint tro h - | Git.repoIsSsh r = gossh - | Git.repoIsHttp r = gohttp - | otherwise = unsupportedurischeme +discoverLFSEndpoint tro h = + case fmap fromProposedAccepted $ M.lookup apiUrlField (unparsedRemoteConfig (remoteConfigs h)) of + Just apiurl | not (null apiurl) -> case parseURIRelaxed apiurl of + Nothing -> unsupportedurischeme +#if MIN_VERSION_git_lfs(1,2,5) + Just apiuri -> case LFS.mkEndpoint apiuri of + Just endpoint -> checkhttpauth endpoint + Nothing -> unsupportedurischeme +#else +#warning Building with old version of git-lfs, apiurl= will not be supported + Just _ -> do + warning $ "Unable to use configured apiurl because this git-annex is not built with version 1.2.5 of the haskell git-lfs library." + return Nothing +#endif + _ + | Git.repoIsSsh r -> gossh + | Git.repoIsHttp r -> gohttp + | otherwise -> unsupportedurischeme where r = remoteRepo h lfsrepouri = case Git.location r of @@ -278,31 +299,33 @@ discoverLFSEndpoint tro h warning "unexpected response from git-lfs remote when doing ssh endpoint discovery" return Nothing Just endpoint -> return (Just endpoint) - + + gohttp = case LFS.guessEndpoint lfsrepouri of + Nothing -> unsupportedurischeme + Just endpoint -> checkhttpauth endpoint + -- The endpoint may or may not need http basic authentication, -- which involves using git-credential to prompt for the password. -- -- To determine if it does, make a download or upload request to -- it, not including any objects in the request, and see if -- the server requests authentication. - gohttp = case LFS.guessEndpoint lfsrepouri of - Nothing -> unsupportedurischeme - Just endpoint -> do - let testreq = LFS.startTransferRequest endpoint transfernothing - flip catchNonAsync (const (returnendpoint endpoint)) $ do - resp <- makeSmallAPIRequest testreq - if needauth (responseStatus resp) - then do - cred <- prompt $ inRepo $ Git.getUrlCredential (show lfsrepouri) - let endpoint' = addbasicauth (Git.credentialBasicAuth cred) endpoint - let testreq' = LFS.startTransferRequest endpoint' transfernothing - flip catchNonAsync (const (returnendpoint endpoint')) $ do - resp' <- makeSmallAPIRequest testreq' - inRepo $ if needauth (responseStatus resp') - then Git.rejectUrlCredential cred - else Git.approveUrlCredential cred - returnendpoint endpoint' - else returnendpoint endpoint + checkhttpauth endpoint = do + let testreq = LFS.startTransferRequest endpoint transfernothing + flip catchNonAsync (const (returnendpoint endpoint)) $ do + resp <- makeSmallAPIRequest testreq + if needauth (responseStatus resp) + then do + cred <- prompt $ inRepo $ Git.getUrlCredential (show lfsrepouri) + let endpoint' = addbasicauth (Git.credentialBasicAuth cred) endpoint + let testreq' = LFS.startTransferRequest endpoint' transfernothing + flip catchNonAsync (const (returnendpoint endpoint')) $ do + resp' <- makeSmallAPIRequest testreq' + inRepo $ if needauth (responseStatus resp') + then Git.rejectUrlCredential cred + else Git.approveUrlCredential cred + returnendpoint endpoint' + else returnendpoint endpoint where transfernothing = LFS.TransferRequest { LFS.req_operation = tro @@ -314,10 +337,10 @@ discoverLFSEndpoint tro h needauth status = status == unauthorized401 - addbasicauth (Just ba) endpoint = - LFS.modifyEndpointRequest endpoint $ + addbasicauth (Just ba) endpoint' = + LFS.modifyEndpointRequest endpoint' $ applyBasicAuth' ba - addbasicauth Nothing endpoint = endpoint + addbasicauth Nothing endpoint' = endpoint' -- The endpoint is cached for later use. getLFSEndpoint :: LFS.TransferRequestOperation -> TVar LFSHandle -> Annex (Maybe LFS.Endpoint) diff --git a/doc/bugs/git-lfs_special_insists_on_https.mdwn b/doc/bugs/git-lfs_special_insists_on_https.mdwn index 62c62f438a..448cab2fbf 100644 --- a/doc/bugs/git-lfs_special_insists_on_https.mdwn +++ b/doc/bugs/git-lfs_special_insists_on_https.mdwn @@ -66,3 +66,5 @@ Nil ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) Love git-annex. Long time supporter. + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/git-lfs_special_insists_on_https/comment_1_5a80951f61874589a4df78f6d78fbfd1._comment b/doc/bugs/git-lfs_special_insists_on_https/comment_1_5a80951f61874589a4df78f6d78fbfd1._comment new file mode 100644 index 0000000000..98d93dd68e --- /dev/null +++ b/doc/bugs/git-lfs_special_insists_on_https/comment_1_5a80951f61874589a4df78f6d78fbfd1._comment @@ -0,0 +1,35 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2025-02-18T16:23:23Z" + content=""" +LFS uses http basic auth, so using it over http probably allows +any man in the middle to take over your storage. + +With that rationalle, +hardcodes a https url at LFS server discovery time. And I don't think it +would be secure for it to do anything else by default; people do clone +git over http and it would be a security hole if LFS then exposed their +password. + +In your case, you're using a nonstandard http port, and it's continuing +to use that same port for https. That seems unlikely to work in almost any +situation. Perhaps a http url should only be upgraded to https when +it's using a standard port. Or perhaps the nonstandard port should be +replaced with the standard https port. I felt that the latter was less +likely to result in security issues, and was more consistent, so I've gone +with that approach. That change is in version 1.2.4 of +. + +git-lfs has git configs `lfs.url` and `remote..lfsurl` +that allow the user to specify the API endpoint to use. The special +remote's url= parameter is the git repository url, not the API endpoint. +So I think that to handle your use case, it makes sense to add an optional +apiurl= parameter to the special remote, which corresponds to those git +configs. + +Unfortunately, adding apiurl= needed a new version 1.2.5 of +, so it will only +be available in builds of git-annex that use that version of the library. +Which will take a while to reach all builds. +"""]] diff --git a/doc/special_remotes/git-lfs.mdwn b/doc/special_remotes/git-lfs.mdwn index 3a6d851de1..ac737eff26 100644 --- a/doc/special_remotes/git-lfs.mdwn +++ b/doc/special_remotes/git-lfs.mdwn @@ -9,7 +9,7 @@ These parameters can be passed to `git annex initremote` to configure the git-lfs special remote: * `url` - Required. The url to the git-lfs repository to use. - Can be either a ssh url (scp-style is also accepted) or a http url. + Can be either a ssh url (scp-style is also accepted) or a https url. * `encryption` - One of "none", "hybrid", "shared", or "pubkey". Required. See [[encryption]]. Also see the encryption notes below. @@ -18,6 +18,10 @@ the git-lfs special remote: git-annex stores in the repository, as well as to encrypt the git repository itself when using gcrypt. +* `apiurl` - Optional. The url to the LFS API endpoint. This can be a https + or a http url. When this is not specified, or is not set to an url, + the API endpoint url is guessed based on the url parameter. + ## efficiency note Since git-lfs uses SHA256 checksums, git-annex needs to keep track of the diff --git a/stack.yaml b/stack.yaml index 5ff6f33d09..0ba403a886 100644 --- a/stack.yaml +++ b/stack.yaml @@ -18,7 +18,7 @@ resolver: nightly-2025-01-20 extra-deps: - filepath-bytestring-1.5.2.0.2 - aws-0.24.4 -- git-lfs-1.2.3 +- git-lfs-1.2.5 - feed-1.3.2.1 allow-newer: true allow-newer-deps: