git-lfs apiurl parameter

git-lfs: Added an optional apiurl parameter.

This needs version 1.2.5 of the haskell git-lfs library to be used.
stack.yaml updated to use that.

Note that git-annex enableremote can be used to add apiurl= to an existing
git-lfs special remote. To allow unsetting the apiurl and instead use
the probed url, support enableremote with apiurl set to an empty string.

Sponsored-by: Luke T. Shumaker

CHANGELOG

@@ -4,6 +4,8 @@ git-annex (10.20250116) UNRELEASED; urgency=medium
   * Allow setting remote.foo.annex-tracking-branch to a branch name
     that contains "/", as long as it's not a remote tracking branch.
   * Added OsPath build flag, which speeds up git-annex's operations on files.
+  * git-lfs: Added an optional apiurl parameter.
+    (This needs version 1.2.5 of the haskell git-lfs library to be used.)
 
  -- Joey Hess <id@joeyh.name>  Mon, 20 Jan 2025 10:24:51 -0400
 

Remote/GitLFS.hs

@@ -7,6 +7,7 @@
 
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE CPP #-}
 
 module Remote.GitLFS (remote, gen, configKnownUrl) where
 
@@ -66,6 +67,8 @@ remote = specialRemoteType $ RemoteType
 	, configParser = mkRemoteConfigParser
 		[ optionalStringParser urlField
 			(FieldDesc "url of git-lfs repository")
+		, optionalStringParser apiUrlField
+			(FieldDesc "url of LFS API endpoint")
 		]
 	, setup = mySetup
 	, exportSupported = exportUnsupported
@@ -76,6 +79,9 @@ remote = specialRemoteType $ RemoteType
 urlField :: RemoteConfigField
 urlField = Accepted "url"
 
+apiUrlField :: RemoteConfigField
+apiUrlField = Accepted "apiurl"
+
 gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> RemoteStateHandle -> Annex (Maybe Remote)
 gen r u rc gc rs = do
 	c <- parsedRemoteConfig remote rc
@@ -87,7 +93,7 @@ gen r u rc gc rs = do
 			liftIO $ Git.GCrypt.encryptedRemote g r
 		else pure r
 	sem <- liftIO $ MSemN.new 1
-	h <- liftIO $ newTVarIO $ LFSHandle Nothing Nothing sem r' gc
+	h <- liftIO $ newTVarIO $ LFSHandle Nothing Nothing sem r' gc c
 	cst <- remoteCost gc c expensiveRemoteCost
 	let specialcfg = (specialRemoteCfg c)
 		-- chunking would not improve git-lfs
@@ -219,6 +225,7 @@ data LFSHandle = LFSHandle
 	, getEndPointLock :: MSemN.MSemN Int 
 	, remoteRepo :: Git.Repo
 	, remoteGitConfig :: RemoteGitConfig
+	, remoteConfigs :: ParsedRemoteConfig
 	}
 
 -- Only let one thread at a time do endpoint discovery.
@@ -230,10 +237,24 @@ withEndPointLock h = bracket_
 	l = getEndPointLock h
 
 discoverLFSEndpoint :: LFS.TransferRequestOperation -> LFSHandle -> Annex (Maybe LFS.Endpoint)
-discoverLFSEndpoint tro h
+discoverLFSEndpoint tro h =
-	| Git.repoIsSsh r = gossh
+	case fmap fromProposedAccepted $ M.lookup apiUrlField (unparsedRemoteConfig (remoteConfigs h)) of
-	| Git.repoIsHttp r = gohttp
+		Just apiurl | not (null apiurl) -> case parseURIRelaxed apiurl of
-	| otherwise = unsupportedurischeme
+			Nothing -> unsupportedurischeme
+#if MIN_VERSION_git_lfs(1,2,5)
+			Just apiuri -> case LFS.mkEndpoint apiuri of
+				Just endpoint -> checkhttpauth endpoint
+				Nothing -> unsupportedurischeme
+#else
+#warning Building with old version of git-lfs, apiurl= will not be supported
+			Just _ -> do
+				warning $ "Unable to use configured apiurl because this git-annex is not built with version 1.2.5 of the haskell git-lfs library."
+				return Nothing
+#endif
+		_
+			| Git.repoIsSsh r -> gossh
+			| Git.repoIsHttp r -> gohttp
+			| otherwise -> unsupportedurischeme
   where
   	r = remoteRepo h
 	lfsrepouri = case Git.location r of
@@ -278,31 +299,33 @@ discoverLFSEndpoint tro h
 						warning "unexpected response from git-lfs remote when doing ssh endpoint discovery"
 						return Nothing
 					Just endpoint -> return (Just endpoint)
-	
+
+	gohttp = case LFS.guessEndpoint lfsrepouri of
+		Nothing -> unsupportedurischeme
+		Just endpoint -> checkhttpauth endpoint
+
 	-- The endpoint may or may not need http basic authentication,
 	-- which involves using git-credential to prompt for the password.
 	--
 	-- To determine if it does, make a download or upload request to
 	-- it, not including any objects in the request, and see if
 	-- the server requests authentication.
-	gohttp = case LFS.guessEndpoint lfsrepouri of
+	checkhttpauth endpoint = do
-		Nothing -> unsupportedurischeme
+		let testreq = LFS.startTransferRequest endpoint transfernothing
-		Just endpoint -> do
+		flip catchNonAsync (const (returnendpoint endpoint)) $ do
-			let testreq = LFS.startTransferRequest endpoint transfernothing
+			resp <- makeSmallAPIRequest testreq
-			flip catchNonAsync (const (returnendpoint endpoint)) $ do
+			if needauth (responseStatus resp)
-				resp <- makeSmallAPIRequest testreq
+				then do
-				if needauth (responseStatus resp)
+					cred <- prompt $ inRepo $ Git.getUrlCredential (show lfsrepouri)
-					then do
+					let endpoint' = addbasicauth (Git.credentialBasicAuth cred) endpoint
-						cred <- prompt $ inRepo $ Git.getUrlCredential (show lfsrepouri)
+					let testreq' = LFS.startTransferRequest endpoint' transfernothing
-						let endpoint' = addbasicauth (Git.credentialBasicAuth cred) endpoint
+					flip catchNonAsync (const (returnendpoint endpoint')) $ do
-						let testreq' = LFS.startTransferRequest endpoint' transfernothing
+						resp' <- makeSmallAPIRequest testreq'
-						flip catchNonAsync (const (returnendpoint endpoint')) $ do
+						inRepo $ if needauth (responseStatus resp')
-							resp' <- makeSmallAPIRequest testreq'
+							then Git.rejectUrlCredential cred
-							inRepo $ if needauth (responseStatus resp')
+							else Git.approveUrlCredential cred
-								then Git.rejectUrlCredential cred
+						returnendpoint endpoint'
-								else Git.approveUrlCredential cred
+				else returnendpoint endpoint
-							returnendpoint endpoint'
-					else returnendpoint endpoint
 	  where
 	  	transfernothing = LFS.TransferRequest
 			{ LFS.req_operation = tro
@@ -314,10 +337,10 @@ discoverLFSEndpoint tro h
 
 		needauth status = status == unauthorized401
 
-		addbasicauth (Just ba) endpoint =
+		addbasicauth (Just ba) endpoint' =
-			LFS.modifyEndpointRequest endpoint $
+			LFS.modifyEndpointRequest endpoint' $
 				applyBasicAuth' ba
-		addbasicauth Nothing endpoint = endpoint
+		addbasicauth Nothing endpoint' = endpoint'
 
 -- The endpoint is cached for later use.
 getLFSEndpoint :: LFS.TransferRequestOperation -> TVar LFSHandle -> Annex (Maybe LFS.Endpoint)

doc/bugs/git-lfs_special_insists_on_https.mdwn

@@ -66,3 +66,5 @@ Nil
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 Love git-annex. Long time supporter.
+
+> [[fixed|done]] --[[Joey]]

doc/bugs/git-lfs_special_insists_on_https/comment_1_5a80951f61874589a4df78f6d78fbfd1._comment

@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2025-02-18T16:23:23Z"
+ content="""
+LFS uses http basic auth, so using it over http probably allows
+any man in the middle to take over your storage.
+
+With that rationalle, <https://hackage.haskell.org/package/git-lfs>
+hardcodes a https url at LFS server discovery time. And I don't think it
+would be secure for it to do anything else by default; people do clone
+git over http and it would be a security hole if LFS then exposed their
+password.
+
+In your case, you're using a nonstandard http port, and it's continuing
+to use that same port for https. That seems unlikely to work in almost any
+situation. Perhaps a http url should only be upgraded to https when
+it's using a standard port. Or perhaps the nonstandard port should be
+replaced with the standard https port. I felt that the latter was less
+likely to result in security issues, and was more consistent, so I've gone
+with that approach. That change is in version 1.2.4 of 
+<https://hackage.haskell.org/package/git-lfs>.
+
+git-lfs has git configs `lfs.url` and `remote.<name>.lfsurl` 
+that allow the user to specify the API endpoint to use. The special
+remote's url= parameter is the git repository url, not the API endpoint.
+So I think that to handle your use case, it makes sense to add an optional
+apiurl= parameter to the special remote, which corresponds to those git
+configs.
+
+Unfortunately, adding apiurl= needed a new version 1.2.5 of
+<https://hackage.haskell.org/package/git-lfs>, so it will only
+be available in builds of git-annex that use that version of the library.
+Which will take a while to reach all builds.
+"""]]

doc/special_remotes/git-lfs.mdwn

@@ -9,7 +9,7 @@ These parameters can be passed to `git annex initremote` to configure
 the git-lfs special remote:
 
 * `url` - Required. The url to the git-lfs repository to use.
-  Can be either a ssh url (scp-style is also accepted) or a http url.
+  Can be either a ssh url (scp-style is also accepted) or a https url.
 
 * `encryption` - One of "none", "hybrid", "shared", or "pubkey".
   Required. See [[encryption]]. Also see the encryption notes below.
@@ -18,6 +18,10 @@ the git-lfs special remote:
   git-annex stores in the repository, as well as to encrypt the git
   repository itself when using gcrypt.
 
+* `apiurl` - Optional. The url to the LFS API endpoint. This can be a https
+  or a http url. When this is not specified, or is not set to an url, 
+  the API endpoint url is guessed based on the url parameter.
+
 ## efficiency note
 
 Since git-lfs uses SHA256 checksums, git-annex needs to keep track of the

stack.yaml

@@ -18,7 +18,7 @@ resolver: nightly-2025-01-20
 extra-deps:
 - filepath-bytestring-1.5.2.0.2
 - aws-0.24.4
-- git-lfs-1.2.3
+- git-lfs-1.2.5
 - feed-1.3.2.1
 allow-newer: true
 allow-newer-deps: