annex.addunlocked support for tree imports

Honor annex.addunlocked configuration when importing a tree from a special
remote.

Note, in a --no-content import, the object file will not be populated
(usually) and so expressions that match on mime type will not match. Tested
this and it works ok, the file just ends up locked. Updated docs for the
mime expressions to mention that they can't match when the file is present

Note that in Command.Sync.pullThirdPartyPopulated, recordImportTree is
called without a AddUnlockedMatcher. Since the tree generated here is not
exposed to the user and does not contain usual filenames, there is no need
of the overhead of checking it.

Annex/Import.hs

@@ -107,9 +107,10 @@ buildImportCommit
 	:: Remote
 	-> ImportTreeConfig
 	-> ImportCommitConfig
+	-> AddUnlockedMatcher
 	-> Imported
 	-> Annex (Maybe Ref)
-buildImportCommit remote importtreeconfig importcommitconfig imported =
+buildImportCommit remote importtreeconfig importcommitconfig addunlockedmatcher imported =
 	case importCommitTracking importcommitconfig of
 		Nothing -> go Nothing
 		Just trackingcommit -> inRepo (Git.Ref.tree trackingcommit) >>= \case
@@ -117,7 +118,7 @@ buildImportCommit remote importtreeconfig importcommitconfig imported =
 			Just _ -> go (Just trackingcommit)
   where
 	go trackingcommit = do
-		(importedtree, updatestate) <- recordImportTree remote importtreeconfig imported
+		(importedtree, updatestate) <- recordImportTree remote importtreeconfig (Just addunlockedmatcher) imported
 		buildImportCommit' remote importcommitconfig trackingcommit importedtree >>= \case
 			Just finalcommit -> do
 				updatestate
@@ -132,10 +133,11 @@ buildImportCommit remote importtreeconfig importcommitconfig imported =
 recordImportTree
 	:: Remote
 	-> ImportTreeConfig
+	-> Maybe AddUnlockedMatcher
 	-> Imported
 	-> Annex (History Sha, Annex ())
-recordImportTree remote importtreeconfig imported = do
+recordImportTree remote importtreeconfig addunlockedmatcher imported = do
-	importedtree@(History finaltree _) <- buildImportTrees basetree subdir imported
+	importedtree@(History finaltree _) <- buildImportTrees basetree subdir addunlockedmatcher imported
 	return (importedtree, updatestate finaltree)
   where
 	basetree = case importtreeconfig of
@@ -177,7 +179,7 @@ recordImportTree remote importtreeconfig imported = do
 			}
 		return oldexport
 
-	-- downloadImport takes care of updating the location log
+	-- importKeys takes care of updating the location log
 	-- for the local repo when keys are downloaded, and also updates
 	-- the location log for the remote for keys that are present in it.
 	-- That leaves updating the location log for the remote for keys
@@ -283,11 +285,12 @@ buildImportCommit' remote importcommitconfig mtrackingcommit imported@(History t
 buildImportTrees
 	:: Ref
 	-> Maybe TopFilePath
+	-> Maybe AddUnlockedMatcher
 	-> Imported
 	-> Annex (History Sha)
-buildImportTrees basetree msubdir (ImportedFull imported) = 
+buildImportTrees basetree msubdir addunlockedmatcher (ImportedFull imported) = 
-	buildImportTreesGeneric convertImportTree basetree msubdir imported
+	buildImportTreesGeneric (convertImportTree addunlockedmatcher) basetree msubdir imported
-buildImportTrees basetree msubdir (ImportedDiff (LastImportedTree oldtree) imported) = do
+buildImportTrees basetree msubdir addunlockedmatcher (ImportedDiff (LastImportedTree oldtree) imported) = do
 	importtree <- if null (importableContents imported)
 		then pure oldtree
 		else applydiff
@@ -312,7 +315,7 @@ buildImportTrees basetree msubdir (ImportedDiff (LastImportedTree oldtree) impor
 			oldtree
 	
 	mktreeitem (loc, DiffChanged v) = 
-		Just <$> mkImportTreeItem msubdir loc v
+		Just <$> mkImportTreeItem addunlockedmatcher msubdir loc v
 	mktreeitem (_, DiffRemoved) = 
 		pure Nothing
 
@@ -320,17 +323,26 @@ buildImportTrees basetree msubdir (ImportedDiff (LastImportedTree oldtree) impor
 		
 	isremoved (_, v) = v == DiffRemoved
 
-convertImportTree :: Maybe TopFilePath -> [(ImportLocation, Either Sha Key)] -> Annex Tree
+convertImportTree :: Maybe AddUnlockedMatcher -> Maybe TopFilePath -> [(ImportLocation, Either Sha Key)] -> Annex Tree
-convertImportTree msubdir ls = 
+convertImportTree maddunlockedmatcher msubdir ls = 
-	treeItemsToTree <$> mapM (uncurry $ mkImportTreeItem msubdir) ls
+	treeItemsToTree <$> mapM (uncurry $ mkImportTreeItem maddunlockedmatcher msubdir) ls
 
-mkImportTreeItem :: Maybe TopFilePath -> ImportLocation -> Either Sha Key -> Annex TreeItem
+mkImportTreeItem :: Maybe AddUnlockedMatcher -> Maybe TopFilePath -> ImportLocation -> Either Sha Key -> Annex TreeItem
-mkImportTreeItem msubdir loc v = case v of
+mkImportTreeItem maddunlockedmatcher msubdir loc v = case v of
-	Right k -> do
+	Right k -> case maddunlockedmatcher of
-		relf <- fromRepo $ fromTopFilePath topf
+		Nothing -> mklink k
-		symlink <- calcRepo $ gitAnnexLink relf k
+		Just addunlockedmatcher -> do
-		linksha <- hashSymlink symlink
+			objfile <- calcRepo (gitAnnexLocation k)
-		return $ TreeItem treepath (fromTreeItemType TreeSymlink) linksha
+			let mi = MatchingFile FileInfo
+				{ contentFile = objfile
+				, matchFile = getTopFilePath topf
+				, matchKey = Just k
+				}
+			ifM (checkAddUnlockedMatcher NoLiveUpdate addunlockedmatcher mi)
+				( mkpointer k
+				, mklink k
+				)
+				
 	Left sha -> 
 		return $ TreeItem treepath (fromTreeItemType TreeFile) sha
   where
@@ -338,6 +350,13 @@ mkImportTreeItem msubdir loc v = case v of
 	treepath = asTopFilePath lf
 	topf = asTopFilePath $
 		maybe lf (\sd -> getTopFilePath sd P.</> lf) msubdir
+	mklink k = do
+		relf <- fromRepo $ fromTopFilePath topf
+		symlink <- calcRepo $ gitAnnexLink relf k
+		linksha <- hashSymlink symlink
+		return $ TreeItem treepath (fromTreeItemType TreeSymlink) linksha
+	mkpointer k = TreeItem treepath (fromTreeItemType TreeFile)
+		<$> hashPointerFile k
 
 {- Builds a history of git trees using ContentIdentifiers.
  -
@@ -604,8 +623,8 @@ getLastImportedTree remote = do
  - generates Keys without downloading.
  -
  - Generates either a Key or a git Sha, depending on annex.largefiles.
- - But when importcontent is False, it cannot match on annex.largefiles
+ - But when importcontent is False, it cannot generate a git Sha, 
- - (or generate a git Sha), so always generates Keys.
+ - so always generates Keys.
  -
  - Supports concurrency when enabled.
  -

CHANGELOG

@@ -11,6 +11,8 @@ git-annex (10.20241203) UNRELEASED; urgency=medium
     default unset behavior.
   * sync: Avoid misleading warning about future preferred content
     transition when preferred content is set to "".
+  * Honor annex.addunlocked configuration when importing a tree from a
+    special remote.
 
  -- Joey Hess <id@joeyh.name>  Mon, 02 Dec 2024 13:41:31 -0400
 

Command/Import.hs

@@ -147,8 +147,10 @@ seek o@(RemoteImportOptions {}) = startConcurrency commandStages $ do
 		(pure Nothing)
 		(Just <$$> inRepo . toTopFilePath . toRawFilePath)
 		(importToSubDir o)
+	addunlockedmatcher <- addUnlockedMatcher
 	seekRemote r (importToBranch o) subdir (importContent o) 
 		(checkGitIgnoreOption o)
+		addunlockedmatcher
 		(messageOption o)
 
 startLocal :: ImportOptions -> AddUnlockedMatcher -> GetFileMatcher -> DuplicateMode -> (RawFilePath, RawFilePath) -> CommandStart
@@ -322,8 +324,8 @@ verifyExisting key destfile (yes, no) = do
 	verifyEnoughCopiesToDrop [] key Nothing Nothing needcopies mincopies [] preverified tocheck
 		(const yes) no
 
-seekRemote :: Remote -> Branch -> Maybe TopFilePath -> Bool -> CheckGitIgnore -> [String] -> CommandSeek
+seekRemote :: Remote -> Branch -> Maybe TopFilePath -> Bool -> CheckGitIgnore -> AddUnlockedMatcher -> [String] -> CommandSeek
-seekRemote remote branch msubdir importcontent ci importmessages = do
+seekRemote remote branch msubdir importcontent ci addunlockedmatcher importmessages = do
 	importtreeconfig <- case msubdir of
 		Nothing -> return ImportTree
 		Just subdir ->
@@ -337,7 +339,7 @@ seekRemote remote branch msubdir importcontent ci importmessages = do
 	trackingcommit <- fromtrackingbranch Git.Ref.sha
 	cmode <- annexCommitMode <$> Annex.getGitConfig
 	let importcommitconfig = ImportCommitConfig trackingcommit cmode importmessages'
-	let commitimport = commitRemote remote branch tb trackingcommit importtreeconfig importcommitconfig
+	let commitimport = commitRemote remote branch tb trackingcommit importtreeconfig importcommitconfig addunlockedmatcher
 
 	importabletvar <- liftIO $ newTVarIO Nothing
 	void $ includeCommandAction (listContents remote importtreeconfig ci importabletvar)
@@ -383,10 +385,10 @@ listContents' remote importtreeconfig ci a =
 			, err
 			]
 
-commitRemote :: Remote -> Branch -> RemoteTrackingBranch -> Maybe Sha -> ImportTreeConfig -> ImportCommitConfig -> Imported -> CommandStart
+commitRemote :: Remote -> Branch -> RemoteTrackingBranch -> Maybe Sha -> ImportTreeConfig -> ImportCommitConfig -> AddUnlockedMatcher -> Imported -> CommandStart
-commitRemote remote branch tb trackingcommit importtreeconfig importcommitconfig imported =
+commitRemote remote branch tb trackingcommit importtreeconfig importcommitconfig addunlockedmatcher imported =
 	starting "update" ai si $ do
-		importcommit <- buildImportCommit remote importtreeconfig importcommitconfig imported
+		importcommit <- buildImportCommit remote importtreeconfig importcommitconfig addunlockedmatcher imported
 		next $ updateremotetrackingbranch importcommit
   where
 	ai = ActionItemOther (Just $ UnquotedString $ fromRef $ fromRemoteTrackingBranch tb)

Command/Sync.hs

@@ -53,6 +53,7 @@ import Annex.Path
 import Annex.Wanted
 import Annex.Content
 import Annex.WorkTree
+import Annex.FileMatcher
 import Command.Get (getKey')
 import qualified Command.Move
 import qualified Command.Export
@@ -77,7 +78,6 @@ import Annex.CurrentBranch
 import Annex.Import
 import Annex.CheckIgnore
 import Annex.PidLock
-import Types.FileMatcher
 import Types.GitConfig
 import Types.Availability
 import qualified Database.Export as Export
@@ -580,7 +580,8 @@ importRemote importcontent o remote currbranch
 			let (branch, subdir) = splitRemoteAnnexTrackingBranchSubdir b
 			if canImportKeys remote importcontent
 				then do
-					Command.Import.seekRemote remote branch subdir importcontent (CheckGitIgnore True) []
+					addunlockedmatcher <- addUnlockedMatcher
+					Command.Import.seekRemote remote branch subdir importcontent (CheckGitIgnore True) addunlockedmatcher []
 					-- Importing generates a branch
 					-- that is not initially connected
 					-- to the current branch, so allow
@@ -607,7 +608,7 @@ pullThirdPartyPopulated o remote
   where
 	go (Just importable) = importChanges remote ImportTree False True importable >>= \case
 		ImportFinished imported -> do
-			(_t, updatestate) <- recordImportTree remote ImportTree imported
+			(_t, updatestate) <- recordImportTree remote ImportTree Nothing imported
 			next $ do
 				updatestate
 				return True

doc/bugs/addunlocked_true_is_not_in_effect_for_import.mdwn

@@ -54,3 +54,5 @@ This was to consider using `import` for a folder with DANDI stats. For now I wil
 
 [[!meta author=yoh]]
 [[!tag projects/dandi]]
+
+> [[fixed|done]] --[[Joey]]

doc/bugs/addunlocked_true_is_not_in_effect_for_import/comment_1_d8dc7bb0e6d3ae4e77a2d17ef672b3ab._comment

@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2024-12-18T19:24:38Z"
+ content="""
+Turns out that while `git-annex import` from a directory does support
+addunlocked, this was forgotten about when implementing the newer special
+remote tree import.
+
+I agree that this should be supported.
+"""]]

doc/bugs/addunlocked_true_is_not_in_effect_for_import/comment_2_40371f91f89cc035c8edc1b719210a5c._comment

@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2024-12-19T15:31:59Z"
+ content="""
+Note that for --no-content imports, it will not be possible for mimetype=
+and mimeencoding= expressions to match.
+
+So if addunlocked is set to such an expression, it will not match and will
+add the file locked. Does not seem like a blocker.
+"""]]

doc/bugs/addunlocked_true_is_not_in_effect_for_import/comment_3_4640377996674e1330d542354b36ef3c._comment

@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2024-12-19T15:34:48Z"
+ content="""
+Implemented this.
+"""]]

doc/git-annex-matching-expression.mdwn

@@ -48,6 +48,10 @@ The following terms can be used:
 
   The MIME types are the same that are displayed by running `file --mime-type`
 
+  This only matches when the content of the file is present in the local
+  repository. Usually that is the case, but eg, when importing from a
+  special remote with --no-content, the content is usually not present.
+
   This is only available to use when git-annex was built with the
   MagicMime build flag.
 
@@ -60,6 +64,10 @@ The following terms can be used:
 
   The MIME encodings are the same that are displayed by running `file --mime-encoding`
 
+  This only matches when the content of the file is present in the local
+  repository. Usually that is the case, but eg, when importing from a
+  special remote with --no-content, the content is usually not present.
+
   This is only available to use when git-annex was built with the
   MagicMime build flag.