mirrors/git-annex
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

annex.addunlocked expressions

Browse source 
* annex.addunlocked can be set to an expression with the same format used by
  annex.largefiles, in case you want to default to unlocking some files but
  not others.
* annex.addunlocked can be configured by git-annex config.

Added a git-annex-matching-expression man page, broken out from
tips/largefiles.

A tricky consequence of this is that git-annex add --relaxed
honors annex.addunlocked, but an expression might want to know the size
or content of an url, which it's not going to download. I decided it was
better not to fail, and just dummy up some plausible data in that case.

Performance impact should be negligible. The global config is already
loaded for annex.largefiles. The expression only has to be parsed once,
and in the simple true/false case, it should not do any additional work
matching it.
This commit is contained in:
Joey Hess 2019-12-20 15:01:34 -04:00
parent f79bd52132
commit 37467a008f
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
25 changed files with 305 additions and 179 deletions
Download patch file Download diff file Expand all files Collapse all files

28
Command/ImportFeed.hs
View file

@ -38,6 +38,7 @@ import Annex.YoutubeDl
import Types.MetaData
import Logs.MetaData
import Annex.MetaData
import Annex.FileMatcher
import Command.AddUrl (addWorkTree)
cmd :: Command
@ -62,11 +63,12 @@ optParser desc = ImportFeedOptions
seek :: ImportFeedOptions -> CommandSeek
seek o = do
addunlockedmatcher <- addUnlockedMatcher
cache <- getCache (templateOption o)
forM_ (feedUrls o) (getFeed o cache)
forM_ (feedUrls o) (getFeed addunlockedmatcher o cache)
getFeed :: ImportFeedOptions -> Cache -> URLString -> CommandSeek
getFeed opts cache url = do
getFeed :: AddUnlockedMatcher -> ImportFeedOptions -> Cache -> URLString -> CommandSeek
getFeed addunlockedmatcher opts cache url = do
showStart' "importfeed" (Just url)
downloadFeed url >>= \case
Nothing -> showEndResult =<< feedProblem url
@ -77,7 +79,7 @@ getFeed opts cache url = do
[] -> debugfeedcontent feedcontent "bad feed content; no enclosures to download"
l -> do
showEndOk
ifM (and <$> mapM (performDownload opts cache) l)
ifM (and <$> mapM (performDownload addunlockedmatcher opts cache) l)
( clearFeedProblem url
, void $ feedProblem url
"problem downloading some item(s) from feed"
@ -153,8 +155,8 @@ downloadFeed url
, return Nothing
)
performDownload :: ImportFeedOptions -> Cache -> ToDownload -> Annex Bool
performDownload opts cache todownload = case location todownload of
performDownload :: AddUnlockedMatcher -> ImportFeedOptions -> Cache -> ToDownload -> Annex Bool
performDownload addunlockedmatcher opts cache todownload = case location todownload of
Enclosure url -> checkknown url $
rundownload url (takeWhile (/= '?') $ takeExtension url) $ \f -> do
r <- Remote.claimingUrl url
@ -171,7 +173,7 @@ performDownload opts cache todownload = case location todownload of
-- don't use youtube-dl
, rawOption = True
}
maybeToList <$> addUrlFile dlopts url urlinfo f
maybeToList <$> addUrlFile addunlockedmatcher dlopts url urlinfo f
else do
res <- tryNonAsync $ maybe
(error $ "unable to checkUrl of " ++ Remote.name r)
@ -181,10 +183,10 @@ performDownload opts cache todownload = case location todownload of
Left _ -> return []
Right (UrlContents sz _) ->
maybeToList <$>
downloadRemoteFile r (downloadOptions opts) url f sz
downloadRemoteFile addunlockedmatcher r (downloadOptions opts) url f sz
Right (UrlMulti l) -> do
kl <- forM l $ \(url', sz, subf) ->
downloadRemoteFile r (downloadOptions opts) url' (f </> fromSafeFilePath subf) sz
downloadRemoteFile addunlockedmatcher r (downloadOptions opts) url' (f </> fromSafeFilePath subf) sz
return $ if all isJust kl
then catMaybes kl
else []
@ -273,7 +275,7 @@ performDownload opts cache todownload = case location todownload of
[] -> ".m"
s -> s
ok <- rundownload linkurl ext $ \f -> do
addWorkTree webUUID mediaurl f mediakey (Just mediafile)
addWorkTree addunlockedmatcher webUUID mediaurl f mediakey (Just mediafile)
return [mediakey]
return (Just ok)
-- youtude-dl didn't support it, so
@ -285,16 +287,16 @@ performDownload opts cache todownload = case location todownload of
return Nothing
return (fromMaybe False r)
where
downloadlink = performDownload opts cache todownload
downloadlink = performDownload addunlockedmatcher opts cache todownload
{ location = Enclosure linkurl }
addmediafast linkurl mediaurl mediakey =
ifM (pure (not (rawOption (downloadOptions opts)))
<&&> youtubeDlSupported linkurl)
( rundownload linkurl ".m" $ \f -> do
addWorkTree webUUID mediaurl f mediakey Nothing
addWorkTree addunlockedmatcher webUUID mediaurl f mediakey Nothing
return [mediakey]
, performDownload opts cache todownload
, performDownload addunlockedmatcher opts cache todownload
{ location = Enclosure linkurl }
)