mirrors/git-annex
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

addurl, youtube-dl: When --check-raw prevents downloading an url, still continue with any downloads that come after it, rather than erroring out

Browse source 
Sponsored-By: Mark Reidenbach on Patreon
This commit is contained in:
Joey Hess 2021-11-28 19:40:06 -04:00
parent 9a1f14e6f0
commit 01a5ee6998
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
3 changed files with 26 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG
View file

@ -1,6 +1,9 @@
git-annex (8.20211124) UNRELEASED; urgency=medium
* Fix build with old versions of feed library.
* addurl, youtube-dl: When --check-raw prevents downloading an url,
still continue with any downloads that come after it, rather than
erroring out.
-- Joey Hess <id@joeyh.name> Tue, 23 Nov 2021 15:58:27 -0400

34
Command/AddUrl.hs
View file

@ -182,7 +182,7 @@ performRemote addunlockedmatcher r o uri file sz = ifAnnexed file adduri geturi
where
loguri = setDownloader uri OtherDownloader
adduri = addUrlChecked o loguri file (Remote.uuid r) checkexistssize
checkexistssize key = return $ case sz of
checkexistssize key = return $ Just $ case sz of
Nothing -> (True, True, loguri)
Just n -> (True, n == fromMaybe n (fromKey keySize key), loguri)
geturi = next $ isJust <$> downloadRemoteFile addunlockedmatcher r (downloadOptions o) uri file sz
@ -270,31 +270,31 @@ performWeb addunlockedmatcher o url file urlinfo = ifAnnexed file addurl geturl
geturl = next $ isJust <$> addUrlFile addunlockedmatcher (downloadOptions o) url urlinfo file
addurl = addUrlChecked o url file webUUID $ \k ->
ifM (pure (not (rawOption (downloadOptions o))) <&&> youtubeDlSupported url)
( return (True, True, setDownloader url YoutubeDownloader)
, checkRaw Nothing (downloadOptions o) $
return (Url.urlExists urlinfo, Url.urlSize urlinfo == fromKey keySize k, url)
( return (Just (True, True, setDownloader url YoutubeDownloader))
, checkRaw Nothing (downloadOptions o) Nothing $
return (Just (Url.urlExists urlinfo, Url.urlSize urlinfo == fromKey keySize k, url))
)
{- Check that the url exists, and has the same size as the key,
- and add it as an url to the key. -}
addUrlChecked :: AddUrlOptions -> URLString -> RawFilePath -> UUID -> (Key -> Annex (Bool, Bool, URLString)) -> Key -> CommandPerform
addUrlChecked :: AddUrlOptions -> URLString -> RawFilePath -> UUID -> (Key -> Annex (Maybe (Bool, Bool, URLString))) -> Key -> CommandPerform
addUrlChecked o url file u checkexistssize key =
ifM ((elem url <$> getUrls key) <&&> (elem u <$> loggedLocations key))
( do
showDestinationFile (fromRawFilePath file)
next $ return True
, do
(exists, samesize, url') <- checkexistssize key
if exists && (samesize || relaxedOption (downloadOptions o))
then do
, checkexistssize key >>= \case
Just (exists, samesize, url')
| exists && (samesize || relaxedOption (downloadOptions o)) -> do
setUrlPresent key url'
logChange key u InfoPresent
next $ return True
else do
| otherwise -> do
warning $ "while adding a new url to an already annexed file, " ++ if exists
then "url does not have expected file size (use --relaxed to bypass this check) " ++ url
else "failed to verify url exists: " ++ url
stop
Nothing -> stop
)
{- Downloads an url (except in fast or relaxed mode) and adds it to the
@ -333,7 +333,7 @@ downloadWeb addunlockedmatcher o url urlinfo file =
in ifAnnexed f
(alreadyannexed (fromRawFilePath f))
(dl f)
Left err -> checkRaw (Just err) o (normalfinish tmp)
Left err -> checkRaw (Just err) o Nothing (normalfinish tmp)
where
dl dest = withTmpWorkDir mediakey $ \workdir -> do
let cleanuptmp = pruneTmpWorkDirBefore tmp (liftIO . removeWhenExistsWith R.removeLink)
@ -347,7 +347,7 @@ downloadWeb addunlockedmatcher o url urlinfo file =
showDestinationFile (fromRawFilePath dest)
addWorkTree canadd addunlockedmatcher webUUID mediaurl dest mediakey (Just (toRawFilePath mediafile))
return $ Just mediakey
Right Nothing -> checkRaw Nothing o (normalfinish tmp)
Right Nothing -> checkRaw Nothing o Nothing (normalfinish tmp)
Left msg -> do
cleanuptmp
warning msg
@ -364,12 +364,14 @@ downloadWeb addunlockedmatcher o url urlinfo file =
warning $ dest ++ " already exists; not overwriting"
return Nothing
checkRaw :: (Maybe String) -> DownloadOptions -> Annex a -> Annex a
checkRaw failreason o a
| noRawOption o = giveup $ "Unable to use youtube-dl or a special remote and --no-raw was specified" ++
checkRaw :: (Maybe String) -> DownloadOptions -> a -> Annex a -> Annex a
checkRaw failreason o f a
| noRawOption o = do
warning $ "Unable to use youtube-dl or a special remote and --no-raw was specified" ++
case failreason of
Just msg -> ": " ++ msg
Nothing -> ""
return f
| otherwise = a
{- The destination file is not known at start time unless the user provided
@ -491,7 +493,7 @@ nodownloadWeb addunlockedmatcher o url urlinfo file
then nomedia
else youtubeDlFileName url >>= \case
Right mediafile -> usemedia (toRawFilePath mediafile)
Left err -> checkRaw (Just err) o nomedia
Left err -> checkRaw (Just err) o Nothing nomedia
| otherwise = do
warning $ "unable to access url: " ++ url
return Nothing

4
Command/ImportFeed.hs
View file

@ -202,7 +202,7 @@ performDownload' started addunlockedmatcher opts cache todownload = case locatio
let f' = fromRawFilePath f
r <- Remote.claimingUrl url
if Remote.uuid r == webUUID || rawOption (downloadOptions opts)
then checkRaw Nothing (downloadOptions opts) $ do
then checkRaw (Just url) (downloadOptions opts) Nothing $ do
let dlopts = (downloadOptions opts)
-- force using the filename
-- chosen here
@ -341,7 +341,7 @@ performDownload' started addunlockedmatcher opts cache todownload = case locatio
, downloadlink False
)
where
downloadlink started' = checkRaw Nothing (downloadOptions opts) $
downloadlink started' = checkRaw (Just linkurl) (downloadOptions opts) False $
performDownload' started' addunlockedmatcher opts cache todownload
{ location = Enclosure linkurl }