0f7143d226
Not yet implemented is recording hashes on download from web and verifying hashes. addurl --verifiable option added with -V short option because I expect a lot of people will want to use this. It seems likely that --verifiable will become the default eventually, and possibly rather soon. While old git-annex versions don't support VURL, that doesn't prevent using them with keys that use VURL. Of course, they won't verify the content on transfer, and fsck will warn that it doesn't know about VURL. So there's not much problem with starting to use VURL even when interoperating with old versions. Sponsored-by: Joshua Antonishen on Patreon
655 lines
21 KiB
Haskell
655 lines
21 KiB
Haskell
{- git-annex command
|
|
-
|
|
- Copyright 2013-2024 Joey Hess <id@joeyh.name>
|
|
-
|
|
- Licensed under the GNU AGPL version 3 or higher.
|
|
-}
|
|
|
|
{-# LANGUAGE OverloadedStrings #-}
|
|
{-# LANGUAGE BangPatterns #-}
|
|
{-# LANGUAGE CPP #-}
|
|
|
|
module Command.ImportFeed where
|
|
|
|
import Text.Feed.Import
|
|
import Text.Feed.Query
|
|
import Text.Feed.Types
|
|
import qualified Data.Set as S
|
|
import qualified Data.Map as M
|
|
import Data.Time.Clock
|
|
import Data.Time.Clock.POSIX
|
|
import Data.Time.Format
|
|
import Data.Time.Calendar
|
|
import Data.Time.LocalTime
|
|
import Control.Concurrent.STM
|
|
import qualified Data.Text as T
|
|
import qualified Data.Text.Encoding as TE
|
|
import qualified System.FilePath.ByteString as P
|
|
import qualified Data.ByteString as B
|
|
|
|
import Command
|
|
import qualified Annex
|
|
import qualified Annex.Url as Url
|
|
import qualified Remote
|
|
import qualified Types.Remote as Remote
|
|
import Types.UrlContents
|
|
import Logs.Web
|
|
import Logs.File
|
|
import qualified Utility.Format
|
|
import Utility.Tmp
|
|
import Utility.Metered
|
|
import Command.AddUrl (addUrlFile, downloadRemoteFile, parseDownloadOptions, DownloadOptions(..), checkClaimingUrl, checkCanAdd, addWorkTree, checkRaw, useYoutubeDl)
|
|
import Annex.UUID
|
|
import Backend.URL (fromUrl)
|
|
import Annex.Content
|
|
import Annex.WorkTree
|
|
import Annex.YoutubeDl
|
|
import Types.MetaData
|
|
import Logs.MetaData
|
|
import Annex.MetaData
|
|
import Annex.FileMatcher
|
|
import Annex.UntrustedFilePath
|
|
import qualified Utility.RawFilePath as R
|
|
import qualified Database.ImportFeed as Db
|
|
|
|
cmd :: Command
|
|
cmd = notBareRepo $ withAnnexOptions os $
|
|
command "importfeed" SectionCommon "import files from podcast feeds"
|
|
(paramRepeating paramUrl) (seek <$$> optParser)
|
|
where
|
|
os = [jobsOption, jsonOptions, jsonProgressOption, backendOption]
|
|
|
|
data ImportFeedOptions = ImportFeedOptions
|
|
{ feedUrls :: CmdParams
|
|
, templateOption :: Maybe String
|
|
, scrapeOption :: Bool
|
|
, downloadOptions :: DownloadOptions
|
|
}
|
|
|
|
optParser :: CmdParamsDesc -> Parser ImportFeedOptions
|
|
optParser desc = ImportFeedOptions
|
|
<$> cmdParams desc
|
|
<*> optional (strOption
|
|
( long "template" <> metavar paramFormat
|
|
<> help "template for filenames"
|
|
))
|
|
<*> switch
|
|
( long "scrape"
|
|
<> help "scrape website for content to import"
|
|
)
|
|
<*> parseDownloadOptions False
|
|
|
|
seek :: ImportFeedOptions -> CommandSeek
|
|
seek o = startConcurrency commandStages $ do
|
|
addunlockedmatcher <- addUnlockedMatcher
|
|
cache <- getCache (templateOption o)
|
|
dlst <- liftIO $ newTMVarIO M.empty
|
|
checkst <- liftIO $ newTVarIO M.empty
|
|
|
|
forM_ (feedUrls o) $ \url -> do
|
|
liftIO $ atomically $ do
|
|
m <- takeTMVar dlst
|
|
putTMVar dlst (M.insert url Nothing m)
|
|
commandAction $ getFeed o url dlst
|
|
startpendingdownloads addunlockedmatcher cache dlst checkst False
|
|
|
|
startpendingdownloads addunlockedmatcher cache dlst checkst True
|
|
|
|
clearfeedproblems checkst
|
|
where
|
|
getpendingdownloads dlst blocking
|
|
| blocking = do
|
|
m <- takeTMVar dlst
|
|
if M.null m
|
|
then do
|
|
putTMVar dlst m
|
|
return m
|
|
else
|
|
let (pending, rest) = M.partition ispending m
|
|
in if M.null pending || not (M.null rest)
|
|
then retry
|
|
else do
|
|
putTMVar dlst rest
|
|
return pending
|
|
| otherwise = do
|
|
m <- takeTMVar dlst
|
|
let (pending, rest) = M.partition ispending m
|
|
putTMVar dlst rest
|
|
return pending
|
|
where
|
|
ispending Nothing = False
|
|
ispending (Just _) = True
|
|
|
|
startpendingdownloads addunlockedmatcher cache dlst checkst blocking = do
|
|
m <- liftIO $ atomically $ getpendingdownloads dlst blocking
|
|
|
|
forM_ (M.toList m) $ \(url, v) -> case v of
|
|
Nothing -> noop
|
|
Just Nothing -> noop
|
|
Just (Just is) ->
|
|
forM_ is $ \i -> do
|
|
cv <- liftIO newEmptyTMVarIO
|
|
liftIO $ atomically $ modifyTVar checkst $
|
|
M.insertWith (++) url [cv]
|
|
commandAction $
|
|
startDownload addunlockedmatcher o cache cv i
|
|
|
|
clearfeedproblems checkst = do
|
|
m <- liftIO $ atomically $ readTVar checkst
|
|
forM_ (M.toList m) $ \(url, cvl) ->
|
|
whenM (and <$> mapM (liftIO . atomically . takeTMVar) cvl) $
|
|
clearFeedProblem url
|
|
|
|
getFeed
|
|
:: ImportFeedOptions
|
|
-> URLString
|
|
-> TMVar (M.Map URLString (Maybe (Maybe [ToDownload])))
|
|
-> CommandStart
|
|
getFeed o url st =
|
|
starting "importfeed" (ActionItemOther (Just (UnquotedString url))) (SeekInput [url]) $
|
|
go `onException` recordfail
|
|
where
|
|
record v = liftIO $ atomically $ do
|
|
m <- takeTMVar st
|
|
putTMVar st (M.insert url v m)
|
|
recordfail = record (Just Nothing)
|
|
|
|
go
|
|
| scrapeOption o = scrape
|
|
| otherwise = get
|
|
|
|
get = withTmpFile "feed" $ \tmpf h -> do
|
|
liftIO $ hClose h
|
|
ifM (downloadFeed url tmpf)
|
|
( parse tmpf
|
|
, do
|
|
recordfail
|
|
next $ feedProblem url
|
|
"downloading the feed failed"
|
|
)
|
|
|
|
-- Use parseFeedFromFile rather than reading the file
|
|
-- ourselves because it goes out of its way to handle encodings.
|
|
parse tmpf = liftIO (parseFeedFromFile' tmpf) >>= \case
|
|
Nothing -> debugfeedcontent tmpf "parsing the feed failed"
|
|
Just f -> do
|
|
case decodeBS $ fromFeedText $ getFeedTitle f of
|
|
"" -> noop
|
|
t -> showNote (UnquotedString ('"' : t ++ "\""))
|
|
case findDownloads url f of
|
|
[] -> debugfeedcontent tmpf "bad feed content; no enclosures to download"
|
|
l -> do
|
|
record (Just (Just l))
|
|
next $ return True
|
|
|
|
debugfeedcontent tmpf msg = do
|
|
feedcontent <- liftIO $ readFile tmpf
|
|
fastDebug "Command.ImportFeed" $ unlines
|
|
[ "start of feed content"
|
|
, feedcontent
|
|
, "end of feed content"
|
|
]
|
|
recordfail
|
|
next $ feedProblem url
|
|
(msg ++ " (use --debug --debugfilter=ImportFeed to see the feed content that was downloaded)")
|
|
|
|
scrape = youtubePlaylist url >>= \case
|
|
Left err -> do
|
|
recordfail
|
|
next $ feedProblem url err
|
|
Right playlist -> do
|
|
record (Just (Just (playlistDownloads url playlist)))
|
|
next $ return True
|
|
|
|
parseFeedFromFile' :: FilePath -> IO (Maybe Feed)
|
|
#if MIN_VERSION_feed(1,1,0)
|
|
parseFeedFromFile' = parseFeedFromFile
|
|
#else
|
|
parseFeedFromFile' f = catchMaybeIO (parseFeedFromFile f)
|
|
#endif
|
|
|
|
data ToDownload = ToDownload
|
|
{ feedurl :: URLString
|
|
, location :: DownloadLocation
|
|
, itemid :: Maybe B.ByteString
|
|
-- Either the parsed or unparsed date.
|
|
, itempubdate :: Maybe (Either String UTCTime)
|
|
-- Fields that are used as metadata and to generate the filename.
|
|
, itemfields :: [(String, String)]
|
|
-- True when youtube-dl found this by scraping, so certainly
|
|
-- supports downloading it.
|
|
, youtubedlscraped :: Bool
|
|
}
|
|
|
|
data DownloadLocation = Enclosure URLString | MediaLink URLString
|
|
|
|
type ItemId = String
|
|
|
|
data Cache = Cache
|
|
{ dbhandle :: Maybe Db.ImportFeedDbHandle
|
|
, template :: Utility.Format.Format
|
|
}
|
|
|
|
getCache :: Maybe String -> Annex Cache
|
|
getCache opttemplate = ifM (Annex.getRead Annex.force)
|
|
( ret Nothing
|
|
, do
|
|
j <- jsonOutputEnabled
|
|
unless j $
|
|
showStartMessage (StartMessage "importfeed" (ActionItemOther (Just "gathering known urls")) (SeekInput []))
|
|
h <- Db.openDb
|
|
unless j
|
|
showEndOk
|
|
ret (Just h)
|
|
)
|
|
where
|
|
tmpl = Utility.Format.gen $ fromMaybe defaultTemplate opttemplate
|
|
ret h = return $ Cache h tmpl
|
|
|
|
findDownloads :: URLString -> Feed -> [ToDownload]
|
|
findDownloads u f = catMaybes $ map mk (feedItems f)
|
|
where
|
|
mk i = case getItemEnclosure i of
|
|
Just (enclosureurl, _, _) ->
|
|
Just $ mk' i
|
|
(Enclosure $ decodeBS $ fromFeedText enclosureurl)
|
|
Nothing -> case getItemLink i of
|
|
Just l -> Just $ mk' i
|
|
(MediaLink $ decodeBS $ fromFeedText l)
|
|
Nothing -> Nothing
|
|
mk' i l = ToDownload
|
|
{ feedurl = u
|
|
, location = l
|
|
, itemid = case getItemId i of
|
|
Just (_, iid) -> Just (fromFeedText iid)
|
|
_ -> Nothing
|
|
, itempubdate = case getItemPublishDate i :: Maybe (Maybe UTCTime) of
|
|
Just (Just d) -> Just (Right d)
|
|
_ -> Left . decodeBS . fromFeedText
|
|
<$> getItemPublishDateString i
|
|
, itemfields = extractFeedItemFields f i u
|
|
, youtubedlscraped = False
|
|
}
|
|
|
|
{- Feeds change, so a feed download cannot be resumed. -}
|
|
downloadFeed :: URLString -> FilePath -> Annex Bool
|
|
downloadFeed url f
|
|
| Url.parseURIRelaxed url == Nothing = giveup "invalid feed url"
|
|
| otherwise = Url.withUrlOptions $
|
|
Url.download nullMeterUpdate Nothing url f
|
|
|
|
startDownload :: AddUnlockedMatcher -> ImportFeedOptions -> Cache -> TMVar Bool -> ToDownload -> CommandStart
|
|
startDownload addunlockedmatcher opts cache cv todownload = case location todownload of
|
|
Enclosure url -> startdownloadenclosure url
|
|
MediaLink linkurl -> do
|
|
let mediaurl = setDownloader linkurl YoutubeDownloader
|
|
let mediakey = Backend.URL.fromUrl mediaurl Nothing (verifiableOption (downloadOptions opts))
|
|
-- Old versions of git-annex that used quvi might have
|
|
-- used the quviurl for this, so check if it's known
|
|
-- to avoid adding it a second time.
|
|
let quviurl = setDownloader linkurl QuviDownloader
|
|
checkknown mediaurl $ checkknown quviurl $
|
|
ifM (Annex.getRead Annex.fast <||> pure (relaxedOption (downloadOptions opts)))
|
|
( addmediafast linkurl mediaurl mediakey
|
|
, downloadmedia linkurl mediaurl mediakey
|
|
)
|
|
where
|
|
{- Avoids downloading any items that are already known to be
|
|
- associated with a file in the annex. -}
|
|
checkknown url a = case dbhandle cache of
|
|
Just db -> ifM (liftIO $ Db.isKnownUrl db url)
|
|
( nothingtodo
|
|
, case itemid todownload of
|
|
Just iid ->
|
|
ifM (liftIO $ Db.isKnownItemId db iid)
|
|
( nothingtodo
|
|
, a
|
|
)
|
|
Nothing -> a
|
|
)
|
|
Nothing -> a
|
|
|
|
nothingtodo = recordsuccess >> stop
|
|
|
|
recordsuccess = liftIO $ atomically $ putTMVar cv True
|
|
|
|
startdownloadenclosure :: URLString -> CommandStart
|
|
startdownloadenclosure url = checkknown url $ startUrlDownload cv todownload url $
|
|
downloadEnclosure addunlockedmatcher opts cache cv todownload url
|
|
|
|
downloadmedia linkurl mediaurl mediakey =
|
|
ifM (useYoutubeDl (downloadOptions opts) <&&> youtubeDlSupported linkurl)
|
|
( startUrlDownload cv todownload linkurl $
|
|
withTmpWorkDir mediakey $ \workdir -> do
|
|
dl <- youtubeDl linkurl (fromRawFilePath workdir) nullMeterUpdate
|
|
case dl of
|
|
Right (Just mediafile) -> do
|
|
let ext = case takeExtension mediafile of
|
|
[] -> ".m"
|
|
s -> s
|
|
runDownload todownload linkurl ext cache cv $ \f ->
|
|
checkCanAdd (downloadOptions opts) f $ \canadd -> do
|
|
addWorkTree canadd addunlockedmatcher webUUID mediaurl f mediakey (Just (toRawFilePath mediafile))
|
|
return (Just [mediakey])
|
|
-- youtube-dl didn't support it, so
|
|
-- download it as if the link were
|
|
-- an enclosure.
|
|
Right Nothing -> contdownloadlink
|
|
Left msg -> do
|
|
warning $ UnquotedString $ linkurl ++ ": " ++ msg
|
|
liftIO $ atomically $ putTMVar cv False
|
|
next $ return False
|
|
, startdownloadlink
|
|
)
|
|
where
|
|
startdownloadlink = checkRaw (Just linkurl) (downloadOptions opts) nothingtodo $
|
|
startdownloadenclosure linkurl
|
|
contdownloadlink = downloadEnclosure addunlockedmatcher opts cache cv todownload linkurl
|
|
|
|
addmediafast linkurl mediaurl mediakey =
|
|
ifM (useYoutubeDl (downloadOptions opts)
|
|
<&&> (pure (youtubedlscraped todownload) <||> youtubeDlSupported linkurl))
|
|
( startUrlDownload cv todownload linkurl $ do
|
|
runDownload todownload linkurl ".m" cache cv $ \f ->
|
|
checkCanAdd (downloadOptions opts) f $ \canadd -> do
|
|
addWorkTree canadd addunlockedmatcher webUUID mediaurl f mediakey Nothing
|
|
return (Just [mediakey])
|
|
, startdownloadenclosure linkurl
|
|
)
|
|
|
|
downloadEnclosure :: AddUnlockedMatcher -> ImportFeedOptions -> Cache -> TMVar Bool -> ToDownload -> URLString -> CommandPerform
|
|
downloadEnclosure addunlockedmatcher opts cache cv todownload url =
|
|
runDownload todownload url (takeWhile (/= '?') $ takeExtension url) cache cv $ \f -> do
|
|
let f' = fromRawFilePath f
|
|
r <- checkClaimingUrl (downloadOptions opts) url
|
|
if Remote.uuid r == webUUID || rawOption (downloadOptions opts)
|
|
then checkRaw (Just url) (downloadOptions opts) (pure Nothing) $ do
|
|
let dlopts = (downloadOptions opts)
|
|
-- force using the filename
|
|
-- chosen here
|
|
{ fileOption = Just f'
|
|
-- don't use youtube-dl
|
|
, rawOption = True
|
|
}
|
|
let go urlinfo = Just . maybeToList <$> addUrlFile addunlockedmatcher dlopts url urlinfo f
|
|
if relaxedOption (downloadOptions opts)
|
|
then go Url.assumeUrlExists
|
|
else Url.withUrlOptions (Url.getUrlInfo url) >>= \case
|
|
Right urlinfo -> go urlinfo
|
|
Left err -> do
|
|
warning (UnquotedString err)
|
|
return (Just [])
|
|
else do
|
|
res <- tryNonAsync $ maybe
|
|
(giveup $ "unable to checkUrl of " ++ Remote.name r)
|
|
(flip id url)
|
|
(Remote.checkUrl r)
|
|
case res of
|
|
Left _ -> return (Just [])
|
|
Right (UrlContents sz _) ->
|
|
Just . maybeToList <$>
|
|
downloadRemoteFile addunlockedmatcher r (downloadOptions opts) url f sz
|
|
Right (UrlMulti l) -> do
|
|
kl <- forM l $ \(url', sz, subf) ->
|
|
let dest = f P.</> toRawFilePath (sanitizeFilePath subf)
|
|
in downloadRemoteFile addunlockedmatcher r (downloadOptions opts) url' dest sz
|
|
return $ Just $ if all isJust kl
|
|
then catMaybes kl
|
|
else []
|
|
|
|
runDownload
|
|
:: ToDownload
|
|
-> URLString
|
|
-> String
|
|
-> Cache
|
|
-> TMVar Bool
|
|
-> (RawFilePath -> Annex (Maybe [Key]))
|
|
-> CommandPerform
|
|
runDownload todownload url extension cache cv getter = do
|
|
dest <- makeunique (1 :: Integer) $
|
|
feedFile (template cache) todownload extension
|
|
case dest of
|
|
Nothing -> do
|
|
recordsuccess
|
|
next $ return True
|
|
Just f -> getter (toRawFilePath f) >>= \case
|
|
Just ks
|
|
-- Download problem.
|
|
| null ks -> do
|
|
broken <- checkFeedBroken (feedurl todownload)
|
|
when broken $
|
|
void $ feedProblem url "download failed"
|
|
liftIO $ atomically $ putTMVar cv broken
|
|
next $ return False
|
|
| otherwise -> do
|
|
forM_ ks $ \key ->
|
|
ifM (annexGenMetaData <$> Annex.getGitConfig)
|
|
( addMetaData key $ extractMetaData todownload
|
|
, addMetaData key $ minimalMetaData todownload
|
|
)
|
|
recordsuccess
|
|
next $ return True
|
|
-- Was not able to add anything, but not
|
|
-- because of a download problem.
|
|
Nothing -> do
|
|
recordsuccess
|
|
next $ return False
|
|
where
|
|
recordsuccess = liftIO $ atomically $ putTMVar cv True
|
|
|
|
forced = Annex.getRead Annex.force
|
|
|
|
{- Find a unique filename to save the url to.
|
|
- If the file exists, prefixes it with a number.
|
|
- When forced, the file may already exist and have the same
|
|
- url, in which case Nothing is returned as it does not need
|
|
- to be re-downloaded. -}
|
|
makeunique n file = ifM alreadyexists
|
|
( ifM forced
|
|
( lookupKey (toRawFilePath f) >>= \case
|
|
Just k -> checksameurl k
|
|
Nothing -> tryanother
|
|
, tryanother
|
|
)
|
|
, return $ Just f
|
|
)
|
|
where
|
|
f = if n < 2
|
|
then file
|
|
else
|
|
let (d, base) = splitFileName file
|
|
in d </> show n ++ "_" ++ base
|
|
tryanother = makeunique (n + 1) file
|
|
alreadyexists = liftIO $ isJust <$> catchMaybeIO (R.getSymbolicLinkStatus (toRawFilePath f))
|
|
checksameurl k = ifM (elem url . map fst . map getDownloader <$> getUrls k)
|
|
( return Nothing
|
|
, tryanother
|
|
)
|
|
|
|
startUrlDownload :: TMVar Bool -> ToDownload -> URLString -> CommandPerform -> CommandStart
|
|
startUrlDownload cv todownload url a = do
|
|
starting "addurl"
|
|
(ActionItemOther (Just (UnquotedString url)))
|
|
(SeekInput [feedurl todownload])
|
|
(go `onException` recordfailure)
|
|
where
|
|
recordfailure = do
|
|
void $ feedProblem url "download failed"
|
|
liftIO $ atomically $ tryPutTMVar cv False
|
|
go = do
|
|
maybeAddJSONField "url" url
|
|
a
|
|
|
|
defaultTemplate :: String
|
|
defaultTemplate = "${feedtitle}/${itemtitle}${extension}"
|
|
|
|
{- Generates a filename to use for a feed item by filling out the template.
|
|
- The filename may not be unique. -}
|
|
feedFile :: Utility.Format.Format -> ToDownload -> String -> FilePath
|
|
feedFile tmpl i extension = sanitizeLeadingFilePathCharacter $
|
|
Utility.Format.format tmpl $
|
|
M.map sanitizeFilePathComponent $ M.fromList $ itemfields i ++
|
|
[ ("extension", extension)
|
|
, extractField "itempubdate" [itempubdatestring]
|
|
, extractField "itempubyear" [itempubyear]
|
|
, extractField "itempubmonth" [itempubmonth]
|
|
, extractField "itempubday" [itempubday]
|
|
, extractField "itempubhour" [itempubhour]
|
|
, extractField "itempubminute" [itempubminute]
|
|
, extractField "itempubsecond" [itempubsecond]
|
|
]
|
|
where
|
|
pubdate = maybe Nothing eitherToMaybe (itempubdate i)
|
|
|
|
itempubdatestring = case itempubdate i of
|
|
Just (Right pd) -> Just $ formatTime defaultTimeLocale "%F" pd
|
|
-- if date cannot be parsed, use the raw string
|
|
Just (Left s) -> Just $ replace "/" "-" s
|
|
Nothing -> Nothing
|
|
|
|
(itempubyear, itempubmonth, itempubday) = case pubdate of
|
|
Nothing -> (Nothing, Nothing, Nothing)
|
|
Just pd ->
|
|
let (y, m, d) = toGregorian (utctDay pd)
|
|
in (Just (show y), Just (show m), Just (show d))
|
|
|
|
(itempubhour, itempubminute, itempubsecond) = case pubdate of
|
|
Nothing -> (Nothing, Nothing, Nothing)
|
|
Just pd ->
|
|
let tod = timeToTimeOfDay (utctDayTime pd)
|
|
in ( Just (show (todHour tod))
|
|
, Just (show (todMin tod))
|
|
-- avoid fractional seconds
|
|
, Just (takeWhile (/= '.') (show (todSec tod)))
|
|
)
|
|
|
|
extractMetaData :: ToDownload -> MetaData
|
|
extractMetaData i = case itempubdate i of
|
|
Just (Right d) -> unionMetaData meta (dateMetaData d meta)
|
|
_ -> meta
|
|
where
|
|
tometa (k, v) = (mkMetaFieldUnchecked (T.pack k), S.singleton (toMetaValue (encodeBS v)))
|
|
meta = MetaData $ M.fromList $ map tometa $ itemfields i
|
|
|
|
minimalMetaData :: ToDownload -> MetaData
|
|
minimalMetaData i = case itemid i of
|
|
Nothing -> emptyMetaData
|
|
Just iid -> MetaData $ M.singleton itemIdField
|
|
(S.singleton $ toMetaValue iid)
|
|
|
|
noneValue :: String
|
|
noneValue = "none"
|
|
|
|
extractField :: String -> [Maybe String] -> (String, String)
|
|
extractField k [] = (k, noneValue)
|
|
extractField k (Just v:_)
|
|
| not (null v) = (k, v)
|
|
extractField k (_:rest) = extractField k rest
|
|
|
|
extractFeedItemFields :: Feed -> Item -> URLString -> [(String, String)]
|
|
extractFeedItemFields f i u = map (uncurry extractField)
|
|
[ ("feedurl", [Just u])
|
|
, ("feedtitle", [feedtitle])
|
|
, ("itemtitle", [itemtitle])
|
|
, ("feedauthor", [feedauthor])
|
|
, ("itemauthor", [itemauthor])
|
|
, ("itemsummary", [decodeBS . fromFeedText <$> getItemSummary i])
|
|
, ("itemdescription", [decodeBS . fromFeedText <$> getItemDescription i])
|
|
, ("itemrights", [decodeBS . fromFeedText <$> getItemRights i])
|
|
, ("itemid", [decodeBS . fromFeedText . snd <$> getItemId i])
|
|
, ("title", [itemtitle, feedtitle])
|
|
, ("author", [itemauthor, feedauthor])
|
|
]
|
|
where
|
|
feedtitle = Just $ decodeBS $ fromFeedText $ getFeedTitle f
|
|
itemtitle = decodeBS . fromFeedText <$> getItemTitle i
|
|
feedauthor = decodeBS . fromFeedText <$> getFeedAuthor f
|
|
itemauthor = decodeBS . fromFeedText <$> getItemAuthor i
|
|
|
|
playlistFields :: URLString -> YoutubePlaylistItem -> [(String, String)]
|
|
playlistFields u i = map (uncurry extractField)
|
|
[ ("feedurl", [Just u])
|
|
, ("feedtitle", [youtube_playlist_title i])
|
|
, ("itemtitle", [youtube_title i])
|
|
, ("feedauthor", [youtube_playlist_uploader i])
|
|
, ("itemauthor", [youtube_playlist_uploader i])
|
|
-- itemsummary omitted, no equivilant in yt-dlp data
|
|
, ("itemdescription", [youtube_description i])
|
|
, ("itemrights", [youtube_license i])
|
|
, ("itemid", [youtube_url i])
|
|
, ("title", [youtube_title i, youtube_playlist_title i])
|
|
, ("author", [youtube_playlist_uploader i])
|
|
]
|
|
|
|
playlistDownloads :: URLString -> [YoutubePlaylistItem] -> [ToDownload]
|
|
playlistDownloads url = mapMaybe go
|
|
where
|
|
go i = do
|
|
iurl <- youtube_url i
|
|
return $ ToDownload
|
|
{ feedurl = url
|
|
, location = MediaLink iurl
|
|
, itemid = Just (encodeBS iurl)
|
|
, itempubdate =
|
|
Right . posixSecondsToUTCTime . fromIntegral
|
|
<$> youtube_timestamp i
|
|
, itemfields = playlistFields url i
|
|
, youtubedlscraped = True
|
|
}
|
|
|
|
{- Called when there is a problem with a feed.
|
|
-
|
|
- If the feed has been broken for some time,
|
|
- returns False, otherwise only warns. -}
|
|
feedProblem :: URLString -> String -> Annex Bool
|
|
feedProblem url message = ifM (checkFeedBroken url)
|
|
( do
|
|
warning $ UnquotedString $ message ++ " (having repeated problems with feed: " ++ url ++ ")"
|
|
return False
|
|
, do
|
|
warning $ UnquotedString $ "warning: " ++ message ++ " (feed: " ++ url ++ ")"
|
|
return True
|
|
)
|
|
|
|
{- A feed is only broken if problems have occurred repeatedly, for at
|
|
- least 23 hours. -}
|
|
checkFeedBroken :: URLString -> Annex Bool
|
|
checkFeedBroken url = checkFeedBroken' url =<< feedState url
|
|
checkFeedBroken' :: URLString -> RawFilePath -> Annex Bool
|
|
checkFeedBroken' url f = do
|
|
prev <- maybe Nothing readish
|
|
<$> liftIO (catchMaybeIO $ readFile (fromRawFilePath f))
|
|
now <- liftIO getCurrentTime
|
|
case prev of
|
|
Nothing -> do
|
|
writeLogFile f $ show now
|
|
return False
|
|
Just prevtime -> do
|
|
let broken = diffUTCTime now prevtime > 60 * 60 * 23
|
|
when broken $
|
|
-- Avoid repeatedly complaining about
|
|
-- broken feed.
|
|
clearFeedProblem url
|
|
return broken
|
|
|
|
clearFeedProblem :: URLString -> Annex ()
|
|
clearFeedProblem url =
|
|
void $ liftIO . tryIO . removeFile . fromRawFilePath
|
|
=<< feedState url
|
|
|
|
feedState :: URLString -> Annex RawFilePath
|
|
feedState url = fromRepo $ gitAnnexFeedState $ fromUrl url Nothing False
|
|
|
|
{- The feed library parses the feed to Text, and does not use the
|
|
- filesystem encoding to do it, so when the locale is not unicode
|
|
- capable, a Text value can still include unicode characters.
|
|
-
|
|
- So, it's not safe to use T.unpack to convert that to a String,
|
|
- because later use of that String by eg encodeBS will crash
|
|
- with an encoding error. Use this instead.
|
|
-
|
|
- This should not be used on a Text that is read using the
|
|
- filesystem encoding because it does not reverse that encoding.
|
|
-}
|
|
fromFeedText :: T.Text -> B.ByteString
|
|
fromFeedText = TE.encodeUtf8
|