mirrors/git-annex
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

importfeed --force: re-download urls that have been seen before

Browse source
This commit is contained in:
Joey Hess 2013-07-31 12:19:00 -04:00
parent 9476355bc3
commit 42ca8aaa61
2 changed files with 16 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
Command/ImportFeed.hs
View file

@ -15,6 +15,7 @@ import qualified Data.Map as M
import Data.Char import Data.Char
import Common.Annex import Common.Annex
import qualified Annex
import Command import Command
import qualified Utility.Url as Url import qualified Utility.Url as Url
import Logs.Web import Logs.Web
@ -68,12 +69,15 @@ data Cache = Cache
} }
getCache :: Maybe String -> Annex Cache getCache :: Maybe String -> Annex Cache
getCache opttemplate = do getCache opttemplate = ifM (Annex.getState Annex.force)
showSideAction "checking known urls" ( ret S.empty
us <- S.fromList <$> knownUrls , do
return $ Cache us tmpl showSideAction "checking known urls"
ret =<< S.fromList <$> knownUrls
)
where where
tmpl = Utility.Format.gen $ fromMaybe defaultTemplate opttemplate tmpl = Utility.Format.gen $ fromMaybe defaultTemplate opttemplate
ret s = return $ Cache s tmpl
findEnclosures :: URLString -> Annex (Maybe [ToDownload]) findEnclosures :: URLString -> Annex (Maybe [ToDownload])
findEnclosures url = go =<< downloadFeed url findEnclosures url = go =<< downloadFeed url
@ -96,19 +100,21 @@ downloadFeed url = do
) )
{- Avoids downloading any urls that are already known to be associated {- Avoids downloading any urls that are already known to be associated
- with a file in the annex. -} - with a file in the annex, unless forced. -}
downloadEnclosure :: Bool -> Cache -> ToDownload -> Annex () downloadEnclosure :: Bool -> Cache -> ToDownload -> Annex ()
downloadEnclosure relaxed cache enclosure downloadEnclosure relaxed cache enclosure
| S.member url (knownurls cache) = noop | S.member url (knownurls cache) =
| otherwise = do whenM (Annex.getState Annex.force) go
| otherwise = go
where
url = location enclosure
go = do
dest <- liftIO $ feedFile (template cache) enclosure dest <- liftIO $ feedFile (template cache) enclosure
showStart "addurl" dest showStart "addurl" dest
ifM (addUrlFile relaxed url dest) ifM (addUrlFile relaxed url dest)
( showEndOk ( showEndOk
, showEndFail , showEndFail
) )
where
url = location enclosure
defaultTemplate :: String defaultTemplate :: String
defaultTemplate = "${feedtitle}/${itemtitle}${extension}" defaultTemplate = "${feedtitle}/${itemtitle}${extension}"

2
doc/git-annex.mdwn
View file

@ -195,7 +195,7 @@ subdirectories).
Imports the contents of podcast feeds. Only downloads files whose Imports the contents of podcast feeds. Only downloads files whose
urls have not already been added to the repository before, so you can urls have not already been added to the repository before, so you can
delete, rename, etc the resulting files and repeated runs won't duplicate delete, rename, etc the resulting files and repeated runs won't duplicate
them. them. (Use --force to force downloading files it's seen before.)
Use --template to control where the files are stored. Use --template to control where the files are stored.
The default template is '${feedtitle}/${itemtitle}${extension}' The default template is '${feedtitle}/${itemtitle}${extension}'