mirrors/git-annex
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

addcomputed --fast and --unreproducible working

Browse source 
For these, use VURL and URL keys, with an "annex-compute:" URI prefix.

These URL keys will look something like this:

	URL--annex-compute&cbar4,63pconvert,3-f4d3d72cf3f16ac9c3e9a8012bde4462

Generally it's too long so most of it gets md5summed. It's a little
ugly, but it's what fell out of the existing URL key generation
machinery. I did consider special casing to eg
"URL--annex-compute&c4d3d72cf3f16ac9c3e9a8012bde4462". But it seems at
least possibly useful that the name of the file that was computed is
visible and perhaps one or two words of the git-annex compute command
parameters.

Note that two different output files from the same computation will get
the same URL key. And these keys should remain stable.
This commit is contained in:
Joey Hess 2025-02-25 16:36:22 -04:00
parent a154e91513
commit 16f529c05f
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
3 changed files with 50 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

56
Command/AddComputed.hs
View file

@ -20,8 +20,9 @@ import Annex.Ingest
import Types.RemoteConfig
import Types.KeySource
import Messages.Progress
import Utility.MonotonicClock
import Logs.Location
import Utility.MonotonicClock
import Backend.URL (fromUrl)
import qualified Data.Map as M
import Data.Time.Clock
@ -42,19 +43,19 @@ optParser :: CmdParamsDesc -> Parser AddComputedOptions
optParser desc = AddComputedOptions
<$> cmdParams desc
<*> (mkParseRemoteOption <$> parseToOption)
<*> (fromMaybe Unreproducible <$> parseReproducible)
<*> (fromMaybe (Reproducible False) <$> parseReproducible)
data Reproducible = Reproducible | Unreproducible
newtype Reproducible = Reproducible { isReproducible :: Bool }
parseReproducible :: Parser (Maybe Reproducible)
parseReproducible = r <|> unr
where
r = flag Nothing (Just Reproducible)
r = flag Nothing (Just (Reproducible True))
( long "reproducible"
<> short 'r'
<> help "computation is fully reproducible"
)
unr = flag Nothing (Just Unreproducible)
unr = flag Nothing (Just (Reproducible False))
( long "unreproducible"
<> short 'u'
<> help "computation is not fully reproducible"
@ -90,17 +91,14 @@ perform o r program = do
, Remote.Compute.computeInputs = mempty
, Remote.Compute.computeOutputs = mempty
, Remote.Compute.computeSubdir = subdir
, Remote.Compute.computeReproducible =
case reproducible o of
Reproducible -> True
Unreproducible -> False
, Remote.Compute.computeReproducible = isreproducible
}
fast <- Annex.getRead Annex.fast
starttime <- liftIO currentMonotonicTimestamp
Remote.Compute.runComputeProgram program state
(Remote.Compute.ImmutableState False)
(getinputcontent fast)
(go starttime)
(go starttime fast)
next $ return True
where
getinputcontent fast p = catKeyFile p >>= \case
@ -117,7 +115,7 @@ perform o r program = do
, giveup $ "The computation needs an input file which does not exist: " ++ fromOsPath p
)
go starttime state tmpdir = do
go starttime fast state tmpdir = do
endtime <- liftIO currentMonotonicTimestamp
let ts = calcduration starttime endtime
let outputs = Remote.Compute.computeOutputs state
@ -125,7 +123,7 @@ perform o r program = do
giveup "The computation succeeded, but it did not generate any files."
oks <- forM (M.keys outputs) $ \outputfile -> do
showAction $ "adding " <> QuotedPath outputfile
k <- catchNonAsync (addfile tmpdir outputfile)
k <- catchNonAsync (addfile fast state tmpdir outputfile)
(\err -> giveup $ "Failed to ingest output file " ++ fromOsPath outputfile ++ ": " ++ show err)
return (outputfile, Just k)
let state' = state
@ -137,24 +135,32 @@ perform o r program = do
k ts state'
logChange NoLiveUpdate k (Remote.uuid r) InfoPresent
addfile tmpdir outputfile = do
let outputfile' = tmpdir </> outputfile
let ld = LockedDown ldc $ KeySource
{ keyFilename = outputfile
, contentLocation = outputfile'
, inodeCache = Nothing
}
sz <- liftIO $ getFileSize outputfile'
metered Nothing sz Nothing $ \_ p ->
ingestAdd p (Just ld) >>= \case
Nothing -> giveup "key generation failed"
Just k -> return k
addfile fast state tmpdir outputfile
| fast || not isreproducible = do
let stateurl = Remote.Compute.computeStateUrl state outputfile
let k = fromUrl stateurl Nothing isreproducible
addSymlink outputfile k Nothing
return k
| otherwise = do
let outputfile' = tmpdir </> outputfile
let ld = LockedDown ldc $ KeySource
{ keyFilename = outputfile
, contentLocation = outputfile'
, inodeCache = Nothing
}
sz <- liftIO $ getFileSize outputfile'
metered Nothing sz Nothing $ \_ p ->
ingestAdd p (Just ld) >>= \case
Nothing -> giveup "key generation failed"
Just k -> return k
ldc = LockDownConfig
{ lockingFile = True
, hardlinkFileTmpDir = Nothing
, checkWritePerms = True
}
calcduration (MonotonicTimestamp starttime) (MonotonicTimestamp endtime) =
fromIntegral (endtime - starttime) :: NominalDiffTime
isreproducible = isReproducible (reproducible o)