add git-annex filter-process

filter-process: New command that can make git add/checkout faster when
there are a lot of unlocked annexed files or non-annexed files, but that
also makes git add of large annexed files slower.

Use it by running: git
config filter.annex.process 'git-annex filter-process'

Fully tested and working, but I have not benchmarked it at all.
And, incremental hashing is not done when git add uses it, so extra work is
done in that case.

Sponsored-by: Mark Reidenbach on Patreon
This commit is contained in:
Joey Hess 2021-11-04 15:02:36 -04:00
parent d706b49979
commit 68257e9076
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
10 changed files with 254 additions and 43 deletions

View file

@ -15,17 +15,18 @@ module Git.FilterProcess (
Version(..),
Capability(..),
readUntilFlushPkt,
readUntilFlushPktOrSize,
discardUntilFlushPkt,
longRunningProcessHandshake,
longRunningFilterProcessHandshake,
FilterRequest(..),
getFilterRequest,
respondFilterRequest,
) where
import Common
import Git.PktLine
import System.IO
import qualified Data.ByteString as B
{- This is a message like "git-filter-client" or "git-filter-server" -}
@ -85,6 +86,24 @@ readUntilFlushPkt = go []
Just pktline | not (isFlushPkt pktline) -> go (pktline:l)
_ -> return (reverse l)
{- Reads PktLines until at least the specified number of bytes have been
- read, or until a flushPkt (or EOF). Returns Right if it did read a
- flushPkt/EOF, and Left if there is still content leftover that needs to
- be read. -}
readUntilFlushPktOrSize :: Int -> IO (Either [PktLine] [PktLine])
readUntilFlushPktOrSize = go []
where
go l n = readPktLine stdin >>= \case
Just pktline
| isFlushPkt pktline -> return (Right (reverse l))
| otherwise ->
let len = B.length (pktLineToByteString pktline)
n' = n - len
in if n' <= 0
then return (Left (reverse (pktline:l)))
else go (pktline:l) n'
Nothing -> return (Right (reverse l))
{- Reads PktLines until a flushPkt (or EOF), and throws them away. -}
discardUntilFlushPkt :: IO ()
discardUntilFlushPkt = readPktLine stdin >>= \case