git-annex log --received modifier option

Only counting received and not dropped makes this show the bandwidth of
data coming into the repository, although only in a sense. Since
git-annex branch updates only happen at the end of a command, and we
don't know when a command started, it's only an approximation of the
actual bandwidth. (A previous git-annex branch update made have
happened in a different repository.)

It would be possible to also add a --dropped option, but I don't know
how useful that would be?

Sponsored-by: Nicholas Golder-Manning on Patreon

CHANGELOG

@@ -15,6 +15,8 @@ git-annex (10.20230927) UNRELEASED; urgency=medium
   * info: Added calculation of combined annex size of all repositories.
   * log: Added options --sizesof, --sizes and --totalsizes that
     display how the size of repositories changed over time.
+  * log: Added options --interval, --bytes, --received 
+    to tune the output of the above added options.
 
  -- Joey Hess <id@joeyh.name>  Tue, 10 Oct 2023 13:17:31 -0400
 

Command/Log.hs

@@ -50,7 +50,8 @@ data LogOptions = LogOptions
 	, sizesOfOption :: Maybe (DeferredParse UUID)
 	, sizesOption :: Bool
 	, totalSizesOption :: Bool
-	, whenOption :: Maybe Duration
+	, intervalOption :: Maybe Duration
+	, receivedOption :: Bool
 	, rawDateOption :: Bool
 	, bytesOption :: Bool
 	, gourceOption :: Bool
@@ -83,6 +84,10 @@ optParser desc = LogOptions
 		( long "interval" <> metavar paramTime
 		<> help "minimum time between displays of changed size"
 		))
+	<*> switch
+		( long "received"
+		<> help "display received data per interval rather than repository sizes"
+		)
 	<*> switch
 		( long "raw-date"
 		<> help "display seconds from unix epoch"
@@ -295,7 +300,7 @@ sizeHistoryInfo mu o = do
 	uuidmap <- getuuidmap
 	zone <- liftIO getCurrentTimeZone
 	liftIO $ displayheader uuidmap
-	let dispst = (zone, False, epoch, Nothing)
+	let dispst = (zone, False, epoch, Nothing, mempty)
 	(l, cleanup) <- getlog
 	g <- Annex.gitRepo
 	liftIO $ catObjectStream g $ \feeder closer reader -> do
@@ -379,7 +384,9 @@ sizeHistoryInfo mu o = do
 		combinedlog = PLog.compactLog (oldlog ++ newlog)
 		combinedlocs = S.fromList (presentlocs combinedlog)
 		addedlocs = S.difference combinedlocs oldlocs
-		removedlocs = S.difference oldlocs combinedlocs
+		removedlocs
+			| receivedOption o = S.empty
+			| otherwise = S.difference oldlocs combinedlocs
 	
 	addnew k sizemap locmap newlog = 
 		( updatesize sizemap (ksz k) locs
@@ -402,23 +409,36 @@ sizeHistoryInfo mu o = do
 				(M.elems uuidmap)
 		| otherwise = return ()
 
-	displaysizes (zone, displayedyet, prevt, prevoutput) trustlog uuidmap sizemap t
+	displaysizes (zone, displayedyet, prevt, prevoutput, prevsizemap) trustlog uuidmap sizemap t
-		| t - prevt >= dt
+		| t - prevt >= dt && changedoutput = do
-		  && (displayedyet || any (/= 0) sizes) 
-		  && (prevoutput /= Just output) = do
 			displayts zone t output
-			return (zone, True, t, Just output)
+			return (zone, True, t, Just output, sizemap')
-		| t < prevt = return (zone, displayedyet, t, Just output)
+		| t < prevt = return (zone, displayedyet, t, Just output, prevsizemap)
-		| otherwise = return (zone, displayedyet, prevt, Just output)
+		| otherwise = return (zone, displayedyet, prevt, prevoutput, prevsizemap)
 	  where
 		output = intercalate "," (map showsize sizes)
 		us = case mu of
 			Just u -> [u]
 			Nothing -> M.keys uuidmap
 		sizes
-			| totalSizesOption o = [sum (M.elems sizemap')]
+			| totalSizesOption o = [sum (M.elems sizedisplaymap)]
-			| otherwise = map (\u -> fromMaybe 0 (M.lookup u sizemap')) us
+			| otherwise = map (\u -> fromMaybe 0 (M.lookup u sizedisplaymap)) us
-		dt = maybe 1 durationToPOSIXTime (whenOption o)
+		dt = maybe 1 durationToPOSIXTime (intervalOption o)
+
+		changedoutput
+			| receivedOption o = 
+				any (/= 0) sizes 
+					|| prevoutput /= Just output
+			| otherwise = 
+				(displayedyet || any (/= 0) sizes)
+					&& (prevoutput /= Just output)
+
+		sizedisplaymap
+			| receivedOption o = 
+				M.unionWith posminus sizemap' prevsizemap
+			| otherwise = sizemap'
+
+		posminus a b = max 0 (a - b)
 
 		-- A verison of sizemap where uuids that are currently dead
 		-- have 0 size.
@@ -433,7 +453,7 @@ sizeHistoryInfo mu o = do
 			then rawTimeStamp t
 			else showTimeStamp zone "%Y-%m-%dT%H:%M:%S" t
 
-	displayendsizes (zone , _, _, Just output) = do
+	displayendsizes (zone , _, _, Just output, _) = do
 		now <- getPOSIXTime
 		displayts zone now output
 	displayendsizes _ = return ()

doc/git-annex-log.mdwn

@@ -41,7 +41,7 @@ false, information may not have been committed to the branch yet.
 * `--sizesof=repository`
 
   Displays a history of the total size of the annexed files in a repository
-  as it changed over time from the creation of the repository to the present.
+  over time from the creation of the repository to the present.
 
   The repository can be "here" for the current repository, or the name of a
   remote, or a repository description or uuid.
@@ -65,10 +65,25 @@ false, information may not have been committed to the branch yet.
 
   When using `--sizesof`, `--sizes`, and `--totalsizes`, this 
   controls the minimum interval between displays of the size.
-  The default is to display each recorded change to the size.
+  The default is to display each new recorded size.
 
   The time is of the form "30d" or "1y".
 
+* `--received`
+
+  Combine this option with `--sizesof` or `--sizes` to display
+  the amount of data received into repositories since the last
+  line was output.
+
+* `--bytes`
+
+  Show sizes in bytes, disabling the default nicer units.
+
+* `--raw-date`
+
+  Rather than the normal display of a date in the local time zone,
+  displays seconds since the unix epoch.
+
 * `--since=date`, `--after=date`, `--until=date`, `--before=date`, `--max-count=N`
 
   These options are passed through to `git log`, and can be used to limit
@@ -79,15 +94,6 @@ false, information may not have been committed to the branch yet.
   These options do not have an affect when using `--sizesof`, `--sizes`, 
   and `--totalsizes`.
 
-* `--bytes`
-
-  Show sizes in bytes, disabling the default nicer units.
-
-* `--raw-date`
-
-  Rather than the normal display of a date in the local time zone,
-  displays seconds since the unix epoch.
-
 * `--gource`
 
   Generates output suitable for the `gource` visualization program.