diff --git a/CmdLine/GitAnnex.hs b/CmdLine/GitAnnex.hs
index f1bb96d3f9..d9aa23b5d9 100644
--- a/CmdLine/GitAnnex.hs
+++ b/CmdLine/GitAnnex.hs
@@ -120,9 +120,7 @@ import qualified Command.WebApp
 import qualified Command.Test
 import qualified Command.FuzzTest
 import qualified Command.TestRemote
-#ifdef WITH_BENCHMARK
 import qualified Command.Benchmark
-#endif
 
 cmds :: Parser TestOptions -> TestRunner -> MkBenchmarkGenerator -> [Command]
 cmds testoptparser testrunner mkbenchmarkgenerator = 
@@ -229,10 +227,8 @@ cmds testoptparser testrunner mkbenchmarkgenerator =
 	, Command.Test.cmd testoptparser testrunner
 	, Command.FuzzTest.cmd
 	, Command.TestRemote.cmd
-#ifdef WITH_BENCHMARK
 	, Command.Benchmark.cmd $
 		mkbenchmarkgenerator $ cmds testoptparser testrunner (\_ _ -> return noop)
-#endif
 	]
 
 run :: Parser TestOptions -> TestRunner -> MkBenchmarkGenerator -> [String] -> IO ()
diff --git a/Command/Benchmark.hs b/Command/Benchmark.hs
index e1e2cf84f7..7af2ed2b18 100644
--- a/Command/Benchmark.hs
+++ b/Command/Benchmark.hs
@@ -5,15 +5,17 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# OPTIONS_GHC -fno-warn-orphans #-}
+{-# LANGUAGE CPP #-}
 
 module Command.Benchmark where
 
 import Command
 import Types.Benchmark
 
+#ifdef WITH_BENCHMARK
 import Criterion.Main
 import Criterion.Main.Options (parseWith, Mode)
+#endif
 
 cmd :: BenchmarkGenerator -> Command
 cmd generator = command "benchmark" SectionTesting
@@ -21,15 +23,27 @@ cmd generator = command "benchmark" SectionTesting
 	paramNothing
 	(seek generator <$$> optParser)
 
+#ifndef WITH_BENCHMARK
+type Mode = ()
+#endif
+
 data BenchmarkOptions = BenchmarkOptions CmdParams Mode
 
 optParser :: CmdParamsDesc -> Parser BenchmarkOptions
 optParser desc = BenchmarkOptions
 	<$> cmdParams desc
+#ifdef WITH_BENCHMARK
 	-- parse criterion's options
 	<*> parseWith defaultConfig
+#else
+	<*> pure ()
+#endif
 
 seek :: BenchmarkGenerator -> BenchmarkOptions -> CommandSeek
+#ifdef WITH_BENCHMARK
 seek generator (BenchmarkOptions ps mode) = do
 	runner <- generator ps
 	liftIO $ runMode mode [ bench (unwords ps) $ nfIO runner ]
+#else
+seek _ _ = giveup "git-annex is not built with benchmarking support"
+#endif
diff --git a/git-annex.cabal b/git-annex.cabal
index 6af2430150..e0b8d9fbd8 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -588,7 +588,6 @@ Executable git-annex
   if flag(Benchmark)
     Build-Depends: criterion, deepseq
     CPP-Options: -DWITH_BENCHMARK
-    Other-Modules: Command.Benchmark
   
   if flag(DebugLocks)
     CPP-Options: -DDEBUGLOCKS
@@ -690,6 +689,7 @@ Executable git-annex
     Command.AddUnused
     Command.AddUrl
     Command.Adjust
+    Command.Benchmark
     Command.CalcKey
     Command.CheckPresentKey
     Command.Commit