diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index aedf418f87..583a590ae4 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -9,7 +9,6 @@
 
 module Annex.Ssh (
 	sshCachingOptions,
-	sshCleanup,
 	sshCacheDir,
 	sshReadPort,
 ) where
@@ -24,6 +23,7 @@ import qualified Build.SysConfig as SysConfig
 import qualified Annex
 import Config
 import Utility.Env
+import Types.CleanupActions
 #ifndef mingw32_HOST_OS
 import Annex.Perms
 #endif
@@ -31,7 +31,9 @@ import Annex.Perms
 {- Generates parameters to ssh to a given host (or user@host) on a given
  - port, with connection caching. -}
 sshCachingOptions :: (String, Maybe Integer) -> [CommandParam] -> Annex [CommandParam]
-sshCachingOptions (host, port) opts = go =<< sshInfo (host, port)
+sshCachingOptions (host, port) opts = do
+	Annex.addCleanup SshCachingCleanup sshCleanup
+	go =<< sshInfo (host, port)
   where
 	go (Nothing, params) = ret params
 	go (Just socketfile, params) = do
@@ -146,6 +148,7 @@ sshCleanup = go =<< sshCacheDir
 					[ Params "-O stop"
 					] ++ params ++ [Param "any"])
 					{ cwd = Just dir }
+		liftIO $ nukeFile socketfile
 		-- Cannot remove the lock file; other processes may
 		-- be waiting on our exclusive lock to use it.
 
diff --git a/CmdLine.hs b/CmdLine.hs
index a920898dce..a165b041ac 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -26,7 +26,6 @@ import qualified Annex
 import qualified Git
 import qualified Git.AutoCorrect
 import Annex.Content
-import Annex.Ssh
 import Annex.Environment
 import Command
 import Types.Messages
@@ -107,4 +106,3 @@ shutdown nocommit = do
 	saveState nocommit
 	sequence_ =<< M.elems <$> Annex.getState Annex.cleanup
 	liftIO reapZombies -- zombies from long-running git processes
-	sshCleanup -- ssh connection caching
diff --git a/Types/CleanupActions.hs b/Types/CleanupActions.hs
index decd2f8734..498d5b4d7d 100644
--- a/Types/CleanupActions.hs
+++ b/Types/CleanupActions.hs
@@ -13,4 +13,5 @@ data CleanupAction
 	= RemoteCleanup UUID
 	| StopHook UUID
 	| FsckCleanup
+	| SshCachingCleanup
 	deriving (Eq, Ord)