diff --git a/CHANGELOG b/CHANGELOG
index 7d40230ff9..b4d11b4c96 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -11,6 +11,8 @@ git-annex (7.20191219) UNRELEASED; urgency=medium
     to more easily set a default that will also be used by clones,
     without needing to shoehorn the expression into the gitattributes file.
     The git config and gitattributes override that.
+  * git-annex-config --set/--unset: No longer change the local git config
+    setting, except for in the special case of annex.securehashesonly.
 
  -- Joey Hess <id@joeyh.name>  Wed, 18 Dec 2019 15:12:40 -0400
 
diff --git a/Command/Config.hs b/Command/Config.hs
index fb64dfdf90..79c78a1685 100644
--- a/Command/Config.hs
+++ b/Command/Config.hs
@@ -5,6 +5,8 @@
  - Licensed under the GNU AGPL version 3 or higher.
  -}
 
+{-# LANGUAGE OverloadedStrings #-}
+
 module Command.Config where
 
 import Command
@@ -54,12 +56,14 @@ seek :: Action -> CommandSeek
 seek (SetConfig ck@(ConfigKey name) val) = commandAction $
 	startingUsualMessages (decodeBS' name) (ActionItemOther (Just (fromConfigValue val))) $ do
 		setGlobalConfig ck val
-		setConfig ck (fromConfigValue val)
+		when (needLocalUpdate ck) $
+			setConfig ck (fromConfigValue val)
 		next $ return True
 seek (UnsetConfig ck@(ConfigKey name)) = commandAction $
 	startingUsualMessages (decodeBS' name) (ActionItemOther (Just "unset")) $do
 		unsetGlobalConfig ck
-		unsetConfig ck
+		when (needLocalUpdate ck) $
+			unsetConfig ck
 		next $ return True
 seek (GetConfig ck) = commandAction $
 	startingCustomOutput (ActionItemOther Nothing) $ do
@@ -67,3 +71,7 @@ seek (GetConfig ck) = commandAction $
 			Nothing -> return ()
 			Just (ConfigValue v) -> liftIO $ S8.putStrLn v
 		next $ return True
+
+needLocalUpdate :: ConfigKey -> Bool
+needLocalUpdate (ConfigKey "annex.securehashesonly") = True
+needLocalUpdate _ = False