diff --git a/Annex/Import.hs b/Annex/Import.hs index 1ab5c01aba..c16eb18213 100644 --- a/Annex/Import.hs +++ b/Annex/Import.hs @@ -184,10 +184,13 @@ recordImportTree remote importtreeconfig importable = do unlessM (stillpresent db oldkey) $ logChange oldkey (Remote.uuid remote) InfoMissing _ -> noop - db <- Export.openDb (Remote.uuid remote) - forM_ (exportedTreeishes oldexport) $ \oldtree -> - Export.runExportDiffUpdater updater db oldtree finaltree - Export.closeDb db + -- When the remote is versioned, it still contains keys + -- that are not present in the new tree. + unless (Remote.versionedExport (Remote.exportActions remote)) $ do + db <- Export.openDb (Remote.uuid remote) + forM_ (exportedTreeishes oldexport) $ \oldtree -> + Export.runExportDiffUpdater updater db oldtree finaltree + Export.closeDb db buildImportCommit' :: Remote -> ImportCommitConfig -> Maybe Sha -> History Sha -> Annex (Maybe Sha) buildImportCommit' remote importcommitconfig mtrackingcommit imported@(History ti _) = diff --git a/CHANGELOG b/CHANGELOG index c393dad101..56f2c781fe 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,8 +6,10 @@ git-annex (10.20221004) UNRELEASED; urgency=medium do not operate on a repository that has an empty name. * move: Fix openFile crash with -J (Fixes a reversion in 8.20201103) - * S3: Speed up importing from a large bucket when fileprefix= is set + * S3: Speed up importing from a large bucket when fileprefix= is set, by only asking for files under the prefix. + * When importing from versioned remotes, fix tracking of the content + of deleted files. -- Joey Hess Mon, 03 Oct 2022 13:36:42 -0400 diff --git a/Command/Export.hs b/Command/Export.hs index 3c5b3ad214..d331bb3030 100644 --- a/Command/Export.hs +++ b/Command/Export.hs @@ -378,7 +378,7 @@ cleanupUnexport r db eks loc = do removeExportedLocation db ek loc flushDbQueue db - -- An versionedExport remote supports removeExportLocation to remove + -- A versionedExport remote supports removeExportLocation to remove -- the file from the exported tree, but still retains the content -- and allows retrieving it. unless (versionedExport (exportActions r)) $ do diff --git a/Types/Remote.hs b/Types/Remote.hs index e9e8a8c815..121d4fd5cd 100644 --- a/Types/Remote.hs +++ b/Types/Remote.hs @@ -270,9 +270,10 @@ data ExportActions a = ExportActions -- Can throw exception if unable to access remote, or if remote -- refuses to remove the content. , removeExport :: Key -> ExportLocation -> a () - -- Set when the content of a Key stored in the remote to an - -- ExportLocation and then removed with removeExport remains - -- accessible to retrieveKeyFile and checkPresent. + -- Set when the remote is versioned, so once a Key is stored + -- to an ExportLocation, a subsequent deletion of that + -- ExportLocation leaves the key still accessible to retrieveKeyFile + -- and checkPresent. , versionedExport :: Bool -- Removes an exported directory. Typically the directory will be -- empty, but it could possibly contain files or other directories, diff --git a/doc/bugs/annex_import_does_not_account_for_versioning_on_S3.mdwn b/doc/bugs/annex_import_does_not_account_for_versioning_on_S3.mdwn index d631fac177..4f6229e34c 100644 --- a/doc/bugs/annex_import_does_not_account_for_versioning_on_S3.mdwn +++ b/doc/bugs/annex_import_does_not_account_for_versioning_on_S3.mdwn @@ -13,3 +13,4 @@ Observed with several versions from 8-10 on linux. As laid out above, I strongly Lots. I love git-annex. +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/annex_import_does_not_account_for_versioning_on_S3/comment_1_a04fbb936785456ed99512ea4c29fd53._comment b/doc/bugs/annex_import_does_not_account_for_versioning_on_S3/comment_1_a04fbb936785456ed99512ea4c29fd53._comment new file mode 100644 index 0000000000..c3d8aa057b --- /dev/null +++ b/doc/bugs/annex_import_does_not_account_for_versioning_on_S3/comment_1_a04fbb936785456ed99512ea4c29fd53._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2022-10-11T16:28:39Z" + content=""" +This looks like a simple fix. After importing from a versioned remote, +it can just skip updating the location logs to remove the keys that are not +present in the current tree. The same as is already done when exporting +to a versioned remote. I've made that change. +"""]] diff --git a/doc/design/external_special_remote_protocol/export_and_import_appendix.mdwn b/doc/design/external_special_remote_protocol/export_and_import_appendix.mdwn index 24004402fe..b2f07bad0b 100644 --- a/doc/design/external_special_remote_protocol/export_and_import_appendix.mdwn +++ b/doc/design/external_special_remote_protocol/export_and_import_appendix.mdwn @@ -150,6 +150,13 @@ support a request, it can reply with `UNSUPPORTED-REQUEST`. Indicates that `IMPORTKEY` can be used. * `IMPORTKEYSUPPORTED-FAILURE` Indicates that `IMPORTKEY` cannot be used. + * `VERSIONED` + Used to check if the special remote is versioned. + Note that this request may be made before or after `PREPARE`. + * `ISVERSIONED` + Indicates that the remote is versioned. + * `NOTVERSIONED` + Indicates that the remote is not versioned. * `LISTIMPORTABLECONTENTS` Used to get a list of all the files that are stored in the special remote. A block of responses @@ -170,6 +177,8 @@ support a request, it can reply with `UNSUPPORTED-REQUEST`. be nested multiple levels deep. This should only be used when the remote supports using "TRANSFER RECEIVE Key" to retrieve historical versions of files. + And, it should only be used when the remote replies `ISVERSIONED` + to the `VERSIONED` message. * `END` Indicates the end of a block of responses. * `LOCATION Name`