From b9466ddce201bc7f9631f9e1d57151bcc1b825b9 Mon Sep 17 00:00:00 2001 From: "beryllium@5bc3c32eb8156390f96e363e4ba38976567425ec" Date: Fri, 31 Jan 2025 00:22:45 +0000 Subject: [PATCH 1/8] --- ...forcing_--historical._exportree__63__.mdwn | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 doc/bugs/whereused_--key_forcing_--historical._exportree__63__.mdwn diff --git a/doc/bugs/whereused_--key_forcing_--historical._exportree__63__.mdwn b/doc/bugs/whereused_--key_forcing_--historical._exportree__63__.mdwn new file mode 100644 index 0000000000..91bd7c9334 --- /dev/null +++ b/doc/bugs/whereused_--key_forcing_--historical._exportree__63__.mdwn @@ -0,0 +1,45 @@ +### Please describe the problem. + +Selectively auditing the files annexed on an S3 container, where the file is stored by keyname, I attempted to confirm that the file is also annexed locally. + +When I run git-annex whereused --key , nothing comes up. But if I look for the file locally, I can see that it exists. + +If I add --historical, I can see the file exists in some previous commit that I thought had been successfully merged, and moved on from. It seems like this might be related to export trees, because that keyword is also present in the output with --historical + +[[!format sh """ +$ git-annex whereused --key SHA256E-s1000013--e435522a9059bcb086b6db5fa5f05a06913266772a7931eefae2b8f7647f5f14.JPG +$ git-annex whereused --historical --key SHA256E-s1000013--e435522a9059bcb086b6db5fa5f05a06913266772a7931eefae2b8f7647f5f14.JPG +SHA256E-s1000013--e435522a9059bcb086b6db5fa5f05a06913266772a7931eefae2b8f7647f5f14.JPG remotes/origin/git-annex~12:export.tree/2010-08-21/042.JPG +$ git-annex contentlocation SHA256E-s1000013--e435522a9059bcb086b6db5fa5f05a06913266772a7931eefae2b8f7647f5f14.JPG +.git/annex/objects/0Z/Z3/SHA256E-s1000013--e435522a9059bcb086b6db5fa5f05a06913266772a7931eefae2b8f7647f5f14.JPG/SHA256E-s1000013--e435522a9059bcb086b6db5fa5f05a06913266772a7931eefae2b8f7647f5f14.JPG +$ ls -l $(git-annex contentlocation SHA256E-s1000013--e435522a9059bcb086b6db5fa5f05a06913266772a7931eefae2b8f7647f5f14.JPG) +-r--r--r-- 1 shaddy shaddy 1000013 May 21 2024 .git/annex/objects/0Z/Z3/SHA256E-s1000013--e435522a9059bcb086b6db5fa5f05a06913266772a7931eefae2b8f7647f5f14.JPG/SHA256E-s1000013--e435522a9059bcb086b6db5fa5f05a06913266772a7931eefae2b8f7647f5f14.JPG +$ git-annex findkeys | grep -F SHA256E-s1000013--e435522a9059bcb086b6db5fa5f05a06913266772a7931eefae2b8f7647f5f14.JPG +SHA256E-s1000013--e435522a9059bcb086b6db5fa5f05a06913266772a7931eefae2b8f7647f5f14.JPG +$ ls -l 2010-08-21/042.JPG +lrwxrwxrwx 1 shaddy shaddy 201 May 20 2024 2010-08-21/042.JPG -> ../.git/annex/objects/0Z/Z3/SHA256E-s1000013--e435522a9059bcb086b6db5fa5f05a06913266772a7931eefae2b8f7647f5f14.JPG/SHA256E-s1000013--e435522a9059bcb086b6db5fa5f05a06913266772a7931eefae2b8f7647f5f14.JPG +$ ls -lL 2010-08-21/042.JPG +-r--r--r-- 1 shaddy shaddy 1000013 May 21 2024 2010-08-21/042.JPG +$ +"""]] + +My reading of the documentation is that the file being present in the local annex shouldn't require the --historical argument, which is much slower. + + +### What steps will reproduce the problem? + +As per above + +### What version of git-annex are you using? On what operating system? + +git-annex/10.20241031-1~ndall+1 on Ubuntu 22.04 LTS: + +Linux computer-ubul 6.8.0-40-generic #40~22.04.3-Ubuntu SMP PREEMPT_DYNAMIC Tue Jul 30 17:30:19 UTC 2 x86_64 x86_64 x86_64 GNU/Linux + +### Please provide any additional information below. + +Nil + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +Love git-annex. Long time supporter. From 6ef9af58609391517e14958aa467a4c5f34857e6 Mon Sep 17 00:00:00 2001 From: lell Date: Tue, 4 Feb 2025 09:35:42 +0000 Subject: [PATCH 2/8] --- ..._refreshing_index_for_untracked_files.mdwn | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 doc/bugs/git_keeps_refreshing_index_for_untracked_files.mdwn diff --git a/doc/bugs/git_keeps_refreshing_index_for_untracked_files.mdwn b/doc/bugs/git_keeps_refreshing_index_for_untracked_files.mdwn new file mode 100644 index 0000000000..c2c6cf2b12 --- /dev/null +++ b/doc/bugs/git_keeps_refreshing_index_for_untracked_files.mdwn @@ -0,0 +1,54 @@ +### Please describe the problem. + +I have a repository that contains thousands (30000) of unlocked files. I see the problem that +git keeps refreshing its index despite the modified time did not change compared to the git index. +I think this should not happen. It significantly slows down (delay of ~10s in my case, but +potentially worse for Windows or NFS file systems) any git operation, and this includes +the display of a bash prompt that shows git information (via PROMPT_COMMAND env var.) and bash +completion after things like `git add`, therefore it is quite annoying. + +### What steps will reproduce the problem? + +I have set up a repository to reproduce the problem. However, it works as expected. I am looking for +ideas on how to find the problem with my repository. + +``` +git init . +git annex init +git annex config --set annex.addunlocked include="py_env/*" +python -m venv py_env +. py_env/bin/activate +pip install datalad # some package with dependencies that creates many files in py_env +git annex add py_env +git status +git status +git status +... +``` + +The only strange thing that I have found is that git annex will forward all `__init__.py` files to the smudge/clean-filter (pipes to filter-process), but only those! It does not do this for all other files in `py_env`, as you can check with + +```sh +strace git status 2>&1 | grep "^write.*pathname=" +``` + +this shows only the `__init__.py` files, despite `find py_env` shows many more files in there and they are skipped. I think that git skips files during `git status` if the mtime and the ctime are the same. But they are always the same, as can be checked by comparing + +```sh +git ls-files --debug "(any file that shows up in strace)" +stat -c $'ctime: %.9Z\nmtime: %.9Y' "(same file)" +``` + +I do not know how to debug further and are grateful for suggestions! + +### What version of git-annex are you using? On what operating system? + +git-annex version: 10.20250115-ga55e1da1aa0eb07b418e899cc200027e082eb82a +git version 2.47.1 +Rocky Linux + + + +### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + + From 34646124457004f693b19abdf0c6eaf15f833897 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Feb 2025 11:41:10 -0400 Subject: [PATCH 3/8] update --- doc/thanks/list | 78 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/doc/thanks/list b/doc/thanks/list index 1f590c5572..9a51514404 100644 --- a/doc/thanks/list +++ b/doc/thanks/list @@ -125,3 +125,81 @@ oz, Lilia.Nanne, Dusty Mabe, mpol, +rjbl, +Land Raider, +Andrew, +ers35, +Madison McGaffin, +Walltime, +Caleb Allen, +AlexS, +Ryan Newton, +don haraway, +Eric Drechsel, +DuncanConstruction, +Pedro Luz, +Christopher Goes, +mo, +Calvin Beck, +SvenDowideit, +fiatjaf, +Trent Lloyd, +Stephan Burkhardt, +Sergey Karpukhin, +Paul Tötterman, +jose_d, +Lukas Platz, +Mica, +tj, +Lee Hinman, +Thomas Schwinge, +Matthieu, +Dave Pifke, +Chris Lamb, +Stan Yamane, +Michal Politowski, +Shane-o, +Willard Korfhage, +Francois Marier, +Peter, +Stephan Meister, +Valeria_, +John Peloquin, +Jo, +anon, +Bruno BEAUFILS, +Renaud Casenave-Péré, +Ben, +Tyler Cipriani, +NinjaTrappeur, +Will Hughes, +Evgeni Ku, +Rémi Vanicat, +Ole-Morten Duesund, +Thomas Hochstein, +Thomas Ferris Nicolaisen, +Boyd Stephen Smith, +Christian Diller, +Art S, +Zoé Cassiopée Gauthier, +Arthur Lutz, +Fernando Jimenez, +Josh Taylor, +Jeff Goeke-Smith, +Matthias Urlichs, +Mattias J, +sww, +encryptio, +Jon D, +James Valleroy, +Thom May, +Amitai Schleier, +Johannes Schlatow, +Lacnic, +Jim, +Jason Woofenden, +Ignacio, +Shawn Butts, +Mika Pflüger, +Anthony DeRobertis, +Christopher Baines, From 874882efc48ff80326442afaf8fdf8a8a5655355 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Feb 2025 11:41:37 -0400 Subject: [PATCH 4/8] Revert "update" This reverts commit 34646124457004f693b19abdf0c6eaf15f833897. --- doc/thanks/list | 78 ------------------------------------------------- 1 file changed, 78 deletions(-) diff --git a/doc/thanks/list b/doc/thanks/list index 9a51514404..1f590c5572 100644 --- a/doc/thanks/list +++ b/doc/thanks/list @@ -125,81 +125,3 @@ oz, Lilia.Nanne, Dusty Mabe, mpol, -rjbl, -Land Raider, -Andrew, -ers35, -Madison McGaffin, -Walltime, -Caleb Allen, -AlexS, -Ryan Newton, -don haraway, -Eric Drechsel, -DuncanConstruction, -Pedro Luz, -Christopher Goes, -mo, -Calvin Beck, -SvenDowideit, -fiatjaf, -Trent Lloyd, -Stephan Burkhardt, -Sergey Karpukhin, -Paul Tötterman, -jose_d, -Lukas Platz, -Mica, -tj, -Lee Hinman, -Thomas Schwinge, -Matthieu, -Dave Pifke, -Chris Lamb, -Stan Yamane, -Michal Politowski, -Shane-o, -Willard Korfhage, -Francois Marier, -Peter, -Stephan Meister, -Valeria_, -John Peloquin, -Jo, -anon, -Bruno BEAUFILS, -Renaud Casenave-Péré, -Ben, -Tyler Cipriani, -NinjaTrappeur, -Will Hughes, -Evgeni Ku, -Rémi Vanicat, -Ole-Morten Duesund, -Thomas Hochstein, -Thomas Ferris Nicolaisen, -Boyd Stephen Smith, -Christian Diller, -Art S, -Zoé Cassiopée Gauthier, -Arthur Lutz, -Fernando Jimenez, -Josh Taylor, -Jeff Goeke-Smith, -Matthias Urlichs, -Mattias J, -sww, -encryptio, -Jon D, -James Valleroy, -Thom May, -Amitai Schleier, -Johannes Schlatow, -Lacnic, -Jim, -Jason Woofenden, -Ignacio, -Shawn Butts, -Mika Pflüger, -Anthony DeRobertis, -Christopher Baines, From 09d47726b7215642a6838fb09efd353fbcb4ddbc Mon Sep 17 00:00:00 2001 From: thk Date: Sat, 8 Feb 2025 06:56:32 +0000 Subject: [PATCH 5/8] Added a comment: iroh --- ...t_7_129a82171aa038c00ff65615ad76e115._comment | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/todo/generic_p2p_socket_transport/comment_7_129a82171aa038c00ff65615ad76e115._comment diff --git a/doc/todo/generic_p2p_socket_transport/comment_7_129a82171aa038c00ff65615ad76e115._comment b/doc/todo/generic_p2p_socket_transport/comment_7_129a82171aa038c00ff65615ad76e115._comment new file mode 100644 index 0000000000..92a0388840 --- /dev/null +++ b/doc/todo/generic_p2p_socket_transport/comment_7_129a82171aa038c00ff65615ad76e115._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="thk" + avatar="http://cdn.libravatar.org/avatar/bfef10a428769701aeee1db978951461" + subject="iroh" + date="2025-02-08T06:56:31Z" + content=""" +Please take a look at [iroh](https://www.iroh.computer). It started as an IPFS implementation in rust, realized that IPFS is slow and overengineered and now pivoted to providing p2p connections with quic. + +* Peers/nodes/endpoints use ed25519 keys as identities. +* The iroh project hosts relay servers for initial NAT hole punching and as connection fallbacks. +* So far there are 4 initial [discovery](https://www.iroh.computer/docs/concepts/discovery) implementations: DNS, Local (mDNS), [Pkarr](https://github.com/nuhvi/pkarr) or Bittorrents Mainline DHT + +I'm waiting for their [FOSDEM talk](https://fosdem.org/2025/schedule/event/fosdem-2025-6053-building-peer-to-peer-quic/). But there is also a good presentation on YT: [A tour of iroh](https://www.youtube.com/watch?v=AkHaIVuFHK4&list=PLvsg-fc7APc3TOrumoQdjggwoLTSfiqRE&index=2). + + +"""]] From e333ef9337965a8ba978ba8abcae3e99bd4c1d5a Mon Sep 17 00:00:00 2001 From: thk Date: Sat, 8 Feb 2025 06:59:34 +0000 Subject: [PATCH 6/8] --- doc/users/thk.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/users/thk.mdwn b/doc/users/thk.mdwn index d44ca28c72..fe2c2564c6 100644 --- a/doc/users/thk.mdwn +++ b/doc/users/thk.mdwn @@ -1,8 +1,9 @@ -I'm thk at debian org +https://blog.koch.ro # My TODO items +- implement p2p for annex with iroh: https://git-annex.branchable.com/todo/generic_p2p_socket_transport/#comment-de273c852db02cb46a6bab4987429a3a - write a tip on using git worktree to inspect the git-annex branch - Is there a way to filter out the directories? - write a tip on how to deal with permission issues on ext formatted USB drives From 2f9a384e48cb4407e6b5b70d1db6efa593654f0e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 10 Feb 2025 17:22:29 -0400 Subject: [PATCH 7/8] stack.yaml: temporarily build with older ghc And without ospath build flag as a consequence. This is a temporary fix to build failures on the github CI for Windows and OSX, which use too old a version of stack to support the nightly ghc. I have sent a patch to those workflows, and after it is applied, this can be reverted. --- stack.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stack.yaml b/stack.yaml index 5ff6f33d09..e062bdde92 100644 --- a/stack.yaml +++ b/stack.yaml @@ -11,10 +11,10 @@ flags: benchmark: true crypton: true servant: true - ospath: true + ospath: false packages: - '.' -resolver: nightly-2025-01-20 +resolver: lts-23.2 extra-deps: - filepath-bytestring-1.5.2.0.2 - aws-0.24.4 From 2057bc1c2bb600bb85824792d103626e3b3c24dc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Feb 2025 16:53:01 -0400 Subject: [PATCH 8/8] fix windows and OSX packaging program builds Broken by commit e5be81f8d4bf7f6cef5ac4ff0b059efbdf6055ea --- Build/NullSoftInstaller.hs | 1 + Build/OSXMkLibs.hs | 1 + 2 files changed, 2 insertions(+) diff --git a/Build/NullSoftInstaller.hs b/Build/NullSoftInstaller.hs index ca209076d3..b8e3d4dd57 100644 --- a/Build/NullSoftInstaller.hs +++ b/Build/NullSoftInstaller.hs @@ -42,6 +42,7 @@ import Utility.SafeCommand import Utility.Process import Utility.Exception import Utility.Directory +import Utility.SystemDirectory import Build.BundledPrograms main = do diff --git a/Build/OSXMkLibs.hs b/Build/OSXMkLibs.hs index c680612d8e..5df85a757b 100644 --- a/Build/OSXMkLibs.hs +++ b/Build/OSXMkLibs.hs @@ -19,6 +19,7 @@ import Prelude import Utility.PartialPrelude import Utility.Directory +import Utility.SystemDirectory import Utility.Process import Utility.Monad import Utility.SafeCommand