From 2391c2802af487f2da4a44aa1512e4714e59195b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Mar 2025 16:02:11 -0400 Subject: [PATCH] add git-annex-compute-wasmedge --- COPYRIGHT | 2 +- doc/special_remotes/compute.mdwn | 7 +++ .../compute/git-annex-compute-wasmedge | 47 +++++++++++++++++ .../git-annex-compute-wasmedge-examples.mdwn | 52 +++++++++++++++++++ 4 files changed, 107 insertions(+), 1 deletion(-) create mode 100755 doc/special_remotes/compute/git-annex-compute-wasmedge create mode 100644 doc/special_remotes/compute/git-annex-compute-wasmedge-examples.mdwn diff --git a/COPYRIGHT b/COPYRIGHT index 7dfe659c6a..54a250abae 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -14,7 +14,7 @@ Files: doc/special_remotes/external/* Copyright: © 2013 Joey Hess License: GPL-3+ -Files: doc/special_remotes/compute/git-annex-compute-imageconvert +Files: doc/special_remotes/compute/git-annex-compute-imageconvert doc/special_remotes/compute/git-annex-compute-wasmedge Copyright: © 2025 Joey Hess License: GPL-3+ diff --git a/doc/special_remotes/compute.mdwn b/doc/special_remotes/compute.mdwn index 36bda5a62b..c85b1a9625 100644 --- a/doc/special_remotes/compute.mdwn +++ b/doc/special_remotes/compute.mdwn @@ -38,3 +38,10 @@ List it here with an example! Uses imagemagick to convert between image formats `git-annex addcomputed --to=imageconvert foo.jpeg foo.gif` + +* [[compute/git-annex-compute-wasmedge]] + Uses [wasmedge](https://WasmEdge.org/) to run WASM programs that are + checked into the git-annex repository, to compute other files in the + repository. + + [[examples|compute/git-annex-compute-wasmedge-examples]] diff --git a/doc/special_remotes/compute/git-annex-compute-wasmedge b/doc/special_remotes/compute/git-annex-compute-wasmedge new file mode 100755 index 0000000000..0c361c6230 --- /dev/null +++ b/doc/special_remotes/compute/git-annex-compute-wasmedge @@ -0,0 +1,47 @@ +#!/bin/sh +# git-annex compute remote program that uses wasmedge to run WASM binaries +# from the git-annex repository. +# +# Copyright 2025 Joey Hess; licenced under the GNU GPL version 3 or higher. +set -e + +if [ -z "$1" ]; then + echo "Usage: file.wasm [inputs] -- [outputs] -- [options]" + echo "Example: concat.wasm foo bar -- baz --" +fi + +stage=1 +wasm="" +while [ -n "$1" ]; do + if [ "$1" = "--" ]; then + stage=$((stage+1)) + shift 1 + else + if [ "$stage" = 1 ]; then + echo "INPUT $1" + read input + if [ -n "$input" ]; then + p="./$1" + mkdir -p "$(dirname "$p")" + ln $(realpath "$input") "$p" + if [ -z "$wasm" ]; then + wasm="$p" + fi + fi + shift 1 + elif [ "$stage" = 2 ]; then + echo "OUTPUT $1" + read output + shift 1 + else + break + fi + fi +done + +if [ -n "$wasm" ]; then + # Use --force-interpreter to avoid wasmedge running AOT native + # instructions, which is insecure if the WASM binary comes from + # an untrusted source. + wasmedge --dir /:$PWD --force-interpreter -- "$wasm" "$@" >&2 +fi diff --git a/doc/special_remotes/compute/git-annex-compute-wasmedge-examples.mdwn b/doc/special_remotes/compute/git-annex-compute-wasmedge-examples.mdwn new file mode 100644 index 0000000000..c483ed5614 --- /dev/null +++ b/doc/special_remotes/compute/git-annex-compute-wasmedge-examples.mdwn @@ -0,0 +1,52 @@ +[[git-annex-compute-wasmedge]] uses [WasmEdge](https://wasmedge.org/) +to run WASM programs, that are checked into the git-annex repository, +to [[compute]] other files in the repository. + +The WASM programs are limited to sandboxed file IO, and cannot access the +network. + +The first parameter is the WASM file to run. It is followed by any other +input files that it should have access to. Then by "--", and a list of all +output files that the program computes. Finally, there can be another "--" +that is followed by any ARGV to pass to the WASM program. + +An example is: + + git-annex initremote wasmedge type=compute program=git-annex-compute-wasmedge + git-annex addcomputed --to=wasmedge -- concat.wasm foo bar -- baz -- baz + +To use that, you will need to write a concat.wasm program that combines +together files foo and bar, and writes the result to a file named baz. + +---- + +Here's another example, using an existing WASM build of python, from +this article . + +Download it and add it to your git-annex repository: + + wget https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/python%2F3.11.1%2B20230127-c8036b4/python-aio-3.11.1.zip + unzip python-aio-3.11.1.zip + rm python-aio-3.11.1.zip bin/python-3.11.1-wasmedge.wasm + + git-annex add bin/python-3.11.1.wasm usr/local/lib/python3.11* + +Notice that the wasm binary needs a few other files that constitute the +python runtime. Those files have to be provided as inputs when using it. + +Here's how to use it to compute a file: + + git-annex initremote wasmedge type=compute program=git-annex-compute-wasmedge + git-annex addcomputed --fast --to wasmedge -- bin/python-3.11.1.wasm \ + usr/local/lib/python311.zip usr/local/lib/python3.11/lib-dynload/.empty \ + usr/local/lib/python3.11/os.py \ + -- foo -- -c 'with open("foo", "w") as f: f.write("hello wasm world\n")' + +Of course, you can replace the python code with something more interesting. +Add additional input files, read them, do whatever. + +While this makes a nice easy example, python built this way is quite slow, and +it would be hard to use other python libraries with it. +It would probably be better to use + +to convert a python program into a WASM binary.