mirrors/zotero
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Generate xz MARs for new updater

Browse source 
The old updater used MARs with bzip2-compressed files, but Mozilla switched to
xz compression, so we generate xz-based MARs using the current MAR tools and
`make_(full|incremental)_update.sh` scripts and add a script to generate
bzip2-based MARs to serve to pre-7.0 builds.
This commit is contained in:
Dan Stillman 2023-05-17 03:47:52 -04:00
parent 6261640a92
commit 33ac72ebac
6 changed files with 212 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

13
app/scripts/fetch_mar_tools
View file

@ -9,13 +9,14 @@ cd "$APP_ROOT_DIR"
mkdir -p "xulrunner/bin" mkdir -p "xulrunner/bin"
if [ "`uname`" = "Darwin" ]; then if [ "`uname`" = "Darwin" ]; then
# Mozilla has Linux executables where the Mac files should be, so supply our own Mac builds # Mozilla has Linux executables where the Mac files should be, so supply our own Mac builds
curl -o "xulrunner/bin/mar" https://zotero-download.s3.us-east-1.amazonaws.com/tools/mac/60.8.0esr/mar curl -o "xulrunner/bin/mar" https://zotero-download.s3.us-east-1.amazonaws.com/tools/mac/102.11.0esr/mar
curl -o "xulrunner/bin/mbsdiff" https://zotero-download.s3.us-east-1.amazonaws.com/tools/mac/60.8.0esr/mbsdiff curl -o "xulrunner/bin/mbsdiff" https://zotero-download.s3.us-east-1.amazonaws.com/tools/mac/102.11.0esr/mbsdiff
elif [ "`uname -o 2> /dev/null`" = "Cygwin" ]; then elif [ "`uname -o 2> /dev/null`" = "Cygwin" ]; then
curl -o "xulrunner/bin/mar.exe" https://ftp.mozilla.org/pub/firefox/nightly/2018/03/2018-03-01-10-01-39-mozilla-central/mar-tools/win64/mar.exe # Mozilla doesn't seem to provide Windows versions via its file server anymore, so supply our own
curl -o "xulrunner/bin/mbsdiff.exe" https://ftp.mozilla.org/pub/firefox/nightly/2018/03/2018-03-01-10-01-39-mozilla-central/mar-tools/win64/mbsdiff.exe curl -o "xulrunner/bin/mar.exe" https://zotero-download.s3.us-east-1.amazonaws.com/tools/win/102.11.0esr/mar.exe
curl -o "xulrunner/bin/mbsdiff.exe" https://zotero-download.s3.us-east-1.amazonaws.com/tools/win/102.11.0esr/mbsdiff.exe
else else
curl -o "xulrunner/bin/mar" https://ftp.mozilla.org/pub/firefox/nightly/2018/03/2018-03-01-10-01-39-mozilla-central/mar-tools/linux64/mar curl -o "xulrunner/bin/mar" https://ftp.mozilla.org/pub/firefox/nightly/2022/05/2022-05-30-09-39-43-mozilla-central/mar-tools/linux64/mar
curl -o "xulrunner/bin/mbsdiff" https://ftp.mozilla.org/pub/firefox/nightly/2018/03/2018-03-01-10-01-39-mozilla-central/mar-tools/linux64/mbsdiff curl -o "xulrunner/bin/mbsdiff" https://ftp.mozilla.org/pub/firefox/nightly/2022/05/2022-05-30-09-39-43-mozilla-central/mar-tools/linux64/mbsdiff
fi fi
chmod 755 xulrunner/bin/mar xulrunner/bin/mbsdiff chmod 755 xulrunner/bin/mar xulrunner/bin/mbsdiff

30
app/update-packaging/build_autoupdate.sh
View file

@ -8,11 +8,11 @@ UPDATE_STAGE_DIR="$SCRIPT_DIR/staging"
function usage { function usage {
cat >&2 <<DONE cat >&2 <<DONE
Usage: $0 -f [-i FROM_VERSION] [-c CHANNEL] [-p PLATFORMS] [-l] VERSION Usage: $0 -c CHANNEL [-f] [-i FROM_VERSION] [-p PLATFORMS] [-l] VERSION
Options Options
-c CHANNEL Release channel ('release', 'beta')
-f Perform full build -f Perform full build
-i FROM Perform incremental build -i FROM_VERSION Perform incremental build
-c CHANNEL Release channel ('release', 'beta') (required for incremental builds)
-p PLATFORMS Platforms to build (m=Mac, w=Windows, l=Linux) -p PLATFORMS Platforms to build (m=Mac, w=Windows, l=Linux)
-l Use local TO directory instead of downloading TO files from S3 -l Use local TO directory instead of downloading TO files from S3
DONE DONE
@ -93,8 +93,8 @@ if [ -z "$FROM" ] && [ $BUILD_FULL -eq 0 ]; then
usage usage
fi fi
if [[ $BUILD_INCREMENTAL -eq 1 ]] && [[ -z "$CHANNEL" ]]; then if [[ -z "$CHANNEL" ]]; then
echo "Channel not provided for incremental builds" >&2 echo "Channel not provided" >&2
exit 1 exit 1
fi fi
@ -121,7 +121,8 @@ for version in "$FROM" "$TO"; do
versiondir="$UPDATE_STAGE_DIR/$version" versiondir="$UPDATE_STAGE_DIR/$version"
# #
# Use main build script's staging directory for TO files rather than downloading the given version. # If -l passed, use main build script's staging directory for TO files rather than downloading
# the given version.
# #
# The caller must ensure that the files in ../staging match the platforms and version given. # The caller must ensure that the files in ../staging match the platforms and version given.
if [[ $version == $TO && $USE_LOCAL_TO == "1" ]]; then if [[ $version == $TO && $USE_LOCAL_TO == "1" ]]; then
@ -259,6 +260,10 @@ for version in "$FROM" "$TO"; do
echo echo
done done
# Set variables for mar command in make_(incremental|full)_update.sh
export MOZ_PRODUCT_VERSION="$TO"
export MAR_CHANNEL_ID="$CHANNEL"
CHANGES_MADE=0 CHANGES_MADE=0
for build in "mac" "win32" "win-x64" "linux-i686" "linux-x86_64"; do for build in "mac" "win32" "win-x64" "linux-i686" "linux-x86_64"; do
if [[ $build == "mac" ]]; then if [[ $build == "mac" ]]; then
@ -293,12 +298,25 @@ for build in "mac" "win32" "win-x64" "linux-i686" "linux-x86_64"; do
"$SCRIPT_DIR/make_incremental_update.sh" "$DIST_DIR/Zotero-${TO}-${FROM}_$build.mar" "$from_dir" "$to_dir" "$SCRIPT_DIR/make_incremental_update.sh" "$DIST_DIR/Zotero-${TO}-${FROM}_$build.mar" "$from_dir" "$to_dir"
CHANGES_MADE=1 CHANGES_MADE=1
# If it's an incremental patch from a 6.0 build, use bzip instead of xz
if [[ $FROM = 6.0* ]]; then
echo "Building bzip2 version of incremental $build update from $FROM to $TO"
"$SCRIPT_DIR/xz_to_bzip" "$DIST_DIR/Zotero-${TO}-${FROM}_$build.mar" "$DIST_DIR/Zotero-${TO}-${FROM}_${build}_bz.mar"
rm "$DIST_DIR/Zotero-${TO}-${FROM}_$build.mar"
mv "$DIST_DIR/Zotero-${TO}-${FROM}_${build}_bz.mar" "$DIST_DIR/Zotero-${TO}-${FROM}_$build.mar"
fi
fi fi
if [[ $BUILD_FULL == 1 ]]; then if [[ $BUILD_FULL == 1 ]]; then
echo echo
echo "Building full $build update for $TO" echo "Building full $build update for $TO"
"$SCRIPT_DIR/make_full_update.sh" "$DIST_DIR/Zotero-${TO}-full_$build.mar" "$UPDATE_STAGE_DIR/$TO/$dir" "$SCRIPT_DIR/make_full_update.sh" "$DIST_DIR/Zotero-${TO}-full_$build.mar" "$UPDATE_STAGE_DIR/$TO/$dir"
CHANGES_MADE=1 CHANGES_MADE=1
# Make a bzip version of all complete patches for serving to <7.0 builds. We can stop this
# once we do a waterfall build that all older versions get updated to.
echo "Building bzip2 version of full $build update for $TO"
"$SCRIPT_DIR/xz_to_bzip" "$DIST_DIR/Zotero-${TO}-full_$build.mar" "$DIST_DIR/Zotero-${TO}-full_bz_${build}.mar"
fi fi
done done

121
app/update-packaging/common.sh
View file

@ -9,10 +9,32 @@
# #
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
QUIET=0
# By default just assume that these tools exist on our path # By default just assume that these tools exist on our path
MAR=${MAR:-mar} MAR=${MAR:-mar}
BZIP2=${BZIP2:-bzip2}
MBSDIFF=${MBSDIFF:-mbsdiff} MBSDIFF=${MBSDIFF:-mbsdiff}
XZ=${XZ:-xz}
$XZ --version > /dev/null 2>&1
if [ $? -ne 0 ]; then
# If $XZ is not set and not found on the path then this is probably
# running on a windows buildbot. Some of the Windows build systems have
# xz.exe in topsrcdir/xz/. Look in the places this would be in both a
# mozilla-central and comm-central build.
XZ="$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe"
$XZ --version > /dev/null 2>&1
if [ $? -ne 0 ]; then
XZ="$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe"
$XZ --version > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo "xz was not found on this system!"
echo "exiting"
exit 1
fi
fi
fi
# Ensure that we're always using the right compression settings
export XZ_OPT="-T1 -7e"
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Helper routines # Helper routines
@ -21,6 +43,12 @@ notice() {
echo "$*" 1>&2 echo "$*" 1>&2
} }
verbose_notice() {
if [ $QUIET -eq 0 ]; then
notice "$*"
fi
}
get_file_size() { get_file_size() {
info=($(ls -ln "$1")) info=($(ls -ln "$1"))
echo ${info[4]} echo ${info[4]}
@ -39,13 +67,7 @@ copy_perm() {
make_add_instruction() { make_add_instruction() {
f="$1" f="$1"
filev2="$2" filev3="$2"
# The third param will be an empty string when a file add instruction is only
# needed in the version 2 manifest. This only happens when the file has an
# add-if-not instruction in the version 3 manifest. This is due to the
# precomplete file prior to the version 3 manifest having a remove instruction
# for this file so the file is removed before applying a complete update.
filev3="$3"
# Used to log to the console # Used to log to the console
if [ $4 ]; then if [ $4 ]; then
@ -60,23 +82,21 @@ make_add_instruction() {
# Use the subdirectory of the extensions folder as the file to test # Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction. # before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " add-if \"$testdir\" \"$f\"" verbose_notice " add-if \"$testdir\" \"$f\""
echo "add-if \"$testdir\" \"$f\"" >> $filev2 echo "add-if \"$testdir\" \"$f\"" >> "$filev3"
if [ ! $filev3 = "" ]; then
echo "add-if \"$testdir\" \"$f\"" >> $filev3
fi
else else
notice " add \"$f\"$forced" verbose_notice " add \"$f\"$forced"
echo "add \"$f\"" >> $filev2 echo "add \"$f\"" >> "$filev3"
if [ ! $filev3 = "" ]; then
echo "add \"$f\"" >> $filev3
fi
fi fi
} }
check_for_add_if_not_update() { check_for_add_if_not_update() {
add_if_not_file_chk="$1" add_if_not_file_chk="$1"
# XXX unconditional add-if for channel-prefs.js to fix mac signature on old installs (bug 1804303)
if [ "$add_if_not_file_chk" = "Contents/Resources/defaults/pref/channel-prefs.js" ]; then
return 1
fi
if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \ if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \
`basename $add_if_not_file_chk` = "update-settings.ini" ]; then `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
## "true" *giggle* ## "true" *giggle*
@ -86,49 +106,35 @@ check_for_add_if_not_update() {
return 1; return 1;
} }
check_for_add_to_manifestv2() {
add_if_not_file_chk="$1"
if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
## "true" *giggle*
return 0;
fi
## 'false'... because this is bash. Oh yay!
return 1;
}
make_add_if_not_instruction() { make_add_if_not_instruction() {
f="$1" f="$1"
filev3="$2" filev3="$2"
notice " add-if-not \"$f\" \"$f\"" verbose_notice " add-if-not \"$f\" \"$f\""
echo "add-if-not \"$f\" \"$f\"" >> $filev3 echo "add-if-not \"$f\" \"$f\"" >> "$filev3"
} }
make_patch_instruction() { make_patch_instruction() {
f="$1" f="$1"
filev2="$2" filev3="$2"
filev3="$3"
# Changed by Zotero for -e
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') || true is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') || true
if [ $is_extension = "1" ]; then if [ $is_extension = "1" ]; then
# Use the subdirectory of the extensions folder as the file to test # Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction. # before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " patch-if \"$testdir\" \"$f.patch\" \"$f\"" verbose_notice " patch-if \"$testdir\" \"$f.patch\" \"$f\""
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2 echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev3"
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3
else else
notice " patch \"$f.patch\" \"$f\"" verbose_notice " patch \"$f.patch\" \"$f\""
echo "patch \"$f.patch\" \"$f\"" >> $filev2 echo "patch \"$f.patch\" \"$f\"" >> "$filev3"
echo "patch \"$f.patch\" \"$f\"" >> $filev3
fi fi
} }
append_remove_instructions() { append_remove_instructions() {
dir="$1" dir="$1"
filev2="$2" filev3="$2"
filev3="$3"
if [ -f "$dir/removed-files" ]; then if [ -f "$dir/removed-files" ]; then
listfile="$dir/removed-files" listfile="$dir/removed-files"
@ -153,19 +159,16 @@ append_remove_instructions() {
# Exclude comments # Exclude comments
if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
if [ $(echo "$f" | grep -c '\/$') = 1 ]; then if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
notice " rmdir \"$f\"" verbose_notice " rmdir \"$f\""
echo "rmdir \"$f\"" >> $filev2 echo "rmdir \"$f\"" >> "$filev3"
echo "rmdir \"$f\"" >> $filev3
elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
# Remove the * # Remove the *
f=$(echo "$f" | sed -e 's:\*$::') f=$(echo "$f" | sed -e 's:\*$::')
notice " rmrfdir \"$f\"" verbose_notice " rmrfdir \"$f\""
echo "rmrfdir \"$f\"" >> $filev2 echo "rmrfdir \"$f\"" >> "$filev3"
echo "rmrfdir \"$f\"" >> $filev3
else else
notice " remove \"$f\"" verbose_notice " remove \"$f\""
echo "remove \"$f\"" >> $filev2 echo "remove \"$f\"" >> "$filev3"
echo "remove \"$f\"" >> $filev3
fi fi
fi fi
fi fi
@ -178,38 +181,36 @@ append_remove_instructions() {
# Pass a variable name and it will be filled as an array. # Pass a variable name and it will be filled as an array.
list_files() { list_files() {
count=0 count=0
temp_filelist=$(mktemp)
find . -type f \ find . -type f \
! -name "update.manifest" \ ! -name "update.manifest" \
! -name "updatev2.manifest" \ ! -name "updatev2.manifest" \
! -name "updatev3.manifest" \ ! -name "updatev3.manifest" \
! -name "temp-dirlist" \
! -name "temp-filelist" \
| sed 's/\.\/\(.*\)/\1/' \ | sed 's/\.\/\(.*\)/\1/' \
| sort -r > "temp-filelist" | sort -r > "${temp_filelist}"
while read file; do while read file; do
eval "${1}[$count]=\"$file\"" eval "${1}[$count]=\"$file\""
# Changed for Zotero to avoid eval as 1 # Changed for Zotero to avoid eval as 1
#(( count++ )) #(( count++ ))
(( ++count )) (( ++count ))
done < "temp-filelist" done < "${temp_filelist}"
rm "temp-filelist" rm "${temp_filelist}"
} }
# List all directories in the current directory, stripping leading "./" # List all directories in the current directory, stripping leading "./"
list_dirs() { list_dirs() {
count=0 count=0
temp_dirlist=$(mktemp)
find . -type d \ find . -type d \
! -name "." \ ! -name "." \
! -name ".." \ ! -name ".." \
| sed 's/\.\/\(.*\)/\1/' \ | sed 's/\.\/\(.*\)/\1/' \
| sort -r > "temp-dirlist" | sort -r > "${temp_dirlist}"
while read dir; do while read dir; do
eval "${1}[$count]=\"$dir\"" eval "${1}[$count]=\"$dir\""
# Changed for Zotero # Changed for Zotero
#(( count++ )) #(( count++ ))
(( ++count )) (( ++count ))
done < "temp-dirlist" done < "${temp_dirlist}"
rm "temp-dirlist" rm "${temp_dirlist}"
} }

36
app/update-packaging/make_full_update.sh
View file

@ -31,12 +31,21 @@ if [ $1 = -h ]; then
notice "" notice ""
notice "Options:" notice "Options:"
notice " -h show this help text" notice " -h show this help text"
notice " -q be less verbose"
notice "" notice ""
exit 1 exit 1
fi fi
if [ $1 = -q ]; then
QUIET=1
export QUIET
shift
fi
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
mar_command="$MAR -V ${MOZ_PRODUCT_VERSION:?} -H ${MAR_CHANNEL_ID:?}"
archive="$1" archive="$1"
targetdir="$2" targetdir="$2"
# Prevent the workdir from being inside the targetdir so it isn't included in # Prevent the workdir from being inside the targetdir so it isn't included in
@ -46,9 +55,8 @@ if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
targetdir=$(echo "$targetdir" | sed -e 's:\/$::') targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
fi fi
workdir="$targetdir.work" workdir="$targetdir.work"
updatemanifestv2="$workdir/updatev2.manifest"
updatemanifestv3="$workdir/updatev3.manifest" updatemanifestv3="$workdir/updatev3.manifest"
targetfiles="updatev2.manifest updatev3.manifest" targetfiles="updatev3.manifest"
mkdir -p "$workdir" mkdir -p "$workdir"
@ -70,13 +78,11 @@ list_files files
popd popd
# Add the type of update to the beginning of the update manifests. # Add the type of update to the beginning of the update manifests.
> $updatemanifestv2 > "$updatemanifestv3"
> $updatemanifestv3
notice "" notice ""
notice "Adding type instruction to update manifests" notice "Adding type instruction to update manifests"
notice " type complete" notice " type complete"
echo "type \"complete\"" >> $updatemanifestv2 echo "type \"complete\"" >> "$updatemanifestv3"
echo "type \"complete\"" >> $updatemanifestv3
notice "" notice ""
notice "Adding file add instructions to update manifests" notice "Adding file add instructions to update manifests"
@ -87,16 +93,13 @@ for ((i=0; $i<$num_files; i=$i+1)); do
if check_for_add_if_not_update "$f"; then if check_for_add_if_not_update "$f"; then
make_add_if_not_instruction "$f" "$updatemanifestv3" make_add_if_not_instruction "$f" "$updatemanifestv3"
if check_for_add_to_manifestv2 "$f"; then
make_add_instruction "$f" "$updatemanifestv2" "" 1
fi
else else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" make_add_instruction "$f" "$updatemanifestv3"
fi fi
dir=$(dirname "$f") dir=$(dirname "$f")
mkdir -p "$workdir/$dir" mkdir -p "$workdir/$dir"
$BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
copy_perm "$targetdir/$f" "$workdir/$f" copy_perm "$targetdir/$f" "$workdir/$f"
targetfiles="$targetfiles \"$f\"" targetfiles="$targetfiles \"$f\""
@ -105,15 +108,12 @@ done
# Append remove instructions for any dead files. # Append remove instructions for any dead files.
notice "" notice ""
notice "Adding file and directory remove instructions from file 'removed-files'" notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3" append_remove_instructions "$targetdir" "$updatemanifestv3"
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
# Changed for Zotero -- -C is unreliable mar_command="$mar_command -C \"$workdir\" -c output.mar"
pushd $workdir > /dev/null eval "$mar_command $targetfiles"
eval "$MAR -c output.mar $targetfiles"
popd > /dev/null
mv -f "$workdir/output.mar" "$archive" mv -f "$workdir/output.mar" "$archive"
# cleanup # cleanup

84
app/update-packaging/make_incremental_update.sh
View file

@ -8,6 +8,7 @@
# Author: Darin Fisher # Author: Darin Fisher
# #
# Added for Zotero
set -eo pipefail set -eo pipefail
. $(dirname "$0")/common.sh . $(dirname "$0")/common.sh
@ -23,6 +24,7 @@ print_usage() {
notice " -h show this help text" notice " -h show this help text"
notice " -f clobber this file in the installation" notice " -f clobber this file in the installation"
notice " Must be a path to a file to clobber in the partial update." notice " Must be a path to a file to clobber in the partial update."
notice " -q be less verbose"
notice "" notice ""
} }
@ -52,6 +54,12 @@ check_for_forced_update() {
return 0; return 0;
fi fi
# notarization ticket
if [ "$forced_file_chk" = "Contents/CodeResources" ]; then
## "true" *giggle*
return 0;
fi
if [ "${forced_file_chk##*.}" = "chk" ]; then if [ "${forced_file_chk##*.}" = "chk" ]; then
## "true" *giggle* ## "true" *giggle*
return 0; return 0;
@ -73,13 +81,16 @@ if [ $# = 0 ]; then
exit 1 exit 1
fi fi
# channel-prefs.js removed for Zotero
requested_forced_updates='Contents/MacOS/firefox' requested_forced_updates='Contents/MacOS/firefox'
while getopts "hf:" flag while getopts "hqf:" flag
do do
case "$flag" in case "$flag" in
h) print_usage; exit 0 h) print_usage; exit 0
;; ;;
q) QUIET=1
;;
f) requested_forced_updates="$requested_forced_updates $OPTARG" f) requested_forced_updates="$requested_forced_updates $OPTARG"
;; ;;
?) print_usage; exit 1 ?) print_usage; exit 1
@ -89,6 +100,9 @@ done
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
mar_command="$MAR -V ${MOZ_PRODUCT_VERSION:?} -H ${MAR_CHANNEL_ID:?}"
# Added for -e for Zotero
set +e set +e
let arg_start=$OPTIND-1 let arg_start=$OPTIND-1
shift $arg_start shift $arg_start
@ -103,10 +117,9 @@ if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
# Remove the / # Remove the /
newdir=$(echo "$newdir" | sed -e 's:\/$::') newdir=$(echo "$newdir" | sed -e 's:\/$::')
fi fi
workdir="$newdir.work" workdir="$(mktemp -d)"
updatemanifestv2="$workdir/updatev2.manifest"
updatemanifestv3="$workdir/updatev3.manifest" updatemanifestv3="$workdir/updatev3.manifest"
archivefiles="updatev2.manifest updatev3.manifest" archivefiles="updatev3.manifest"
mkdir -p "$workdir" mkdir -p "$workdir"
@ -141,10 +154,8 @@ popd
# Add the type of update to the beginning of the update manifests. # Add the type of update to the beginning of the update manifests.
notice "" notice ""
notice "Adding type instruction to update manifests" notice "Adding type instruction to update manifests"
> $updatemanifestv2
> $updatemanifestv3 > $updatemanifestv3
notice " type partial" notice " type partial"
echo "type \"partial\"" >> $updatemanifestv2
echo "type \"partial\"" >> $updatemanifestv3 echo "type \"partial\"" >> $updatemanifestv3
notice "" notice ""
@ -163,7 +174,7 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
if check_for_add_if_not_update "$f"; then if check_for_add_if_not_update "$f"; then
# The full workdir may not exist yet, so create it if necessary. # The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"` mkdir -p `dirname "$workdir/$f"`
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f" copy_perm "$newdir/$f" "$workdir/$f"
make_add_if_not_instruction "$f" "$updatemanifestv3" make_add_if_not_instruction "$f" "$updatemanifestv3"
archivefiles="$archivefiles \"$f\"" archivefiles="$archivefiles \"$f\""
@ -173,9 +184,9 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
if check_for_forced_update "$requested_forced_updates" "$f"; then if check_for_forced_update "$requested_forced_updates" "$f"; then
# The full workdir may not exist yet, so create it if necessary. # The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"` mkdir -p `dirname "$workdir/$f"`
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f" copy_perm "$newdir/$f" "$workdir/$f"
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1 make_add_instruction "$f" "$updatemanifestv3" 1
archivefiles="$archivefiles \"$f\"" archivefiles="$archivefiles \"$f\""
continue 1 continue 1
fi fi
@ -185,7 +196,7 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
# compare the sizes. Then choose the smaller of the two to package. # compare the sizes. Then choose the smaller of the two to package.
dir=$(dirname "$workdir/$f") dir=$(dirname "$workdir/$f")
mkdir -p "$dir" mkdir -p "$dir"
notice "diffing \"$f\"" verbose_notice "diffing \"$f\""
# MBSDIFF_HOOK represents the communication interface with funsize and, # MBSDIFF_HOOK represents the communication interface with funsize and,
# if enabled, caches the intermediate patches for future use and # if enabled, caches the intermediate patches for future use and
# compute avoidance # compute avoidance
@ -196,36 +207,38 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
# myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \ # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
# PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL # PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
# #
# Note: patches are bzipped stashed in funsize to gain more speed # Note: patches are bzipped or xz stashed in funsize to gain more speed
# if service is not enabled then default to old behavior # if service is not enabled then default to old behavior
if [ -z "$MBSDIFF_HOOK" ]; then # Disabled for Zotero
#if [ -z "$MBSDIFF_HOOK" ]; then
if true; then
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$BZIP2 -z9 "$workdir/$f.patch" $XZ $XZ_OPT --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
else else
# if service enabled then check patch existence for retrieval # if service enabled then check patch existence for retrieval
if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then
notice "file \"$f\" found in funsize, diffing skipped" verbose_notice "file \"$f\" found in funsize, diffing skipped"
else else
# if not found already - compute it and cache it for future use # if not found already - compute it and cache it for future use
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$BZIP2 -z9 "$workdir/$f.patch" $XZ $XZ_OPT --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
$MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2" $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"
fi fi
fi fi
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f" copy_perm "$newdir/$f" "$workdir/$f"
patchfile="$workdir/$f.patch.bz2" patchfile="$workdir/$f.patch.xz"
patchsize=$(get_file_size "$patchfile") patchsize=$(get_file_size "$patchfile")
fullsize=$(get_file_size "$workdir/$f") fullsize=$(get_file_size "$workdir/$f")
if [ $patchsize -lt $fullsize ]; then if [ $patchsize -lt $fullsize ]; then
make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" make_patch_instruction "$f" "$updatemanifestv3"
mv -f "$patchfile" "$workdir/$f.patch" mv -f "$patchfile" "$workdir/$f.patch"
rm -f "$workdir/$f" rm -f "$workdir/$f"
archivefiles="$archivefiles \"$f.patch\"" archivefiles="$archivefiles \"$f.patch\""
else else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" make_add_instruction "$f" "$updatemanifestv3"
rm -f "$patchfile" rm -f "$patchfile"
archivefiles="$archivefiles \"$f\"" archivefiles="$archivefiles \"$f\""
fi fi
@ -258,13 +271,13 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); do
dir=$(dirname "$workdir/$f") dir=$(dirname "$workdir/$f")
mkdir -p "$dir" mkdir -p "$dir"
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f" copy_perm "$newdir/$f" "$workdir/$f"
if check_for_add_if_not_update "$f"; then if check_for_add_if_not_update "$f"; then
make_add_if_not_instruction "$f" "$updatemanifestv3" make_add_if_not_instruction "$f" "$updatemanifestv3"
else else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" make_add_instruction "$f" "$updatemanifestv3"
fi fi
@ -275,15 +288,14 @@ notice ""
notice "Adding file remove instructions to update manifests" notice "Adding file remove instructions to update manifests"
for ((i=0; $i<$num_removes; i=$i+1)); do for ((i=0; $i<$num_removes; i=$i+1)); do
f="${remove_array[$i]}" f="${remove_array[$i]}"
notice " remove \"$f\"" verbose_notice " remove \"$f\""
echo "remove \"$f\"" >> $updatemanifestv2
echo "remove \"$f\"" >> $updatemanifestv3 echo "remove \"$f\"" >> $updatemanifestv3
done done
# Add remove instructions for any dead files. # Add remove instructions for any dead files.
notice "" notice ""
notice "Adding file and directory remove instructions from file 'removed-files'" notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3" append_remove_instructions "$newdir" "$updatemanifestv3"
notice "" notice ""
notice "Adding directory remove instructions for directories that no longer exist" notice "Adding directory remove instructions for directories that no longer exist"
@ -293,29 +305,15 @@ for ((i=0; $i<$num_olddirs; i=$i+1)); do
f="${olddirs[$i]}" f="${olddirs[$i]}"
# If this dir doesn't exist in the new directory remove it. # If this dir doesn't exist in the new directory remove it.
if [ ! -d "$newdir/$f" ]; then if [ ! -d "$newdir/$f" ]; then
notice " rmdir $f/" verbose_notice " rmdir $f/"
echo "rmdir \"$f/\"" >> $updatemanifestv2
echo "rmdir \"$f/\"" >> $updatemanifestv3 echo "rmdir \"$f/\"" >> $updatemanifestv3
fi fi
done done
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
mar_command="$MAR" mar_command="$mar_command -C \"$workdir\" -c output.mar"
if [[ -n $MOZ_PRODUCT_VERSION ]]
then
mar_command="$mar_command -V $MOZ_PRODUCT_VERSION"
fi
if [[ -n $MOZ_CHANNEL_ID ]]
then
mar_command="$mar_command -H $MOZ_CHANNEL_ID"
fi
# Changed for Zotero -- -C is unreliable
pushd $workdir > /dev/null
mar_command="$mar_command -c output.mar"
eval "$mar_command $archivefiles" eval "$mar_command $archivefiles"
popd > /dev/null
mv -f "$workdir/output.mar" "$archive" mv -f "$workdir/output.mar" "$archive"
# cleanup # cleanup

61
app/update-packaging/xz_to_bzip Executable file
View file

@ -0,0 +1,61 @@
#!/bin/bash
set -euo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
APP_ROOT_DIR="$(dirname "$SCRIPT_DIR")"
. "$APP_ROOT_DIR/config.sh"
infile=${1:-}
outfile=${2:-}
if [[ -z "$infile" ]] || [[ -z "$infile" ]]; then
echo "Usage: $0 infile.mar outfile.mar"
exit 1
fi
mar="$APP_ROOT_DIR/xulrunner/bin/mar"
if [ ! -f "$mar" ]; then
echo "$mar not found"
exit 1
fi
tmp_dir=`mktemp -d`
function cleanup {
rm -rf $tmp_dir
}
trap cleanup EXIT
cd "$tmp_dir"
mkdir in
cd in
echo "Extracting MAR"
$mar -x "$infile"
echo "Renaming all files to .xz"
find . -type f -exec mv {} {}.xz \;
echo "Uncompressing files"
find . -type f -exec unxz {} \;
echo "Recompressing files with bzip2"
find . -type f -exec bzip2 {} \;
echo "Removing .bz extension"
find . -type f | while read f; do mv "$f" "${f%.bz2}"; done
channel=$($mar -T "$infile" | sed -n -r 's/.+MAR channel name: ([^\s]+)/\1/p')
version=$($mar -T "$infile" | sed -n -r 's/.+Product version: ([^\s]+)/\1/p')
if [ -z "$channel" ]; then
echo "Could not detect channel"
exit 1
fi
if [ -z "$version" ]; then
echo "Could not detect version"
exit 1
fi
echo "MAR channel name: $channel"
echo "Product version: $version"
echo "Creating new MAR"
$mar -V $version -H $channel -c "$outfile" `find . -type f | sed 's/^\.\///'`
echo
$mar -T "$outfile"