Generate xz MARs for new updater
The old updater used MARs with bzip2-compressed files, but Mozilla switched to xz compression, so we generate xz-based MARs using the current MAR tools and `make_(full|incremental)_update.sh` scripts and add a script to generate bzip2-based MARs to serve to pre-7.0 builds.
This commit is contained in:
Dan Stillman
parent
6261640a92
commit
33ac72ebac
6 changed files with 212 additions and 133 deletions
|
|
@ -9,13 +9,14 @@ cd "$APP_ROOT_DIR"
|
|||
mkdir -p "xulrunner/bin"
|
||||
if [ "`uname`" = "Darwin" ]; then
|
||||
# Mozilla has Linux executables where the Mac files should be, so supply our own Mac builds
|
||||
curl -o "xulrunner/bin/mar" https://zotero-download.s3.us-east-1.amazonaws.com/tools/mac/60.8.0esr/mar
|
||||
curl -o "xulrunner/bin/mbsdiff" https://zotero-download.s3.us-east-1.amazonaws.com/tools/mac/60.8.0esr/mbsdiff
|
||||
curl -o "xulrunner/bin/mar" https://zotero-download.s3.us-east-1.amazonaws.com/tools/mac/102.11.0esr/mar
|
||||
curl -o "xulrunner/bin/mbsdiff" https://zotero-download.s3.us-east-1.amazonaws.com/tools/mac/102.11.0esr/mbsdiff
|
||||
elif [ "`uname -o 2> /dev/null`" = "Cygwin" ]; then
|
||||
curl -o "xulrunner/bin/mar.exe" https://ftp.mozilla.org/pub/firefox/nightly/2018/03/2018-03-01-10-01-39-mozilla-central/mar-tools/win64/mar.exe
|
||||
curl -o "xulrunner/bin/mbsdiff.exe" https://ftp.mozilla.org/pub/firefox/nightly/2018/03/2018-03-01-10-01-39-mozilla-central/mar-tools/win64/mbsdiff.exe
|
||||
# Mozilla doesn't seem to provide Windows versions via its file server anymore, so supply our own
|
||||
curl -o "xulrunner/bin/mar.exe" https://zotero-download.s3.us-east-1.amazonaws.com/tools/win/102.11.0esr/mar.exe
|
||||
curl -o "xulrunner/bin/mbsdiff.exe" https://zotero-download.s3.us-east-1.amazonaws.com/tools/win/102.11.0esr/mbsdiff.exe
|
||||
else
|
||||
curl -o "xulrunner/bin/mar" https://ftp.mozilla.org/pub/firefox/nightly/2018/03/2018-03-01-10-01-39-mozilla-central/mar-tools/linux64/mar
|
||||
curl -o "xulrunner/bin/mbsdiff" https://ftp.mozilla.org/pub/firefox/nightly/2018/03/2018-03-01-10-01-39-mozilla-central/mar-tools/linux64/mbsdiff
|
||||
curl -o "xulrunner/bin/mar" https://ftp.mozilla.org/pub/firefox/nightly/2022/05/2022-05-30-09-39-43-mozilla-central/mar-tools/linux64/mar
|
||||
curl -o "xulrunner/bin/mbsdiff" https://ftp.mozilla.org/pub/firefox/nightly/2022/05/2022-05-30-09-39-43-mozilla-central/mar-tools/linux64/mbsdiff
|
||||
fi
|
||||
chmod 755 xulrunner/bin/mar xulrunner/bin/mbsdiff
|
||||
|
|
|
|||
|
|
@ -8,11 +8,11 @@ UPDATE_STAGE_DIR="$SCRIPT_DIR/staging"
|
|||
|
||||
function usage {
|
||||
cat >&2 <<DONE
|
||||
Usage: $0 -f [-i FROM_VERSION] [-c CHANNEL] [-p PLATFORMS] [-l] VERSION
|
||||
Usage: $0 -c CHANNEL [-f] [-i FROM_VERSION] [-p PLATFORMS] [-l] VERSION
|
||||
Options
|
||||
-c CHANNEL Release channel ('release', 'beta')
|
||||
-f Perform full build
|
||||
-i FROM Perform incremental build
|
||||
-c CHANNEL Release channel ('release', 'beta') (required for incremental builds)
|
||||
-i FROM_VERSION Perform incremental build
|
||||
-p PLATFORMS Platforms to build (m=Mac, w=Windows, l=Linux)
|
||||
-l Use local TO directory instead of downloading TO files from S3
|
||||
DONE
|
||||
|
|
@ -93,8 +93,8 @@ if [ -z "$FROM" ] && [ $BUILD_FULL -eq 0 ]; then
|
|||
usage
|
||||
fi
|
||||
|
||||
if [[ $BUILD_INCREMENTAL -eq 1 ]] && [[ -z "$CHANNEL" ]]; then
|
||||
echo "Channel not provided for incremental builds" >&2
|
||||
if [[ -z "$CHANNEL" ]]; then
|
||||
echo "Channel not provided" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
@ -121,7 +121,8 @@ for version in "$FROM" "$TO"; do
|
|||
versiondir="$UPDATE_STAGE_DIR/$version"
|
||||
|
||||
#
|
||||
# Use main build script's staging directory for TO files rather than downloading the given version.
|
||||
# If -l passed, use main build script's staging directory for TO files rather than downloading
|
||||
# the given version.
|
||||
#
|
||||
# The caller must ensure that the files in ../staging match the platforms and version given.
|
||||
if [[ $version == $TO && $USE_LOCAL_TO == "1" ]]; then
|
||||
|
|
@ -259,6 +260,10 @@ for version in "$FROM" "$TO"; do
|
|||
echo
|
||||
done
|
||||
|
||||
# Set variables for mar command in make_(incremental|full)_update.sh
|
||||
export MOZ_PRODUCT_VERSION="$TO"
|
||||
export MAR_CHANNEL_ID="$CHANNEL"
|
||||
|
||||
CHANGES_MADE=0
|
||||
for build in "mac" "win32" "win-x64" "linux-i686" "linux-x86_64"; do
|
||||
if [[ $build == "mac" ]]; then
|
||||
|
|
@ -293,12 +298,25 @@ for build in "mac" "win32" "win-x64" "linux-i686" "linux-x86_64"; do
|
|||
|
||||
"$SCRIPT_DIR/make_incremental_update.sh" "$DIST_DIR/Zotero-${TO}-${FROM}_$build.mar" "$from_dir" "$to_dir"
|
||||
CHANGES_MADE=1
|
||||
|
||||
# If it's an incremental patch from a 6.0 build, use bzip instead of xz
|
||||
if [[ $FROM = 6.0* ]]; then
|
||||
echo "Building bzip2 version of incremental $build update from $FROM to $TO"
|
||||
"$SCRIPT_DIR/xz_to_bzip" "$DIST_DIR/Zotero-${TO}-${FROM}_$build.mar" "$DIST_DIR/Zotero-${TO}-${FROM}_${build}_bz.mar"
|
||||
rm "$DIST_DIR/Zotero-${TO}-${FROM}_$build.mar"
|
||||
mv "$DIST_DIR/Zotero-${TO}-${FROM}_${build}_bz.mar" "$DIST_DIR/Zotero-${TO}-${FROM}_$build.mar"
|
||||
fi
|
||||
fi
|
||||
if [[ $BUILD_FULL == 1 ]]; then
|
||||
echo
|
||||
echo "Building full $build update for $TO"
|
||||
"$SCRIPT_DIR/make_full_update.sh" "$DIST_DIR/Zotero-${TO}-full_$build.mar" "$UPDATE_STAGE_DIR/$TO/$dir"
|
||||
CHANGES_MADE=1
|
||||
|
||||
# Make a bzip version of all complete patches for serving to <7.0 builds. We can stop this
|
||||
# once we do a waterfall build that all older versions get updated to.
|
||||
echo "Building bzip2 version of full $build update for $TO"
|
||||
"$SCRIPT_DIR/xz_to_bzip" "$DIST_DIR/Zotero-${TO}-full_$build.mar" "$DIST_DIR/Zotero-${TO}-full_bz_${build}.mar"
|
||||
fi
|
||||
done
|
||||
|
||||
|
|
|
|||
|
|
@ -9,10 +9,32 @@
|
|||
#
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
QUIET=0
|
||||
|
||||
# By default just assume that these tools exist on our path
|
||||
MAR=${MAR:-mar}
|
||||
BZIP2=${BZIP2:-bzip2}
|
||||
MBSDIFF=${MBSDIFF:-mbsdiff}
|
||||
XZ=${XZ:-xz}
|
||||
$XZ --version > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]; then
|
||||
# If $XZ is not set and not found on the path then this is probably
|
||||
# running on a windows buildbot. Some of the Windows build systems have
|
||||
# xz.exe in topsrcdir/xz/. Look in the places this would be in both a
|
||||
# mozilla-central and comm-central build.
|
||||
XZ="$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe"
|
||||
$XZ --version > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]; then
|
||||
XZ="$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe"
|
||||
$XZ --version > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "xz was not found on this system!"
|
||||
echo "exiting"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
# Ensure that we're always using the right compression settings
|
||||
export XZ_OPT="-T1 -7e"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Helper routines
|
||||
|
|
@ -21,6 +43,12 @@ notice() {
|
|||
echo "$*" 1>&2
|
||||
}
|
||||
|
||||
verbose_notice() {
|
||||
if [ $QUIET -eq 0 ]; then
|
||||
notice "$*"
|
||||
fi
|
||||
}
|
||||
|
||||
get_file_size() {
|
||||
info=($(ls -ln "$1"))
|
||||
echo ${info[4]}
|
||||
|
|
@ -39,13 +67,7 @@ copy_perm() {
|
|||
|
||||
make_add_instruction() {
|
||||
f="$1"
|
||||
filev2="$2"
|
||||
# The third param will be an empty string when a file add instruction is only
|
||||
# needed in the version 2 manifest. This only happens when the file has an
|
||||
# add-if-not instruction in the version 3 manifest. This is due to the
|
||||
# precomplete file prior to the version 3 manifest having a remove instruction
|
||||
# for this file so the file is removed before applying a complete update.
|
||||
filev3="$3"
|
||||
filev3="$2"
|
||||
|
||||
# Used to log to the console
|
||||
if [ $4 ]; then
|
||||
|
|
@ -60,23 +82,21 @@ make_add_instruction() {
|
|||
# Use the subdirectory of the extensions folder as the file to test
|
||||
# before performing this add instruction.
|
||||
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
|
||||
notice " add-if \"$testdir\" \"$f\""
|
||||
echo "add-if \"$testdir\" \"$f\"" >> $filev2
|
||||
if [ ! $filev3 = "" ]; then
|
||||
echo "add-if \"$testdir\" \"$f\"" >> $filev3
|
||||
fi
|
||||
verbose_notice " add-if \"$testdir\" \"$f\""
|
||||
echo "add-if \"$testdir\" \"$f\"" >> "$filev3"
|
||||
else
|
||||
notice " add \"$f\"$forced"
|
||||
echo "add \"$f\"" >> $filev2
|
||||
if [ ! $filev3 = "" ]; then
|
||||
echo "add \"$f\"" >> $filev3
|
||||
fi
|
||||
verbose_notice " add \"$f\"$forced"
|
||||
echo "add \"$f\"" >> "$filev3"
|
||||
fi
|
||||
}
|
||||
|
||||
check_for_add_if_not_update() {
|
||||
add_if_not_file_chk="$1"
|
||||
|
||||
# XXX unconditional add-if for channel-prefs.js to fix mac signature on old installs (bug 1804303)
|
||||
if [ "$add_if_not_file_chk" = "Contents/Resources/defaults/pref/channel-prefs.js" ]; then
|
||||
return 1
|
||||
fi
|
||||
if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \
|
||||
`basename $add_if_not_file_chk` = "update-settings.ini" ]; then
|
||||
## "true" *giggle*
|
||||
|
|
@ -86,49 +106,35 @@ check_for_add_if_not_update() {
|
|||
return 1;
|
||||
}
|
||||
|
||||
check_for_add_to_manifestv2() {
|
||||
add_if_not_file_chk="$1"
|
||||
|
||||
if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
|
||||
## "true" *giggle*
|
||||
return 0;
|
||||
fi
|
||||
## 'false'... because this is bash. Oh yay!
|
||||
return 1;
|
||||
}
|
||||
|
||||
make_add_if_not_instruction() {
|
||||
f="$1"
|
||||
filev3="$2"
|
||||
|
||||
notice " add-if-not \"$f\" \"$f\""
|
||||
echo "add-if-not \"$f\" \"$f\"" >> $filev3
|
||||
verbose_notice " add-if-not \"$f\" \"$f\""
|
||||
echo "add-if-not \"$f\" \"$f\"" >> "$filev3"
|
||||
}
|
||||
|
||||
make_patch_instruction() {
|
||||
f="$1"
|
||||
filev2="$2"
|
||||
filev3="$3"
|
||||
filev3="$2"
|
||||
|
||||
# Changed by Zotero for -e
|
||||
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') || true
|
||||
if [ $is_extension = "1" ]; then
|
||||
# Use the subdirectory of the extensions folder as the file to test
|
||||
# before performing this add instruction.
|
||||
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
|
||||
notice " patch-if \"$testdir\" \"$f.patch\" \"$f\""
|
||||
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2
|
||||
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3
|
||||
verbose_notice " patch-if \"$testdir\" \"$f.patch\" \"$f\""
|
||||
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev3"
|
||||
else
|
||||
notice " patch \"$f.patch\" \"$f\""
|
||||
echo "patch \"$f.patch\" \"$f\"" >> $filev2
|
||||
echo "patch \"$f.patch\" \"$f\"" >> $filev3
|
||||
verbose_notice " patch \"$f.patch\" \"$f\""
|
||||
echo "patch \"$f.patch\" \"$f\"" >> "$filev3"
|
||||
fi
|
||||
}
|
||||
|
||||
append_remove_instructions() {
|
||||
dir="$1"
|
||||
filev2="$2"
|
||||
filev3="$3"
|
||||
filev3="$2"
|
||||
|
||||
if [ -f "$dir/removed-files" ]; then
|
||||
listfile="$dir/removed-files"
|
||||
|
|
@ -153,19 +159,16 @@ append_remove_instructions() {
|
|||
# Exclude comments
|
||||
if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
|
||||
if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
|
||||
notice " rmdir \"$f\""
|
||||
echo "rmdir \"$f\"" >> $filev2
|
||||
echo "rmdir \"$f\"" >> $filev3
|
||||
verbose_notice " rmdir \"$f\""
|
||||
echo "rmdir \"$f\"" >> "$filev3"
|
||||
elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
|
||||
# Remove the *
|
||||
f=$(echo "$f" | sed -e 's:\*$::')
|
||||
notice " rmrfdir \"$f\""
|
||||
echo "rmrfdir \"$f\"" >> $filev2
|
||||
echo "rmrfdir \"$f\"" >> $filev3
|
||||
verbose_notice " rmrfdir \"$f\""
|
||||
echo "rmrfdir \"$f\"" >> "$filev3"
|
||||
else
|
||||
notice " remove \"$f\""
|
||||
echo "remove \"$f\"" >> $filev2
|
||||
echo "remove \"$f\"" >> $filev3
|
||||
verbose_notice " remove \"$f\""
|
||||
echo "remove \"$f\"" >> "$filev3"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
|
@ -178,38 +181,36 @@ append_remove_instructions() {
|
|||
# Pass a variable name and it will be filled as an array.
|
||||
list_files() {
|
||||
count=0
|
||||
|
||||
temp_filelist=$(mktemp)
|
||||
find . -type f \
|
||||
! -name "update.manifest" \
|
||||
! -name "updatev2.manifest" \
|
||||
! -name "updatev3.manifest" \
|
||||
! -name "temp-dirlist" \
|
||||
! -name "temp-filelist" \
|
||||
| sed 's/\.\/\(.*\)/\1/' \
|
||||
| sort -r > "temp-filelist"
|
||||
| sort -r > "${temp_filelist}"
|
||||
while read file; do
|
||||
eval "${1}[$count]=\"$file\""
|
||||
# Changed for Zotero to avoid eval as 1
|
||||
#(( count++ ))
|
||||
(( ++count ))
|
||||
done < "temp-filelist"
|
||||
rm "temp-filelist"
|
||||
done < "${temp_filelist}"
|
||||
rm "${temp_filelist}"
|
||||
}
|
||||
|
||||
# List all directories in the current directory, stripping leading "./"
|
||||
list_dirs() {
|
||||
count=0
|
||||
|
||||
temp_dirlist=$(mktemp)
|
||||
find . -type d \
|
||||
! -name "." \
|
||||
! -name ".." \
|
||||
| sed 's/\.\/\(.*\)/\1/' \
|
||||
| sort -r > "temp-dirlist"
|
||||
| sort -r > "${temp_dirlist}"
|
||||
while read dir; do
|
||||
eval "${1}[$count]=\"$dir\""
|
||||
# Changed for Zotero
|
||||
#(( count++ ))
|
||||
(( ++count ))
|
||||
done < "temp-dirlist"
|
||||
rm "temp-dirlist"
|
||||
done < "${temp_dirlist}"
|
||||
rm "${temp_dirlist}"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,12 +31,21 @@ if [ $1 = -h ]; then
|
|||
notice ""
|
||||
notice "Options:"
|
||||
notice " -h show this help text"
|
||||
notice " -q be less verbose"
|
||||
notice ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $1 = -q ]; then
|
||||
QUIET=1
|
||||
export QUIET
|
||||
shift
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
mar_command="$MAR -V ${MOZ_PRODUCT_VERSION:?} -H ${MAR_CHANNEL_ID:?}"
|
||||
|
||||
archive="$1"
|
||||
targetdir="$2"
|
||||
# Prevent the workdir from being inside the targetdir so it isn't included in
|
||||
|
|
@ -46,9 +55,8 @@ if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
|
|||
targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
|
||||
fi
|
||||
workdir="$targetdir.work"
|
||||
updatemanifestv2="$workdir/updatev2.manifest"
|
||||
updatemanifestv3="$workdir/updatev3.manifest"
|
||||
targetfiles="updatev2.manifest updatev3.manifest"
|
||||
targetfiles="updatev3.manifest"
|
||||
|
||||
mkdir -p "$workdir"
|
||||
|
||||
|
|
@ -70,13 +78,11 @@ list_files files
|
|||
popd
|
||||
|
||||
# Add the type of update to the beginning of the update manifests.
|
||||
> $updatemanifestv2
|
||||
> $updatemanifestv3
|
||||
> "$updatemanifestv3"
|
||||
notice ""
|
||||
notice "Adding type instruction to update manifests"
|
||||
notice " type complete"
|
||||
echo "type \"complete\"" >> $updatemanifestv2
|
||||
echo "type \"complete\"" >> $updatemanifestv3
|
||||
echo "type \"complete\"" >> "$updatemanifestv3"
|
||||
|
||||
notice ""
|
||||
notice "Adding file add instructions to update manifests"
|
||||
|
|
@ -87,16 +93,13 @@ for ((i=0; $i<$num_files; i=$i+1)); do
|
|||
|
||||
if check_for_add_if_not_update "$f"; then
|
||||
make_add_if_not_instruction "$f" "$updatemanifestv3"
|
||||
if check_for_add_to_manifestv2 "$f"; then
|
||||
make_add_instruction "$f" "$updatemanifestv2" "" 1
|
||||
fi
|
||||
else
|
||||
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
|
||||
make_add_instruction "$f" "$updatemanifestv3"
|
||||
fi
|
||||
|
||||
dir=$(dirname "$f")
|
||||
mkdir -p "$workdir/$dir"
|
||||
$BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
|
||||
$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
|
||||
copy_perm "$targetdir/$f" "$workdir/$f"
|
||||
|
||||
targetfiles="$targetfiles \"$f\""
|
||||
|
|
@ -105,15 +108,12 @@ done
|
|||
# Append remove instructions for any dead files.
|
||||
notice ""
|
||||
notice "Adding file and directory remove instructions from file 'removed-files'"
|
||||
append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
|
||||
append_remove_instructions "$targetdir" "$updatemanifestv3"
|
||||
|
||||
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
|
||||
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
|
||||
$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
|
||||
|
||||
# Changed for Zotero -- -C is unreliable
|
||||
pushd $workdir > /dev/null
|
||||
eval "$MAR -c output.mar $targetfiles"
|
||||
popd > /dev/null
|
||||
mar_command="$mar_command -C \"$workdir\" -c output.mar"
|
||||
eval "$mar_command $targetfiles"
|
||||
mv -f "$workdir/output.mar" "$archive"
|
||||
|
||||
# cleanup
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
# Author: Darin Fisher
|
||||
#
|
||||
|
||||
# Added for Zotero
|
||||
set -eo pipefail
|
||||
|
||||
. $(dirname "$0")/common.sh
|
||||
|
|
@ -23,6 +24,7 @@ print_usage() {
|
|||
notice " -h show this help text"
|
||||
notice " -f clobber this file in the installation"
|
||||
notice " Must be a path to a file to clobber in the partial update."
|
||||
notice " -q be less verbose"
|
||||
notice ""
|
||||
}
|
||||
|
||||
|
|
@ -52,6 +54,12 @@ check_for_forced_update() {
|
|||
return 0;
|
||||
fi
|
||||
|
||||
# notarization ticket
|
||||
if [ "$forced_file_chk" = "Contents/CodeResources" ]; then
|
||||
## "true" *giggle*
|
||||
return 0;
|
||||
fi
|
||||
|
||||
if [ "${forced_file_chk##*.}" = "chk" ]; then
|
||||
## "true" *giggle*
|
||||
return 0;
|
||||
|
|
@ -73,13 +81,16 @@ if [ $# = 0 ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# channel-prefs.js removed for Zotero
|
||||
requested_forced_updates='Contents/MacOS/firefox'
|
||||
|
||||
while getopts "hf:" flag
|
||||
while getopts "hqf:" flag
|
||||
do
|
||||
case "$flag" in
|
||||
h) print_usage; exit 0
|
||||
;;
|
||||
q) QUIET=1
|
||||
;;
|
||||
f) requested_forced_updates="$requested_forced_updates $OPTARG"
|
||||
;;
|
||||
?) print_usage; exit 1
|
||||
|
|
@ -89,6 +100,9 @@ done
|
|||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
mar_command="$MAR -V ${MOZ_PRODUCT_VERSION:?} -H ${MAR_CHANNEL_ID:?}"
|
||||
|
||||
# Added for -e for Zotero
|
||||
set +e
|
||||
let arg_start=$OPTIND-1
|
||||
shift $arg_start
|
||||
|
|
@ -103,10 +117,9 @@ if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
|
|||
# Remove the /
|
||||
newdir=$(echo "$newdir" | sed -e 's:\/$::')
|
||||
fi
|
||||
workdir="$newdir.work"
|
||||
updatemanifestv2="$workdir/updatev2.manifest"
|
||||
workdir="$(mktemp -d)"
|
||||
updatemanifestv3="$workdir/updatev3.manifest"
|
||||
archivefiles="updatev2.manifest updatev3.manifest"
|
||||
archivefiles="updatev3.manifest"
|
||||
|
||||
mkdir -p "$workdir"
|
||||
|
||||
|
|
@ -141,10 +154,8 @@ popd
|
|||
# Add the type of update to the beginning of the update manifests.
|
||||
notice ""
|
||||
notice "Adding type instruction to update manifests"
|
||||
> $updatemanifestv2
|
||||
> $updatemanifestv3
|
||||
notice " type partial"
|
||||
echo "type \"partial\"" >> $updatemanifestv2
|
||||
echo "type \"partial\"" >> $updatemanifestv3
|
||||
|
||||
notice ""
|
||||
|
|
@ -163,7 +174,7 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
|
|||
if check_for_add_if_not_update "$f"; then
|
||||
# The full workdir may not exist yet, so create it if necessary.
|
||||
mkdir -p `dirname "$workdir/$f"`
|
||||
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
|
||||
$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
|
||||
copy_perm "$newdir/$f" "$workdir/$f"
|
||||
make_add_if_not_instruction "$f" "$updatemanifestv3"
|
||||
archivefiles="$archivefiles \"$f\""
|
||||
|
|
@ -173,9 +184,9 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
|
|||
if check_for_forced_update "$requested_forced_updates" "$f"; then
|
||||
# The full workdir may not exist yet, so create it if necessary.
|
||||
mkdir -p `dirname "$workdir/$f"`
|
||||
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
|
||||
$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
|
||||
copy_perm "$newdir/$f" "$workdir/$f"
|
||||
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
|
||||
make_add_instruction "$f" "$updatemanifestv3" 1
|
||||
archivefiles="$archivefiles \"$f\""
|
||||
continue 1
|
||||
fi
|
||||
|
|
@ -185,7 +196,7 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
|
|||
# compare the sizes. Then choose the smaller of the two to package.
|
||||
dir=$(dirname "$workdir/$f")
|
||||
mkdir -p "$dir"
|
||||
notice "diffing \"$f\""
|
||||
verbose_notice "diffing \"$f\""
|
||||
# MBSDIFF_HOOK represents the communication interface with funsize and,
|
||||
# if enabled, caches the intermediate patches for future use and
|
||||
# compute avoidance
|
||||
|
|
@ -196,36 +207,38 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
|
|||
# myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
|
||||
# PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
|
||||
#
|
||||
# Note: patches are bzipped stashed in funsize to gain more speed
|
||||
# Note: patches are bzipped or xz stashed in funsize to gain more speed
|
||||
|
||||
# if service is not enabled then default to old behavior
|
||||
if [ -z "$MBSDIFF_HOOK" ]; then
|
||||
# Disabled for Zotero
|
||||
#if [ -z "$MBSDIFF_HOOK" ]; then
|
||||
if true; then
|
||||
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
|
||||
$BZIP2 -z9 "$workdir/$f.patch"
|
||||
$XZ $XZ_OPT --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
|
||||
else
|
||||
# if service enabled then check patch existence for retrieval
|
||||
if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
|
||||
notice "file \"$f\" found in funsize, diffing skipped"
|
||||
if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then
|
||||
verbose_notice "file \"$f\" found in funsize, diffing skipped"
|
||||
else
|
||||
# if not found already - compute it and cache it for future use
|
||||
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
|
||||
$BZIP2 -z9 "$workdir/$f.patch"
|
||||
$MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
|
||||
$XZ $XZ_OPT --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
|
||||
$MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"
|
||||
fi
|
||||
fi
|
||||
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
|
||||
$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
|
||||
copy_perm "$newdir/$f" "$workdir/$f"
|
||||
patchfile="$workdir/$f.patch.bz2"
|
||||
patchfile="$workdir/$f.patch.xz"
|
||||
patchsize=$(get_file_size "$patchfile")
|
||||
fullsize=$(get_file_size "$workdir/$f")
|
||||
|
||||
if [ $patchsize -lt $fullsize ]; then
|
||||
make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
|
||||
make_patch_instruction "$f" "$updatemanifestv3"
|
||||
mv -f "$patchfile" "$workdir/$f.patch"
|
||||
rm -f "$workdir/$f"
|
||||
archivefiles="$archivefiles \"$f.patch\""
|
||||
else
|
||||
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
|
||||
make_add_instruction "$f" "$updatemanifestv3"
|
||||
rm -f "$patchfile"
|
||||
archivefiles="$archivefiles \"$f\""
|
||||
fi
|
||||
|
|
@ -258,13 +271,13 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); do
|
|||
dir=$(dirname "$workdir/$f")
|
||||
mkdir -p "$dir"
|
||||
|
||||
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
|
||||
$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
|
||||
copy_perm "$newdir/$f" "$workdir/$f"
|
||||
|
||||
if check_for_add_if_not_update "$f"; then
|
||||
make_add_if_not_instruction "$f" "$updatemanifestv3"
|
||||
else
|
||||
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
|
||||
make_add_instruction "$f" "$updatemanifestv3"
|
||||
fi
|
||||
|
||||
|
||||
|
|
@ -275,15 +288,14 @@ notice ""
|
|||
notice "Adding file remove instructions to update manifests"
|
||||
for ((i=0; $i<$num_removes; i=$i+1)); do
|
||||
f="${remove_array[$i]}"
|
||||
notice " remove \"$f\""
|
||||
echo "remove \"$f\"" >> $updatemanifestv2
|
||||
verbose_notice " remove \"$f\""
|
||||
echo "remove \"$f\"" >> $updatemanifestv3
|
||||
done
|
||||
|
||||
# Add remove instructions for any dead files.
|
||||
notice ""
|
||||
notice "Adding file and directory remove instructions from file 'removed-files'"
|
||||
append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3"
|
||||
append_remove_instructions "$newdir" "$updatemanifestv3"
|
||||
|
||||
notice ""
|
||||
notice "Adding directory remove instructions for directories that no longer exist"
|
||||
|
|
@ -293,29 +305,15 @@ for ((i=0; $i<$num_olddirs; i=$i+1)); do
|
|||
f="${olddirs[$i]}"
|
||||
# If this dir doesn't exist in the new directory remove it.
|
||||
if [ ! -d "$newdir/$f" ]; then
|
||||
notice " rmdir $f/"
|
||||
echo "rmdir \"$f/\"" >> $updatemanifestv2
|
||||
verbose_notice " rmdir $f/"
|
||||
echo "rmdir \"$f/\"" >> $updatemanifestv3
|
||||
fi
|
||||
done
|
||||
|
||||
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
|
||||
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
|
||||
$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
|
||||
|
||||
mar_command="$MAR"
|
||||
if [[ -n $MOZ_PRODUCT_VERSION ]]
|
||||
then
|
||||
mar_command="$mar_command -V $MOZ_PRODUCT_VERSION"
|
||||
fi
|
||||
if [[ -n $MOZ_CHANNEL_ID ]]
|
||||
then
|
||||
mar_command="$mar_command -H $MOZ_CHANNEL_ID"
|
||||
fi
|
||||
# Changed for Zotero -- -C is unreliable
|
||||
pushd $workdir > /dev/null
|
||||
mar_command="$mar_command -c output.mar"
|
||||
mar_command="$mar_command -C \"$workdir\" -c output.mar"
|
||||
eval "$mar_command $archivefiles"
|
||||
popd > /dev/null
|
||||
mv -f "$workdir/output.mar" "$archive"
|
||||
|
||||
# cleanup
|
||||
|
|
|
|||
61
app/update-packaging/xz_to_bzip
Executable file
61
app/update-packaging/xz_to_bzip
Executable file
|
|
@ -0,0 +1,61 @@
|
|||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
APP_ROOT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
. "$APP_ROOT_DIR/config.sh"
|
||||
|
||||
infile=${1:-}
|
||||
outfile=${2:-}
|
||||
if [[ -z "$infile" ]] || [[ -z "$infile" ]]; then
|
||||
echo "Usage: $0 infile.mar outfile.mar"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mar="$APP_ROOT_DIR/xulrunner/bin/mar"
|
||||
if [ ! -f "$mar" ]; then
|
||||
echo "$mar not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tmp_dir=`mktemp -d`
|
||||
function cleanup {
|
||||
rm -rf $tmp_dir
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cd "$tmp_dir"
|
||||
mkdir in
|
||||
cd in
|
||||
|
||||
echo "Extracting MAR"
|
||||
$mar -x "$infile"
|
||||
|
||||
echo "Renaming all files to .xz"
|
||||
find . -type f -exec mv {} {}.xz \;
|
||||
echo "Uncompressing files"
|
||||
find . -type f -exec unxz {} \;
|
||||
echo "Recompressing files with bzip2"
|
||||
find . -type f -exec bzip2 {} \;
|
||||
echo "Removing .bz extension"
|
||||
find . -type f | while read f; do mv "$f" "${f%.bz2}"; done
|
||||
|
||||
channel=$($mar -T "$infile" | sed -n -r 's/.+MAR channel name: ([^\s]+)/\1/p')
|
||||
version=$($mar -T "$infile" | sed -n -r 's/.+Product version: ([^\s]+)/\1/p')
|
||||
|
||||
if [ -z "$channel" ]; then
|
||||
echo "Could not detect channel"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$version" ]; then
|
||||
echo "Could not detect version"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "MAR channel name: $channel"
|
||||
echo "Product version: $version"
|
||||
|
||||
echo "Creating new MAR"
|
||||
$mar -V $version -H $channel -c "$outfile" `find . -type f | sed 's/^\.\///'`
|
||||
echo
|
||||
$mar -T "$outfile"
|
||||
Loading…
Reference in a new issue