Download debian jessie sysroot images

script/install-sysroot.py

@@ -3,19 +3,23 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# Script to install a Debian Wheezy sysroot for making official Google Chrome
+"""Install Debian sysroots for building chromium.
-# Linux builds.
+"""
-# The sysroot is needed to make Chrome work for Debian Wheezy.
-# This script can be run manually but is more often run as part of gclient
-# hooks. When run from hooks this script should be a no-op on non-linux
-# platforms.
 
-# The sysroot image could be constructed from scratch based on the current
+# The sysroot is needed to ensure that binaries that get built will run on
-# state or Debian Wheezy but for consistency we currently use a pre-built root
+# the oldest stable version of Debian that we currently support.
-# image. The image will normally need to be rebuilt every time chrome's build
+# This script can be run manually but is more often run as part of gclient
-# dependancies are changed.
+# hooks. When run from hooks this script is a no-op on non-linux platforms.
+
+# The sysroot image could be constructed from scratch based on the current state
+# of the Debian archive but for consistency we use a pre-built root image (we
+# don't want upstream changes to Debian to effect the chromium build until we
+# choose to pull them in). The images will normally need to be rebuilt every
+# time chrome's build dependencies are changed but should also be updated
+# periodically to include upstream security fixes from Debian.
 
 import hashlib
+import json
 import platform
 import optparse
 import os
@@ -23,27 +27,19 @@ import re
 import shutil
 import subprocess
 import sys
-
+import urllib2
-from lib.util import get_host_arch
-
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-URL_PREFIX = 'https://github.com'
-URL_PATH = 'atom/debian-sysroot-image-creator/releases/download'
-REVISION_AMD64 = 'v0.5.0'
-REVISION_I386 = 'v0.5.0'
-REVISION_ARM = 'v0.5.0'
-TARBALL_AMD64 = 'debian_wheezy_amd64_sysroot.tgz'
-TARBALL_I386 = 'debian_wheezy_i386_sysroot.tgz'
-TARBALL_ARM = 'debian_wheezy_arm_sysroot.tgz'
-TARBALL_AMD64_SHA1SUM = '981b2440d446156801c6fdecffb5edcadf27593c'
-TARBALL_I386_SHA1SUM = '2e4e43c1e8718595e37c6b6ab89256dae53adf23'
-TARBALL_ARM_SHA1SUM = '448e635f38e99d6d860db538a9db85ac74d36e41'
-SYSROOT_DIR_AMD64 = 'debian_wheezy_amd64-sysroot'
-SYSROOT_DIR_I386 = 'debian_wheezy_i386-sysroot'
-SYSROOT_DIR_ARM = 'debian_wheezy_arm-sysroot'
 
-valid_archs = ('arm', 'i386', 'amd64')
+
+URL_PREFIX = 'http://s3.amazonaws.com'
+URL_PATH = 'gh-contractor-zcbenz/toolchain'
+
+VALID_ARCHS = ('arm', 'arm64', 'i386', 'amd64')
+
+
+class Error(Exception):
+  pass
 
 
 def GetSha1(filename):
@@ -58,81 +54,74 @@ def GetSha1(filename):
   return sha1.hexdigest()
 
 
-def DetectArch(gyp_defines):
+def InstallDefaultSysroots(host_arch):
-  # Check for optional target_arch and only install for that architecture.
+  """Install the default set of sysroot images.
-  # If target_arch is not specified, then only install for the host
-  # architecture.
-  if 'target_arch=x64' in gyp_defines:
-    return 'amd64'
-  elif 'target_arch=ia32' in gyp_defines:
-    return 'i386'
-  elif 'target_arch=arm' in gyp_defines:
-    return 'arm'
 
-  detected_host_arch = get_host_arch()
+  This includes at least the sysroot for host architecture, and the 32-bit
-  if detected_host_arch == 'x64':
+  sysroot for building the v8 snapshot image.  It can also include the cross
-    return 'amd64'
+  compile sysroot for ARM/MIPS if cross compiling environment can be detected.
-  elif detected_host_arch == 'ia32':
+
-    return 'i386'
+  Another reason we're installing this by default is so that developers can
-  elif detected_host_arch == 'arm':
+  compile and run on our supported platforms without having to worry about
-    return 'arm'
+  flipping things back and forth and whether the sysroots have been downloaded
+  or not.
+  """
+  InstallSysroot('amd64')
+  InstallSysroot('i386')
+  InstallSysroot('arm')
+  InstallSysroot('arm64')
+
+
+def main(args):
+  parser = optparse.OptionParser('usage: %prog [OPTIONS]', description=__doc__)
+  parser.add_option('--running-as-hook', action='store_true',
+                    default=False, help='Used when running from gclient hooks.'
+                                        ' Installs default sysroot images.')
+  parser.add_option('--arch', type='choice', choices=VALID_ARCHS,
+                    help='Sysroot architecture: %s' % ', '.join(VALID_ARCHS))
+  parser.add_option('--all', action='store_true',
+                    help='Install all sysroot images (useful when updating the'
+                         ' images)')
+  options, _ = parser.parse_args(args)
+  if options.running_as_hook and not sys.platform.startswith('linux'):
+    return 0
+
+  if options.running_as_hook:
+    return 0
+  elif options.arch:
+    InstallDefaultSysrootForArch(options.arch)
+  elif options.all:
+    for arch in VALID_ARCHS:
+      InstallDefaultSysrootForArch(arch)
   else:
-    print "Unknown host arch: %s" % detected_host_arch
+    print 'You much specify either --arch, --all or --running-as-hook'
+    return 1
 
-  return None
+  return 0
 
 
-def main():
+def InstallDefaultSysrootForArch(target_arch):
-  if options.linux_only:
+  if target_arch not in VALID_ARCHS:
-    # This argument is passed when run from the gclient hooks.
+    raise Error('Unknown architecture: %s' % target_arch)
-    # In this case we return early on non-linux platforms.
+  InstallSysroot('Jessie', target_arch)
-    if not sys.platform.startswith('linux'):
-      return 0
 
-  gyp_defines = os.environ.get('GYP_DEFINES', '')
-
-  if options.arch:
-    target_arch = options.arch
-  else:
-    target_arch = DetectArch(gyp_defines)
-    if not target_arch:
-      print 'Unable to detect host architecture'
-      return 1
-
-  if options.linux_only and target_arch != 'arm':
-    # When run from runhooks, only install the sysroot for an Official Chrome
-    # Linux build, except on ARM where we always use a sysroot.
-    defined = ['branding=Chrome', 'buildtype=Official']
-    undefined = ['chromeos=1']
-    for option in defined:
-      if option not in gyp_defines:
-        return 0
-    for option in undefined:
-      if option in gyp_defines:
-        return 0
 
+def InstallSysroot(target_platform, target_arch):
   # The sysroot directory should match the one specified in build/common.gypi.
-  # TODO(thestig) Consider putting this else where to avoid having to recreate
+  # TODO(thestig) Consider putting this elsewhere to avoid having to recreate
   # it on every build.
-  linux_dir = os.path.join(SCRIPT_DIR, '..', 'vendor')
+  linux_dir = os.path.dirname(SCRIPT_DIR)
-  if target_arch == 'amd64':
+
-    sysroot = os.path.join(linux_dir, SYSROOT_DIR_AMD64)
+  sysroots_file = os.path.join(SCRIPT_DIR, 'sysroots.json')
-    tarball_filename = TARBALL_AMD64
+  sysroots = json.load(open(sysroots_file))
-    tarball_sha1sum = TARBALL_AMD64_SHA1SUM
+  sysroot_key = '%s_%s' % (target_platform.lower(), target_arch)
-    revision = REVISION_AMD64
+  if sysroot_key not in sysroots:
-  elif target_arch == 'arm':
+    raise Error('No sysroot for: %s %s' % (target_platform, target_arch))
-    sysroot = os.path.join(linux_dir, SYSROOT_DIR_ARM)
+  sysroot_dict = sysroots[sysroot_key]
-    tarball_filename = TARBALL_ARM
+  revision = sysroot_dict['Revision']
-    tarball_sha1sum = TARBALL_ARM_SHA1SUM
+  tarball_filename = sysroot_dict['Tarball']
-    revision = REVISION_ARM
+  tarball_sha1sum = sysroot_dict['Sha1Sum']
-  elif target_arch == 'i386':
+  sysroot = os.path.join(linux_dir, sysroot_dict['SysrootDir'])
-    sysroot = os.path.join(linux_dir, SYSROOT_DIR_I386)
-    tarball_filename = TARBALL_I386
-    tarball_sha1sum = TARBALL_I386_SHA1SUM
-    revision = REVISION_I386
-  else:
-    print 'Unknown architecture: %s' % target_arch
-    assert(False)
 
   url = '%s/%s/%s/%s' % (URL_PREFIX, URL_PATH, revision, tarball_filename)
 
@@ -140,11 +129,12 @@ def main():
   if os.path.exists(stamp):
     with open(stamp) as s:
       if s.read() == url:
-        print 'Debian Wheezy %s root image already up-to-date: %s' % \
+        print '%s %s sysroot image already up to date: %s' % \
-            (target_arch, sysroot)
+            (target_platform, target_arch, sysroot)
-        return 0
+        return
 
-  print 'Installing Debian Wheezy %s root image: %s' % (target_arch, sysroot)
+  print 'Installing Debian %s %s root image: %s' % \
+      (target_platform, target_arch, sysroot)
   if os.path.isdir(sysroot):
     shutil.rmtree(sysroot)
   os.mkdir(sysroot)
@@ -152,26 +142,30 @@ def main():
   print 'Downloading %s' % url
   sys.stdout.flush()
   sys.stderr.flush()
-  subprocess.check_call(['curl', '--fail', '-L', url, '-o', tarball])
+  for _ in range(3):
+    try:
+      response = urllib2.urlopen(url)
+      with open(tarball, "wb") as f:
+        f.write(response.read())
+      break
+    except:
+      pass
+  else:
+    raise Error('Failed to download %s' % url)
   sha1sum = GetSha1(tarball)
   if sha1sum != tarball_sha1sum:
-    print 'Tarball sha1sum is wrong.'
+    raise Error('Tarball sha1sum is wrong.'
-    print 'Expected %s, actual: %s' % (tarball_sha1sum, sha1sum)
+                'Expected %s, actual: %s' % (tarball_sha1sum, sha1sum))
-    return 1
   subprocess.check_call(['tar', 'xf', tarball, '-C', sysroot])
   os.remove(tarball)
 
   with open(stamp, 'w') as s:
     s.write(url)
-  return 0
 
 
 if __name__ == '__main__':
-  parser = optparse.OptionParser('usage: %prog [OPTIONS]')
+  try:
-  parser.add_option('--linux-only', action='store_true',
+    sys.exit(main(sys.argv[1:]))
-                    default=False, help='Only install sysroot for official '
+  except Error as e:
-                                        'Linux builds')
+    sys.stderr.write(str(e) + '\n')
-  parser.add_option('--arch', type='choice', choices=valid_archs,
+    sys.exit(1)
-                    help='Sysroot architecture: %s' % ', '.join(valid_archs))
-  options, _ = parser.parse_args()
-  sys.exit(main())

script/sysroots.json

@@ -0,0 +1,32 @@
+{
+    "jessie_amd64": {
+        "Revision": "d18016aaa5283f85bd8c722c42e55d2f5f13384e",
+        "Sha1Sum": "82c0c27b1cbb7a7c2e496aeb6d9f6e28373126db",
+        "SysrootDir": "debian_jessie_amd64-sysroot",
+        "Tarball": "debian_jessie_amd64_sysroot.tgz"
+    },
+    "jessie_arm": {
+        "Revision": "d18016aaa5283f85bd8c722c42e55d2f5f13384e",
+        "Sha1Sum": "c41fd22e33afe9a6037ecac8880e40f99a6a2522",
+        "SysrootDir": "debian_jessie_arm-sysroot",
+        "Tarball": "debian_jessie_arm_sysroot.tgz"
+    },
+    "jessie_arm64": {
+        "Revision": "d18016aaa5283f85bd8c722c42e55d2f5f13384e",
+        "Sha1Sum": "a8579b53d7e6c14cd6f15bf9f25d3a938755d834",
+        "SysrootDir": "debian_jessie_arm64-sysroot",
+        "Tarball": "debian_jessie_arm64_sysroot.tgz"
+    },
+    "jessie_i386": {
+        "Revision": "d18016aaa5283f85bd8c722c42e55d2f5f13384e",
+        "Sha1Sum": "85c86f16036e952e33a70ae74e37775f26c54d48",
+        "SysrootDir": "debian_jessie_i386-sysroot",
+        "Tarball": "debian_jessie_i386_sysroot.tgz"
+    },
+    "jessie_mips": {
+        "Revision": "d18016aaa5283f85bd8c722c42e55d2f5f13384e",
+        "Sha1Sum": "b6d4cdb21fb61c5669eea5135f7c87a19fc16876",
+        "SysrootDir": "debian_jessie_mips-sysroot",
+        "Tarball": "debian_jessie_mips_sysroot.tgz"
+    }
+}