Merge pull request #9547 from electron/jessie-sysroot

Build with Debian Jessie sysroot image

.gitignore

@@ -6,6 +6,10 @@
 /external_binaries/
 /out/
 /vendor/download/
+/vendor/debian_jessie_amd64-sysroot/
+/vendor/debian_jessie_arm-sysroot/
+/vendor/debian_jessie_arm64-sysroot/
+/vendor/debian_jessie_i386-sysroot/
 /vendor/debian_wheezy_amd64-sysroot/
 /vendor/debian_wheezy_arm-sysroot/
 /vendor/debian_wheezy_i386-sysroot/

script/bootstrap.py

@@ -228,7 +228,8 @@ def download_sysroot(target_arch):
     target_arch = 'amd64'
   execute_stdout([sys.executable,
                   os.path.join(SOURCE_ROOT, 'script', 'install-sysroot.py'),
-                  '--arch', target_arch])
+                  '--arch', target_arch],
+                  cwd=VENDOR_DIR)
 
 def create_chrome_version_h():
   version_file = os.path.join(VENDOR_DIR, 'libchromiumcontent', 'VERSION')

script/install-sysroot.py

@@ -3,19 +3,23 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# Script to install a Debian Wheezy sysroot for making official Google Chrome
+"""Install Debian sysroots for building chromium.
-# Linux builds.
+"""
-# The sysroot is needed to make Chrome work for Debian Wheezy.
-# This script can be run manually but is more often run as part of gclient
-# hooks. When run from hooks this script should be a no-op on non-linux
-# platforms.
 
-# The sysroot image could be constructed from scratch based on the current
+# The sysroot is needed to ensure that binaries that get built will run on
-# state or Debian Wheezy but for consistency we currently use a pre-built root
+# the oldest stable version of Debian that we currently support.
-# image. The image will normally need to be rebuilt every time chrome's build
+# This script can be run manually but is more often run as part of gclient
-# dependancies are changed.
+# hooks. When run from hooks this script is a no-op on non-linux platforms.
+
+# The sysroot image could be constructed from scratch based on the current state
+# of the Debian archive but for consistency we use a pre-built root image (we
+# don't want upstream changes to Debian to effect the chromium build until we
+# choose to pull them in). The images will normally need to be rebuilt every
+# time chrome's build dependencies are changed but should also be updated
+# periodically to include upstream security fixes from Debian.
 
 import hashlib
+import json
 import platform
 import optparse
 import os
@@ -23,27 +27,19 @@ import re
 import shutil
 import subprocess
 import sys
-
+import urllib2
-from lib.util import get_host_arch
-
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-URL_PREFIX = 'https://github.com'
-URL_PATH = 'atom/debian-sysroot-image-creator/releases/download'
-REVISION_AMD64 = 'v0.5.0'
-REVISION_I386 = 'v0.5.0'
-REVISION_ARM = 'v0.5.0'
-TARBALL_AMD64 = 'debian_wheezy_amd64_sysroot.tgz'
-TARBALL_I386 = 'debian_wheezy_i386_sysroot.tgz'
-TARBALL_ARM = 'debian_wheezy_arm_sysroot.tgz'
-TARBALL_AMD64_SHA1SUM = '981b2440d446156801c6fdecffb5edcadf27593c'
-TARBALL_I386_SHA1SUM = '2e4e43c1e8718595e37c6b6ab89256dae53adf23'
-TARBALL_ARM_SHA1SUM = '448e635f38e99d6d860db538a9db85ac74d36e41'
-SYSROOT_DIR_AMD64 = 'debian_wheezy_amd64-sysroot'
-SYSROOT_DIR_I386 = 'debian_wheezy_i386-sysroot'
-SYSROOT_DIR_ARM = 'debian_wheezy_arm-sysroot'
 
-valid_archs = ('arm', 'i386', 'amd64')
+
+URL_PREFIX = 'http://s3.amazonaws.com'
+URL_PATH = 'gh-contractor-zcbenz/toolchain'
+
+VALID_ARCHS = ('arm', 'arm64', 'i386', 'amd64')
+
+
+class Error(Exception):
+  pass
 
 
 def GetSha1(filename):
@@ -58,81 +54,59 @@ def GetSha1(filename):
   return sha1.hexdigest()
 
 
-def DetectArch(gyp_defines):
+def main(args):
-  # Check for optional target_arch and only install for that architecture.
+  parser = optparse.OptionParser('usage: %prog [OPTIONS]', description=__doc__)
-  # If target_arch is not specified, then only install for the host
+  parser.add_option('--running-as-hook', action='store_true',
-  # architecture.
+                    default=False, help='Used when running from gclient hooks.'
-  if 'target_arch=x64' in gyp_defines:
+                                        ' Installs default sysroot images.')
-    return 'amd64'
+  parser.add_option('--arch', type='choice', choices=VALID_ARCHS,
-  elif 'target_arch=ia32' in gyp_defines:
+                    help='Sysroot architecture: %s' % ', '.join(VALID_ARCHS))
-    return 'i386'
+  parser.add_option('--all', action='store_true',
-  elif 'target_arch=arm' in gyp_defines:
+                    help='Install all sysroot images (useful when updating the'
-    return 'arm'
+                         ' images)')
+  options, _ = parser.parse_args(args)
+  if options.running_as_hook and not sys.platform.startswith('linux'):
+    return 0
 
-  detected_host_arch = get_host_arch()
+  if options.running_as_hook:
-  if detected_host_arch == 'x64':
+    return 0
-    return 'amd64'
+  elif options.arch:
-  elif detected_host_arch == 'ia32':
+    InstallDefaultSysrootForArch(options.arch)
-    return 'i386'
+  elif options.all:
-  elif detected_host_arch == 'arm':
+    for arch in VALID_ARCHS:
-    return 'arm'
+      InstallDefaultSysrootForArch(arch)
   else:
-    print "Unknown host arch: %s" % detected_host_arch
+    print 'You much specify either --arch, --all or --running-as-hook'
+    return 1
 
-  return None
+  return 0
 
 
-def main():
+def InstallDefaultSysrootForArch(target_arch):
-  if options.linux_only:
+  if target_arch not in VALID_ARCHS:
-    # This argument is passed when run from the gclient hooks.
+    raise Error('Unknown architecture: %s' % target_arch)
-    # In this case we return early on non-linux platforms.
+  if target_arch == 'arm64':
-    if not sys.platform.startswith('linux'):
+    InstallSysroot('Jessie', target_arch)
-      return 0
-
-  gyp_defines = os.environ.get('GYP_DEFINES', '')
-
-  if options.arch:
-    target_arch = options.arch
   else:
-    target_arch = DetectArch(gyp_defines)
+    InstallSysroot('Wheezy', target_arch)
-    if not target_arch:
-      print 'Unable to detect host architecture'
-      return 1
 
-  if options.linux_only and target_arch != 'arm':
-    # When run from runhooks, only install the sysroot for an Official Chrome
-    # Linux build, except on ARM where we always use a sysroot.
-    defined = ['branding=Chrome', 'buildtype=Official']
-    undefined = ['chromeos=1']
-    for option in defined:
-      if option not in gyp_defines:
-        return 0
-    for option in undefined:
-      if option in gyp_defines:
-        return 0
 
+def InstallSysroot(target_platform, target_arch):
   # The sysroot directory should match the one specified in build/common.gypi.
-  # TODO(thestig) Consider putting this else where to avoid having to recreate
+  # TODO(thestig) Consider putting this elsewhere to avoid having to recreate
   # it on every build.
-  linux_dir = os.path.join(SCRIPT_DIR, '..', 'vendor')
+  linux_dir = os.path.dirname(SCRIPT_DIR)
-  if target_arch == 'amd64':
+
-    sysroot = os.path.join(linux_dir, SYSROOT_DIR_AMD64)
+  sysroots_file = os.path.join(SCRIPT_DIR, 'sysroots.json')
-    tarball_filename = TARBALL_AMD64
+  sysroots = json.load(open(sysroots_file))
-    tarball_sha1sum = TARBALL_AMD64_SHA1SUM
+  sysroot_key = '%s_%s' % (target_platform.lower(), target_arch)
-    revision = REVISION_AMD64
+  if sysroot_key not in sysroots:
-  elif target_arch == 'arm':
+    raise Error('No sysroot for: %s %s' % (target_platform, target_arch))
-    sysroot = os.path.join(linux_dir, SYSROOT_DIR_ARM)
+  sysroot_dict = sysroots[sysroot_key]
-    tarball_filename = TARBALL_ARM
+  revision = sysroot_dict['Revision']
-    tarball_sha1sum = TARBALL_ARM_SHA1SUM
+  tarball_filename = sysroot_dict['Tarball']
-    revision = REVISION_ARM
+  tarball_sha1sum = sysroot_dict['Sha1Sum']
-  elif target_arch == 'i386':
+  sysroot = os.path.join(linux_dir, 'vendor', sysroot_dict['SysrootDir'])
-    sysroot = os.path.join(linux_dir, SYSROOT_DIR_I386)
-    tarball_filename = TARBALL_I386
-    tarball_sha1sum = TARBALL_I386_SHA1SUM
-    revision = REVISION_I386
-  else:
-    print 'Unknown architecture: %s' % target_arch
-    assert(False)
 
   url = '%s/%s/%s/%s' % (URL_PREFIX, URL_PATH, revision, tarball_filename)
 
@@ -140,11 +114,12 @@ def main():
   if os.path.exists(stamp):
     with open(stamp) as s:
       if s.read() == url:
-        print 'Debian Wheezy %s root image already up-to-date: %s' % \
+        print '%s %s sysroot image already up to date: %s' % \
-            (target_arch, sysroot)
+            (target_platform, target_arch, sysroot)
-        return 0
+        return
 
-  print 'Installing Debian Wheezy %s root image: %s' % (target_arch, sysroot)
+  print 'Installing Debian %s %s root image: %s' % \
+      (target_platform, target_arch, sysroot)
   if os.path.isdir(sysroot):
     shutil.rmtree(sysroot)
   os.mkdir(sysroot)
@@ -152,26 +127,30 @@ def main():
   print 'Downloading %s' % url
   sys.stdout.flush()
   sys.stderr.flush()
-  subprocess.check_call(['curl', '--fail', '-L', url, '-o', tarball])
+  for _ in range(3):
+    try:
+      response = urllib2.urlopen(url)
+      with open(tarball, "wb") as f:
+        f.write(response.read())
+      break
+    except Exception:
+      pass
+  else:
+    raise Error('Failed to download %s' % url)
   sha1sum = GetSha1(tarball)
   if sha1sum != tarball_sha1sum:
-    print 'Tarball sha1sum is wrong.'
+    raise Error('Tarball sha1sum is wrong.'
-    print 'Expected %s, actual: %s' % (tarball_sha1sum, sha1sum)
+                'Expected %s, actual: %s' % (tarball_sha1sum, sha1sum))
-    return 1
   subprocess.check_call(['tar', 'xf', tarball, '-C', sysroot])
   os.remove(tarball)
 
   with open(stamp, 'w') as s:
     s.write(url)
-  return 0
 
 
 if __name__ == '__main__':
-  parser = optparse.OptionParser('usage: %prog [OPTIONS]')
+  try:
-  parser.add_option('--linux-only', action='store_true',
+    sys.exit(main(sys.argv[1:]))
-                    default=False, help='Only install sysroot for official '
+  except Error as e:
-                                        'Linux builds')
+    sys.stderr.write(str(e) + '\n')
-  parser.add_option('--arch', type='choice', choices=valid_archs,
+    sys.exit(1)
-                    help='Sysroot architecture: %s' % ', '.join(valid_archs))
-  options, _ = parser.parse_args()
-  sys.exit(main())

script/lib/util.py

@@ -170,11 +170,11 @@ def execute(argv, env=os.environ):
     raise e
 
 
-def execute_stdout(argv, env=os.environ):
+def execute_stdout(argv, env=os.environ, cwd=None):
   if is_verbose_mode():
     print ' '.join(argv)
     try:
-      subprocess.check_call(argv, env=env)
+      subprocess.check_call(argv, env=env, cwd=cwd)
     except subprocess.CalledProcessError as e:
       print e.output
       raise e

script/sysroots.json

@@ -0,0 +1,56 @@
+{
+    "jessie_amd64": {
+        "Revision": "d65c31e063bab0486665e087a1b4c5bb7bc7423c",
+        "Sha1Sum": "8b2167b36f3cd85ebbec5c2a39a1842ef613f6a2",
+        "SysrootDir": "debian_jessie_amd64-sysroot",
+        "Tarball": "debian_jessie_amd64_sysroot.tgz"
+    },
+    "jessie_arm": {
+        "Revision": "d65c31e063bab0486665e087a1b4c5bb7bc7423c",
+        "Sha1Sum": "3fa13635be0c6d8ed461715ad51cdb3809a19422",
+        "SysrootDir": "debian_jessie_arm-sysroot",
+        "Tarball": "debian_jessie_arm_sysroot.tgz"
+    },
+    "jessie_arm64": {
+        "Revision": "d65c31e063bab0486665e087a1b4c5bb7bc7423c",
+        "Sha1Sum": "bcf92ed2a033b4b2d1032df3b53eac4910c78fde",
+        "SysrootDir": "debian_jessie_arm64-sysroot",
+        "Tarball": "debian_jessie_arm64_sysroot.tgz"
+    },
+    "jessie_i386": {
+        "Revision": "d65c31e063bab0486665e087a1b4c5bb7bc7423c",
+        "Sha1Sum": "4e83ed9a1b457a1ca59512c3d4823e87e950deb4",
+        "SysrootDir": "debian_jessie_i386-sysroot",
+        "Tarball": "debian_jessie_i386_sysroot.tgz"
+    },
+    "jessie_mips": {
+        "Revision": "d65c31e063bab0486665e087a1b4c5bb7bc7423c",
+        "Sha1Sum": "0f550cd150f077a6e749a629e2fda0f0a4348eae",
+        "SysrootDir": "debian_jessie_mips-sysroot",
+        "Tarball": "debian_jessie_mips_sysroot.tgz"
+    },
+    "wheezy_amd64": {
+        "Revision": "d65c31e063bab0486665e087a1b4c5bb7bc7423c",
+        "Sha1Sum": "fe372c4394ece7fd1d853a205de8c13b46e2f45a",
+        "SysrootDir": "debian_wheezy_amd64-sysroot",
+        "Tarball": "debian_wheezy_amd64_sysroot.tgz"
+    },
+    "wheezy_arm": {
+        "Revision": "d65c31e063bab0486665e087a1b4c5bb7bc7423c",
+        "Sha1Sum": "73323fae0b5597398a38f0d7e64f48309da112fa",
+        "SysrootDir": "debian_wheezy_arm-sysroot",
+        "Tarball": "debian_wheezy_arm_sysroot.tgz"
+    },
+    "wheezy_i386": {
+        "Revision": "d65c31e063bab0486665e087a1b4c5bb7bc7423c",
+        "Sha1Sum": "7e96584297a5c9916b3bed4b88abe332fc79a247",
+        "SysrootDir": "debian_wheezy_i386-sysroot",
+        "Tarball": "debian_wheezy_i386_sysroot.tgz"
+    },
+    "wheezy_mips": {
+        "Revision": "d65c31e063bab0486665e087a1b4c5bb7bc7423c",
+        "Sha1Sum": "b2f173905a41ec9f23c329fe529508b4bdc76230",
+        "SysrootDir": "debian_wheezy_mips-sysroot",
+        "Tarball": "debian_wheezy_mips_sysroot.tgz"
+    }
+}

toolchain.gypi

@@ -52,6 +52,9 @@
               # incorrect results when passed to pkg-config
               'sysroot%': '<(source_root)/vendor/debian_wheezy_arm-sysroot',
             }],
+            ['target_arch=="arm64"', {
+              'sysroot%': '<(source_root)/vendor/debian_jessie_arm64-sysroot',
+            }],
             ['target_arch=="ia32"', {
               'sysroot%': '<(source_root)/vendor/debian_wheezy_i386-sysroot',
             }],