Enable easy rebuild of native modules for unreleased electron (#12217)

* Enable easy rebuild of native modules for unreleased electron

Sometimes when developing locally (as I found this morning) you want to rebuild native modules
against an unreleased version of Electron (in this case local master).  This PR adds a simple
script to launch a local HTTP server that generates and hosts the required header and checksum
files to rebuild modules.  So far only tested on macOS but should work on all OS's

* Allow custom port

* clean up serve script

* make optionality more obvious

package.json

@@ -20,6 +20,7 @@
     "remark-cli": "^4.0.0",
     "remark-preset-lint-markdown-style-guide": "^2.1.1",
     "request": "^2.68.0",
+    "serve": "^6.5.3",
     "standard": "^10.0.0",
     "standard-markdown": "^4.0.0",
     "sumchecker": "^2.0.2",

script/serve-node-headers.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+import argparse
+import atexit
+import os
+import shutil
+import sys
+import tarfile
+import time
+
+from subprocess import Popen, PIPE
+from lib.util import execute_stdout
+
+SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+DIST_DIR    = os.path.join(SOURCE_ROOT, 'dist')
+
+
+def main():
+  args = parse_args()
+  header_dir = os.path.join(DIST_DIR, args.version)
+
+  # Generate Headers
+  script_path = os.path.join(SOURCE_ROOT, 'script', 'create-node-headers.py')
+  execute_stdout([sys.executable, script_path, '--version', args.version,
+                  '--directory', header_dir])
+
+  # Launch server
+  script_path = os.path.join(SOURCE_ROOT, 'node_modules', 'serve', 'bin',
+                            'serve.js')
+  server = Popen(['node', script_path, '--port=' + args.port], stdout=PIPE,
+                 cwd=DIST_DIR)
+  def cleanup():
+    server.kill()
+  atexit.register(cleanup)
+
+  time.sleep(1)
+
+  # Generate Checksums
+  script_path = os.path.join(SOURCE_ROOT, 'script', 'upload-node-checksums.py')
+  execute_stdout([sys.executable, script_path, '--version', args.version,
+                  '--dist-url', 'http://localhost:' + args.port,
+                  '--target-dir', header_dir])
+
+  print("Point your npm config at 'http://localhost:" + args.port + "'")
+  server.wait()
+
+def parse_args():
+  parser = argparse.ArgumentParser(description='create node header tarballs')
+  parser.add_argument('-v', '--version', help='Specify the version',
+                      required=True)
+  parser.add_argument('-p', '--port', help='Specify port to run local server',
+                      default='4321')
+  return parser.parse_args()
+
+if __name__ == '__main__':
+  sys.exit(main())

script/upload-node-checksums.py

@@ -3,10 +3,11 @@
 import argparse
 import hashlib
 import os
+import shutil
 import tempfile
 
 from lib.config import s3_config
-from lib.util import download, rm_rf, s3put
+from lib.util import download, rm_rf, s3put, safe_mkdir
 
 
 DIST_URL = 'https://atom.io/download/electron/'
@@ -14,17 +15,23 @@ DIST_URL = 'https://atom.io/download/electron/'
 
 def main():
   args = parse_args()
+  dist_url = args.dist_url
+  if dist_url[-1] != "/":
+    dist_url += "/"
 
-  url = DIST_URL + args.version + '/'
+  url = dist_url + args.version + '/'
   directory, files = download_files(url, get_files_list(args.version))
   checksums = [
     create_checksum('sha1', directory, 'SHASUMS.txt', files),
     create_checksum('sha256', directory, 'SHASUMS256.txt', files)
   ]
 
-  bucket, access_key, secret_key = s3_config()
+  if args.target_dir is None:
-  s3put(bucket, access_key, secret_key, directory,
+    bucket, access_key, secret_key = s3_config()
-        'atom-shell/dist/{0}'.format(args.version), checksums)
+    s3put(bucket, access_key, secret_key, directory,
+          'atom-shell/dist/{0}'.format(args.version), checksums)
+  else:
+    copy_files(checksums, args.target_dir)
 
   rm_rf(directory)
 
@@ -33,27 +40,39 @@ def parse_args():
   parser = argparse.ArgumentParser(description='upload sumsha file')
   parser.add_argument('-v', '--version', help='Specify the version',
                       required=True)
+  parser.add_argument('-u', '--dist-url',
+                      help='Specify the dist url for downloading',
+                      required=False, default=DIST_URL)
+  parser.add_argument('-t', '--target-dir',
+                      help='Specify target dir of checksums',
+                      required=False)
   return parser.parse_args()
 
-
 def get_files_list(version):
   return [
-    'node-{0}.tar.gz'.format(version),
+    { "filename": 'node-{0}.tar.gz'.format(version), "required": True },
-    'iojs-{0}.tar.gz'.format(version),
+    { "filename": 'iojs-{0}.tar.gz'.format(version), "required": True },
-    'iojs-{0}-headers.tar.gz'.format(version),
+    { "filename": 'iojs-{0}-headers.tar.gz'.format(version), "required": True },
-    'node.lib',
+    { "filename": 'node.lib', "required": False },
-    'x64/node.lib',
+    { "filename": 'x64/node.lib', "required": False },
-    'win-x86/iojs.lib',
+    { "filename": 'win-x86/iojs.lib', "required": False },
-    'win-x64/iojs.lib',
+    { "filename": 'win-x64/iojs.lib', "required": False }
   ]
 
 
 def download_files(url, files):
   directory = tempfile.mkdtemp(prefix='electron-tmp')
-  return directory, [
+  result = []
-    download(f, url + f, os.path.join(directory, f))
+  for optional_f in files:
-    for f in files
+    required = optional_f.required
-  ]
+    f = optional_f.filename
+    try:
+      result.append(download(f, url + f, os.path.join(directory, f)))
+    except Exception:
+      if required:
+        raise
+
+  return directory, result
 
 
 def create_checksum(algorithm, directory, filename, files):
@@ -69,6 +88,11 @@ def create_checksum(algorithm, directory, filename, files):
     f.write('\n'.join(lines) + '\n')
   return checksum_file
 
+def copy_files(source_files, output_dir):
+  for source_file in source_files:
+    output_path = os.path.join(output_dir, os.path.basename(source_file))
+    safe_mkdir(os.path.dirname(output_path))
+    shutil.copy2(source_file, output_path)
 
 if __name__ == '__main__':
   import sys