cf497ea478
This commit uploads the output of the windows toolchain profiler.
The windows toolchain profiler is aimed at allowing the compilation
of electron release artifacts using the same windows toolchain on user
builder. This combined with the ability of electron of outputting
deterministic build should allow in future to check electron build
artifacts against CI artifacts.
See: 97959b5e5c include windows toolchain profiler (contd #20949)
479 lines
16 KiB
JavaScript
Executable file
479 lines
16 KiB
JavaScript
Executable file
#!/usr/bin/env node
|
|
|
|
if (!process.env.CI) require('dotenv-safe').load()
|
|
|
|
const args = require('minimist')(process.argv.slice(2), {
|
|
boolean: [
|
|
'validateRelease',
|
|
'skipVersionCheck',
|
|
'automaticRelease',
|
|
'verboseNugget'
|
|
],
|
|
default: { 'verboseNugget': false }
|
|
})
|
|
const fs = require('fs')
|
|
const { execSync } = require('child_process')
|
|
const nugget = require('nugget')
|
|
const got = require('got')
|
|
const pkg = require('../../package.json')
|
|
const pkgVersion = `v${pkg.version}`
|
|
const path = require('path')
|
|
const sumchecker = require('sumchecker')
|
|
const temp = require('temp').track()
|
|
const { URL } = require('url')
|
|
|
|
require('colors')
|
|
const pass = '✓'.green
|
|
const fail = '✗'.red
|
|
|
|
const { ELECTRON_DIR } = require('../lib/utils')
|
|
|
|
const octokit = require('@octokit/rest')({
|
|
auth: process.env.ELECTRON_GITHUB_TOKEN
|
|
})
|
|
|
|
const targetRepo = pkgVersion.indexOf('nightly') > 0 ? 'nightlies' : 'electron'
|
|
let failureCount = 0
|
|
|
|
async function getDraftRelease (version, skipValidation) {
|
|
const releaseInfo = await octokit.repos.listReleases({
|
|
owner: 'electron',
|
|
repo: targetRepo
|
|
})
|
|
|
|
const versionToCheck = version || pkgVersion
|
|
const drafts = releaseInfo.data.filter(release => {
|
|
return release.tag_name === versionToCheck && release.draft === true
|
|
})
|
|
|
|
const draft = drafts[0]
|
|
if (!skipValidation) {
|
|
failureCount = 0
|
|
check(drafts.length === 1, 'one draft exists', true)
|
|
if (versionToCheck.indexOf('beta') > -1) {
|
|
check(draft.prerelease, 'draft is a prerelease')
|
|
}
|
|
check(draft.body.length > 50 && !draft.body.includes('(placeholder)'), 'draft has release notes')
|
|
check((failureCount === 0), `Draft release looks good to go.`, true)
|
|
}
|
|
return draft
|
|
}
|
|
|
|
async function validateReleaseAssets (release, validatingRelease) {
|
|
const requiredAssets = assetsForVersion(release.tag_name, validatingRelease).sort()
|
|
const extantAssets = release.assets.map(asset => asset.name).sort()
|
|
const downloadUrls = release.assets.map(asset => asset.browser_download_url).sort()
|
|
|
|
failureCount = 0
|
|
requiredAssets.forEach(asset => {
|
|
check(extantAssets.includes(asset), asset)
|
|
})
|
|
check((failureCount === 0), `All required GitHub assets exist for release`, true)
|
|
|
|
if (!validatingRelease || !release.draft) {
|
|
if (release.draft) {
|
|
await verifyAssets(release)
|
|
} else {
|
|
await verifyShasums(downloadUrls)
|
|
.catch(err => {
|
|
console.log(`${fail} error verifyingShasums`, err)
|
|
})
|
|
}
|
|
const s3Urls = s3UrlsForVersion(release.tag_name)
|
|
await verifyShasums(s3Urls, true)
|
|
}
|
|
}
|
|
|
|
function check (condition, statement, exitIfFail = false) {
|
|
if (condition) {
|
|
console.log(`${pass} ${statement}`)
|
|
} else {
|
|
failureCount++
|
|
console.log(`${fail} ${statement}`)
|
|
if (exitIfFail) process.exit(1)
|
|
}
|
|
}
|
|
|
|
function assetsForVersion (version, validatingRelease) {
|
|
const patterns = [
|
|
`chromedriver-${version}-darwin-x64.zip`,
|
|
`chromedriver-${version}-linux-arm64.zip`,
|
|
`chromedriver-${version}-linux-armv7l.zip`,
|
|
`chromedriver-${version}-linux-ia32.zip`,
|
|
`chromedriver-${version}-linux-x64.zip`,
|
|
`chromedriver-${version}-mas-x64.zip`,
|
|
`chromedriver-${version}-win32-ia32.zip`,
|
|
`chromedriver-${version}-win32-x64.zip`,
|
|
`chromedriver-${version}-win32-arm64.zip`,
|
|
`electron-${version}-darwin-x64-dsym.zip`,
|
|
`electron-${version}-darwin-x64-symbols.zip`,
|
|
`electron-${version}-darwin-x64.zip`,
|
|
`electron-${version}-linux-arm64-symbols.zip`,
|
|
`electron-${version}-linux-arm64.zip`,
|
|
`electron-${version}-linux-armv7l-symbols.zip`,
|
|
`electron-${version}-linux-armv7l.zip`,
|
|
`electron-${version}-linux-ia32-symbols.zip`,
|
|
`electron-${version}-linux-ia32.zip`,
|
|
`electron-${version}-linux-x64-debug.zip`,
|
|
`electron-${version}-linux-x64-symbols.zip`,
|
|
`electron-${version}-linux-x64.zip`,
|
|
`electron-${version}-mas-x64-dsym.zip`,
|
|
`electron-${version}-mas-x64-symbols.zip`,
|
|
`electron-${version}-mas-x64.zip`,
|
|
`electron-${version}-win32-ia32-pdb.zip`,
|
|
`electron-${version}-win32-ia32-symbols.zip`,
|
|
`electron-${version}-win32-ia32.zip`,
|
|
`electron-${version}-win32-x64-pdb.zip`,
|
|
`electron-${version}-win32-x64-symbols.zip`,
|
|
`electron-${version}-win32-x64.zip`,
|
|
`electron-${version}-win32-arm64-pdb.zip`,
|
|
`electron-${version}-win32-arm64-symbols.zip`,
|
|
`electron-${version}-win32-arm64.zip`,
|
|
`electron-api.json`,
|
|
`electron.d.ts`,
|
|
`hunspell_dictionaries.zip`,
|
|
`ffmpeg-${version}-darwin-x64.zip`,
|
|
`ffmpeg-${version}-linux-arm64.zip`,
|
|
`ffmpeg-${version}-linux-armv7l.zip`,
|
|
`ffmpeg-${version}-linux-ia32.zip`,
|
|
`ffmpeg-${version}-linux-x64.zip`,
|
|
`ffmpeg-${version}-mas-x64.zip`,
|
|
`ffmpeg-${version}-win32-ia32.zip`,
|
|
`ffmpeg-${version}-win32-x64.zip`,
|
|
`ffmpeg-${version}-win32-arm64.zip`,
|
|
`mksnapshot-${version}-darwin-x64.zip`,
|
|
`mksnapshot-${version}-linux-arm64-x64.zip`,
|
|
`mksnapshot-${version}-linux-armv7l-x64.zip`,
|
|
`mksnapshot-${version}-linux-ia32.zip`,
|
|
`mksnapshot-${version}-linux-x64.zip`,
|
|
`mksnapshot-${version}-mas-x64.zip`,
|
|
`mksnapshot-${version}-win32-ia32.zip`,
|
|
`mksnapshot-${version}-win32-x64.zip`,
|
|
`mksnapshot-${version}-win32-arm64-x64.zip`,
|
|
`toolchain-profile-${version}-win32-ia32.zip`,
|
|
`toolchain-profile-${version}-win32-x64.zip`,
|
|
`toolchain-profile-${version}-win32-arm64.zip`
|
|
]
|
|
if (!validatingRelease) {
|
|
patterns.push('SHASUMS256.txt')
|
|
}
|
|
return patterns
|
|
}
|
|
|
|
function s3UrlsForVersion (version) {
|
|
const bucket = `https://gh-contractor-zcbenz.s3.amazonaws.com/`
|
|
const patterns = [
|
|
`${bucket}atom-shell/dist/${version}/iojs-${version}-headers.tar.gz`,
|
|
`${bucket}atom-shell/dist/${version}/iojs-${version}.tar.gz`,
|
|
`${bucket}atom-shell/dist/${version}/node-${version}.tar.gz`,
|
|
`${bucket}atom-shell/dist/${version}/node.lib`,
|
|
`${bucket}atom-shell/dist/${version}/win-x64/iojs.lib`,
|
|
`${bucket}atom-shell/dist/${version}/win-x86/iojs.lib`,
|
|
`${bucket}atom-shell/dist/${version}/x64/node.lib`,
|
|
`${bucket}atom-shell/dist/${version}/SHASUMS.txt`,
|
|
`${bucket}atom-shell/dist/${version}/SHASUMS256.txt`,
|
|
`${bucket}atom-shell/dist/index.json`
|
|
]
|
|
return patterns
|
|
}
|
|
|
|
function runScript (scriptName, scriptArgs, cwd) {
|
|
const scriptCommand = `${scriptName} ${scriptArgs.join(' ')}`
|
|
const scriptOptions = {
|
|
encoding: 'UTF-8'
|
|
}
|
|
if (cwd) scriptOptions.cwd = cwd
|
|
try {
|
|
return execSync(scriptCommand, scriptOptions)
|
|
} catch (err) {
|
|
console.log(`${fail} Error running ${scriptName}`, err)
|
|
process.exit(1)
|
|
}
|
|
}
|
|
|
|
function uploadNodeShasums () {
|
|
console.log('Uploading Node SHASUMS file to S3.')
|
|
const scriptPath = path.join(ELECTRON_DIR, 'script', 'release', 'uploaders', 'upload-node-checksums.py')
|
|
runScript(scriptPath, ['-v', pkgVersion])
|
|
console.log(`${pass} Done uploading Node SHASUMS file to S3.`)
|
|
}
|
|
|
|
function uploadIndexJson () {
|
|
console.log('Uploading index.json to S3.')
|
|
const scriptPath = path.join(ELECTRON_DIR, 'script', 'release', 'uploaders', 'upload-index-json.py')
|
|
runScript(scriptPath, [pkgVersion])
|
|
console.log(`${pass} Done uploading index.json to S3.`)
|
|
}
|
|
|
|
async function createReleaseShasums (release) {
|
|
const fileName = 'SHASUMS256.txt'
|
|
const existingAssets = release.assets.filter(asset => asset.name === fileName)
|
|
if (existingAssets.length > 0) {
|
|
console.log(`${fileName} already exists on GitHub; deleting before creating new file.`)
|
|
await octokit.repos.deleteReleaseAsset({
|
|
owner: 'electron',
|
|
repo: targetRepo,
|
|
asset_id: existingAssets[0].id
|
|
}).catch(err => {
|
|
console.log(`${fail} Error deleting ${fileName} on GitHub:`, err)
|
|
})
|
|
}
|
|
console.log(`Creating and uploading the release ${fileName}.`)
|
|
const scriptPath = path.join(ELECTRON_DIR, 'script', 'release', 'merge-electron-checksums.py')
|
|
const checksums = runScript(scriptPath, ['-v', pkgVersion])
|
|
|
|
console.log(`${pass} Generated release SHASUMS.`)
|
|
const filePath = await saveShaSumFile(checksums, fileName)
|
|
|
|
console.log(`${pass} Created ${fileName} file.`)
|
|
await uploadShasumFile(filePath, fileName, release.id)
|
|
|
|
console.log(`${pass} Successfully uploaded ${fileName} to GitHub.`)
|
|
}
|
|
|
|
async function uploadShasumFile (filePath, fileName, releaseId) {
|
|
const uploadUrl = `https://uploads.github.com/repos/electron/${targetRepo}/releases/${releaseId}/assets{?name,label}`
|
|
return octokit.repos.uploadReleaseAsset({
|
|
url: uploadUrl,
|
|
headers: {
|
|
'content-type': 'text/plain',
|
|
'content-length': fs.statSync(filePath).size
|
|
},
|
|
file: fs.createReadStream(filePath),
|
|
name: fileName
|
|
}).catch(err => {
|
|
console.log(`${fail} Error uploading ${filePath} to GitHub:`, err)
|
|
process.exit(1)
|
|
})
|
|
}
|
|
|
|
function saveShaSumFile (checksums, fileName) {
|
|
return new Promise((resolve, reject) => {
|
|
temp.open(fileName, (err, info) => {
|
|
if (err) {
|
|
console.log(`${fail} Could not create ${fileName} file`)
|
|
process.exit(1)
|
|
} else {
|
|
fs.writeFileSync(info.fd, checksums)
|
|
fs.close(info.fd, (err) => {
|
|
if (err) {
|
|
console.log(`${fail} Could close ${fileName} file`)
|
|
process.exit(1)
|
|
}
|
|
resolve(info.path)
|
|
})
|
|
}
|
|
})
|
|
})
|
|
}
|
|
|
|
async function publishRelease (release) {
|
|
return octokit.repos.updateRelease({
|
|
owner: 'electron',
|
|
repo: targetRepo,
|
|
release_id: release.id,
|
|
tag_name: release.tag_name,
|
|
draft: false
|
|
}).catch(err => {
|
|
console.log(`${fail} Error publishing release:`, err)
|
|
process.exit(1)
|
|
})
|
|
}
|
|
|
|
async function makeRelease (releaseToValidate) {
|
|
if (releaseToValidate) {
|
|
if (releaseToValidate === true) {
|
|
releaseToValidate = pkgVersion
|
|
} else {
|
|
console.log('Release to validate !=== true')
|
|
}
|
|
console.log(`Validating release ${releaseToValidate}`)
|
|
const release = await getDraftRelease(releaseToValidate)
|
|
await validateReleaseAssets(release, true)
|
|
} else {
|
|
let draftRelease = await getDraftRelease()
|
|
uploadNodeShasums()
|
|
uploadIndexJson()
|
|
|
|
await createReleaseShasums(draftRelease)
|
|
|
|
// Fetch latest version of release before verifying
|
|
draftRelease = await getDraftRelease(pkgVersion, true)
|
|
await validateReleaseAssets(draftRelease)
|
|
await publishRelease(draftRelease)
|
|
console.log(`${pass} SUCCESS!!! Release has been published. Please run ` +
|
|
`"npm run publish-to-npm" to publish release to npm.`)
|
|
}
|
|
}
|
|
|
|
async function makeTempDir () {
|
|
return new Promise((resolve, reject) => {
|
|
temp.mkdir('electron-publish', (err, dirPath) => {
|
|
if (err) {
|
|
reject(err)
|
|
} else {
|
|
resolve(dirPath)
|
|
}
|
|
})
|
|
})
|
|
}
|
|
|
|
async function verifyAssets (release) {
|
|
const downloadDir = await makeTempDir()
|
|
|
|
console.log(`Downloading files from GitHub to verify shasums`)
|
|
const shaSumFile = 'SHASUMS256.txt'
|
|
|
|
let filesToCheck = await Promise.all(release.assets.map(async asset => {
|
|
const requestOptions = await octokit.repos.getReleaseAsset.endpoint({
|
|
owner: 'electron',
|
|
repo: targetRepo,
|
|
asset_id: asset.id,
|
|
headers: {
|
|
Accept: 'application/octet-stream'
|
|
}
|
|
})
|
|
|
|
const { url, headers } = requestOptions
|
|
headers.authorization = `token ${process.env.ELECTRON_GITHUB_TOKEN}`
|
|
|
|
const response = await got(url, {
|
|
followRedirect: false,
|
|
method: 'HEAD',
|
|
headers
|
|
})
|
|
|
|
await downloadFiles(response.headers.location, downloadDir, asset.name)
|
|
return asset.name
|
|
})).catch(err => {
|
|
console.log(`${fail} Error downloading files from GitHub`, err)
|
|
process.exit(1)
|
|
})
|
|
|
|
filesToCheck = filesToCheck.filter(fileName => fileName !== shaSumFile)
|
|
let checkerOpts
|
|
await validateChecksums({
|
|
algorithm: 'sha256',
|
|
filesToCheck,
|
|
fileDirectory: downloadDir,
|
|
shaSumFile,
|
|
checkerOpts,
|
|
fileSource: 'GitHub'
|
|
})
|
|
}
|
|
|
|
function downloadFiles (urls, directory, targetName) {
|
|
return new Promise((resolve, reject) => {
|
|
const nuggetOpts = { dir: directory }
|
|
nuggetOpts.quiet = !args.verboseNugget
|
|
if (targetName) nuggetOpts.target = targetName
|
|
|
|
nugget(urls, nuggetOpts, (err) => {
|
|
if (err) {
|
|
reject(err)
|
|
} else {
|
|
console.log(`${pass} all files downloaded successfully!`)
|
|
resolve()
|
|
}
|
|
})
|
|
})
|
|
}
|
|
|
|
async function verifyShasums (urls, isS3) {
|
|
const fileSource = isS3 ? 'S3' : 'GitHub'
|
|
console.log(`Downloading files from ${fileSource} to verify shasums`)
|
|
const downloadDir = await makeTempDir()
|
|
let filesToCheck = []
|
|
try {
|
|
if (!isS3) {
|
|
await downloadFiles(urls, downloadDir)
|
|
filesToCheck = urls.map(url => {
|
|
const currentUrl = new URL(url)
|
|
return path.basename(currentUrl.pathname)
|
|
}).filter(file => file.indexOf('SHASUMS') === -1)
|
|
} else {
|
|
const s3VersionPath = `/atom-shell/dist/${pkgVersion}/`
|
|
await Promise.all(urls.map(async (url) => {
|
|
const currentUrl = new URL(url)
|
|
const dirname = path.dirname(currentUrl.pathname)
|
|
const filename = path.basename(currentUrl.pathname)
|
|
const s3VersionPathIdx = dirname.indexOf(s3VersionPath)
|
|
if (s3VersionPathIdx === -1 || dirname === s3VersionPath) {
|
|
if (s3VersionPathIdx !== -1 && filename.indexof('SHASUMS') === -1) {
|
|
filesToCheck.push(filename)
|
|
}
|
|
await downloadFiles(url, downloadDir)
|
|
} else {
|
|
const subDirectory = dirname.substr(s3VersionPathIdx + s3VersionPath.length)
|
|
const fileDirectory = path.join(downloadDir, subDirectory)
|
|
try {
|
|
fs.statSync(fileDirectory)
|
|
} catch (err) {
|
|
fs.mkdirSync(fileDirectory)
|
|
}
|
|
filesToCheck.push(path.join(subDirectory, filename))
|
|
await downloadFiles(url, fileDirectory)
|
|
}
|
|
}))
|
|
}
|
|
} catch (err) {
|
|
console.log(`${fail} Error downloading files from ${fileSource}`, err)
|
|
process.exit(1)
|
|
}
|
|
console.log(`${pass} Successfully downloaded the files from ${fileSource}.`)
|
|
let checkerOpts
|
|
if (isS3) {
|
|
checkerOpts = { defaultTextEncoding: 'binary' }
|
|
}
|
|
|
|
await validateChecksums({
|
|
algorithm: 'sha256',
|
|
filesToCheck,
|
|
fileDirectory: downloadDir,
|
|
shaSumFile: 'SHASUMS256.txt',
|
|
checkerOpts,
|
|
fileSource
|
|
})
|
|
|
|
if (isS3) {
|
|
await validateChecksums({
|
|
algorithm: 'sha1',
|
|
filesToCheck,
|
|
fileDirectory: downloadDir,
|
|
shaSumFile: 'SHASUMS.txt',
|
|
checkerOpts,
|
|
fileSource
|
|
})
|
|
}
|
|
}
|
|
|
|
async function validateChecksums (validationArgs) {
|
|
console.log(`Validating checksums for files from ${validationArgs.fileSource} ` +
|
|
`against ${validationArgs.shaSumFile}.`)
|
|
const shaSumFilePath = path.join(validationArgs.fileDirectory, validationArgs.shaSumFile)
|
|
const checker = new sumchecker.ChecksumValidator(validationArgs.algorithm,
|
|
shaSumFilePath, validationArgs.checkerOpts)
|
|
await checker.validate(validationArgs.fileDirectory, validationArgs.filesToCheck)
|
|
.catch(err => {
|
|
if (err instanceof sumchecker.ChecksumMismatchError) {
|
|
console.error(`${fail} The checksum of ${err.filename} from ` +
|
|
`${validationArgs.fileSource} did not match the shasum in ` +
|
|
`${validationArgs.shaSumFile}`)
|
|
} else if (err instanceof sumchecker.ChecksumParseError) {
|
|
console.error(`${fail} The checksum file ${validationArgs.shaSumFile} ` +
|
|
`from ${validationArgs.fileSource} could not be parsed.`, err)
|
|
} else if (err instanceof sumchecker.NoChecksumFoundError) {
|
|
console.error(`${fail} The file ${err.filename} from ` +
|
|
`${validationArgs.fileSource} was not in the shasum file ` +
|
|
`${validationArgs.shaSumFile}.`)
|
|
} else {
|
|
console.error(`${fail} Error matching files from ` +
|
|
`${validationArgs.fileSource} shasums in ${validationArgs.shaSumFile}.`, err)
|
|
}
|
|
process.exit(1)
|
|
})
|
|
console.log(`${pass} All files from ${validationArgs.fileSource} match ` +
|
|
`shasums defined in ${validationArgs.shaSumFile}.`)
|
|
}
|
|
|
|
makeRelease(args.validateRelease)
|