restructure the output layout

also reorganize the scripts folder
This commit is contained in:
Andrew Stanton-Nurse 2015-11-10 17:30:01 -08:00
parent df3a5fba7a
commit 4cc15b1246
61 changed files with 926 additions and 3213 deletions

145
scripts/publish/publish.ps1 Normal file
View file

@ -0,0 +1,145 @@
param(
[string]$file = $(throw "Specify the full path to the file to be uploaded")
)
function CheckRequiredVariables
{
if([string]::IsNullOrEmpty($env:DOTNET_BUILD_VERSION))
{
return $false
}
# this variable is set by the CI system
if([string]::IsNullOrEmpty($env:SASTOKEN))
{
return $false
}
# this variable is set by the CI system
if([string]::IsNullOrEmpty($env:STORAGE_ACCOUNT))
{
return $false
}
# this variable is set by the CI system
if([string]::IsNullOrEmpty($env:STORAGE_CONTAINER))
{
return $false
}
# this variable is set by the CI system
if([string]::IsNullOrEmpty($env:CHANNEL))
{
return $false
}
return $true
}
function UploadFile($Upload_URI, $Uploadfile)
{
Write-Host "Uploading $Uploadfile to dotnet feed to.."
$statusCode = (Invoke-WebRequest -URI "$Upload_URI" -Method PUT -Headers @{"x-ms-blob-type"="BlockBlob"; "x-ms-date"="2015-10-23";"x-ms-version"="2013-08-15"} -InFile $Uploadfile).StatusCode
if($statusCode -eq 201)
{
Write-Host "Successfully uploaded $Uploadfile to dotnet feed."
return $true
}
else
{
Write-Host "Failed to upload $Uploadfile to dotnet feed."
return $false
}
}
function UploadBinaries($zipFile)
{
$result = -1
$fileName = [System.IO.Path]::GetFileName($zipFile)
$Upload_URI = "https://$env:STORAGE_ACCOUNT.blob.core.windows.net/$env:STORAGE_CONTAINER/$env:CHANNEL/Binaries/$env:DOTNET_BUILD_VERSION/$fileName$env:SASTOKEN"
if(-Not (UploadFile $Upload_URI $zipFile))
{
return -1
}
Write-Host "Updating the latest dotnet binaries for windows.."
$Upload_URI_Latest = "https://$env:STORAGE_ACCOUNT.blob.core.windows.net/$env:STORAGE_CONTAINER/$env:CHANNEL/Binaries/Latest/dotnet-win-x64.latest.zip$env:SASTOKEN"
if(-Not (UploadFile $Upload_URI_Latest $zipFile))
{
return -1
}
# update the index file too
$indexContent = "Binaries/$env:DOTNET_BUILD_VERSION/$fileName"
$indexFile = "$env:TEMP\latest.win.index"
$indexContent | Out-File -FilePath $indexFile
# upload the index file
$Upload_URI = "https://$env:STORAGE_ACCOUNT.blob.core.windows.net/$env:STORAGE_CONTAINER/$env:CHANNEL/dnvm/latest.win.index$env:SASTOKEN"
if(-Not (UploadFile $Upload_URI $indexFile))
{
return -1
}
# update the version file
$versionFile = Convert-Path $PSScriptRoot\..\..\artifacts\win7-x64\stage2\.version
$Version_URI = "https://$env:STORAGE_ACCOUNT.blob.core.windows.net/$env:STORAGE_CONTAINER/$env:CHANNEL/dnvm/latest.win.version$env:SASTOKEN"
if(-Not (UploadFile $Version_URI $versionFile))
{
return -1
}
return 0
}
function UploadInstallers($msiFile)
{
$fileName = [System.IO.Path]::GetFileName($msiFile)
$Upload_URI = "https://$env:STORAGE_ACCOUNT.blob.core.windows.net/$env:STORAGE_CONTAINER/$env:CHANNEL/Installers/$env:DOTNET_BUILD_VERSION/$fileName$env:SASTOKEN"
if(-Not (UploadFile $Upload_URI $msiFile))
{
return -1
}
Write-Host "Updating the latest dotnet installer for windows.."
$Upload_URI_Latest = "https://$env:STORAGE_ACCOUNT.blob.core.windows.net/$env:STORAGE_CONTAINER/$env:CHANNEL/Installers/Latest/dotnet-win-x64.latest.msi$env:SASTOKEN"
if(-Not (UploadFile $Upload_URI_Latest $msiFile))
{
return -1
}
return 0
}
if(!(CheckRequiredVariables))
{
# fail silently if the required variables are not available for publishing the file
exit 0
}
if(![System.IO.File]::Exists($file))
{
throw "$file not found"
}
$result = $false
if([System.IO.Path]::GetExtension($file).ToLower() -eq ".zip")
{
$result = UploadBinaries $file
}
elseif([System.IO.Path]::GetExtension($file).ToLower() -eq ".msi")
{
$result = UploadInstallers $file
}
exit $result

218
scripts/publish/publish.sh Executable file
View file

@ -0,0 +1,218 @@
#!/bin/bash
#
# Usage: publish.sh [file to be uploaded]
#
# Environment Dependencies:
# $STORAGE_CONTAINER
# $STORAGE_ACCOUNT
# $SASTOKEN
# $REPO_ID
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
REPOROOT="$( cd -P "$SCRIPT_DIR/../.." && pwd )"
source "$SCRIPT_DIR/../_common.sh"
UPLOAD_FILE=$1
UPLOAD_JSON_FILE="package_upload.json"
header "Publishing package"
execute(){
if ! validate_env_variables; then
# fail silently if the required variables are not available for publishing the file.
exit 0
fi
if [[ ! -f "$UPLOAD_FILE" ]]; then
error "\"$UPLOAD_FILE\" file does not exist"
exit 1
fi
if [[ $UPLOAD_FILE == *.deb || $UPLOAD_FILE == *.pkg ]]; then
upload_installers_to_blob_storage $UPLOAD_FILE
result=$?
elif [[ $UPLOAD_FILE == *.tar.gz ]]; then
upload_binaries_to_blob_storage $UPLOAD_FILE
result=$?
fi
exit $result
}
validate_env_variables(){
local ret=0
if [[ -z "$DOTNET_BUILD_VERSION" ]]; then
warning "DOTNET_BUILD_VERSION environment variable not set"
ret=1
fi
if [[ -z "$SASTOKEN" ]]; then
warning "SASTOKEN environment variable not set"
ret=1
fi
if [[ -z "$STORAGE_ACCOUNT" ]]; then
warning "STORAGE_ACCOUNT environment variable not set"
ret=1
fi
if [[ -z "$STORAGE_CONTAINER" ]]; then
warning "STORAGE_CONTAINER environment variable not set"
ret=1
fi
if [[ -z "$CHANNEL" ]]; then
warning "CHANNEL environment variable not set"
ret=1
fi
if [[ -z "$CONNECTION_STRING" ]]; then
warning "CONNECTION_STRING environment variable not set"
ret=1
fi
return $ret
}
upload_file_to_blob_storage_azure_cli(){
local blob=$1
local file=$2
header "Uploading $file to blob storage"
# use azure cli to upload to blob storage. We cannot use curl to do this becuase azure has a max limit of 64mb that can be uploaded using REST
# statusCode=$(curl -s -w "%{http_code}" -L -H "x-ms-blob-type: BlockBlob" -H "x-ms-date: 2015-10-21" -H "x-ms-version: 2013-08-15" $upload_URL -T $file)
azure storage blob upload --quiet --container $STORAGE_CONTAINER --blob $blob --blobtype block --connection-string "$CONNECTION_STRING" --file $file
result=$?
if [ "$result" -eq "0" ]; then
info "successfully uploaded $filename to blob storage."
return 0
else
error "uploading the $filename to blob storage - $statusCode"
return 1
fi
}
update_file_in_blob_storage(){
local update_URL=$1
local file=$2
local filecontent=$3
header "Updating $file in blob storage"
statusCode=$(curl -s -w "%{http_code}" -L -H "x-ms-blob-type: BlockBlob" -H "x-ms-date: 2015-10-21" -H "x-ms-version: 2013-08-15" -H "Content-Type: text/plain" $update_URL --data-binary $filecontent --request PUT )
if [ "$statusCode" -eq "201" ]; then
info "successfully updated $file in blob storage."
return 0
else
error "updating the $file in blob storage - $statusCode"
return 1
fi
}
upload_binaries_to_blob_storage(){
local tarfile=$1
local filename=$(basename $tarfile)
local blob="$CHANNEL/Binaries/$DOTNET_BUILD_VERSION/$filename"
if ! upload_file_to_blob_storage_azure_cli $blob $tarfile; then
return 1
fi
# create the latest blob
echo "Updating the latest dotnet binaries.."
local latestblob="$CHANNEL/Binaries/Latest/dotnet-$OSNAME-x64.latest.tar.gz"
if ! upload_file_to_blob_storage_azure_cli $latestblob $tarfile; then
return 1
fi
# update the index file
local indexContent="Binaries/$DOTNET_BUILD_VERSION/$filename"
local indexfile="latest.$OSNAME.index"
local index_URL="https://$STORAGE_ACCOUNT.blob.core.windows.net/$STORAGE_CONTAINER/$CHANNEL/dnvm/$indexfile$SASTOKEN"
update_file_in_blob_storage $index_URL $indexfile $indexContent
# update the version file
# the "@" prefix tells curl to upload the content of the file
local versionContent="@$REPOROOT/artifacts/$RID/stage2/.version"
local versionfile="latest.$OSNAME.version"
local version_URL="https://$STORAGE_ACCOUNT.blob.core.windows.net/$STORAGE_CONTAINER/$CHANNEL/dnvm/$versionfile$SASTOKEN"
update_file_in_blob_storage $version_URL $versionfile $versionContent
return $?
}
upload_installers_to_blob_storage(){
local installfile=$1
local filename=$(basename $installfile)
local blob="$CHANNEL/Installers/$DOTNET_BUILD_VERSION/$filename"
if ! upload_file_to_blob_storage_azure_cli $blob $installfile; then
return 1
fi
# create the latest blob
echo "Updating the latest dotnet installer.."
local extension="${filename##*.}"
local latestblob="$CHANNEL/Installers/Latest/dotnet-$OSNAME-x64.latest.$extension"
if ! upload_file_to_blob_storage_azure_cli $latestblob $installfile; then
return 1
fi
# debain packages need to be uploaded to the PPA feed too
if [[ $installfile == *.deb ]]; then
DEB_FILE=$installfile
UPLOAD_URL="https://$STORAGE_ACCOUNT.blob.core.windows.net/$STORAGE_CONTAINER/$blob"
generate_repoclient_json
call_repo_client
fi
return 0
}
generate_repoclient_json(){
# Clean any existing json file
rm -f $SCRIPT_DIR/$UPLOAD_JSON_FILE
echo "{" >> "$SCRIPT_DIR/$UPLOAD_JSON_FILE"
echo " \"name\":\"$(_get_package_name)\"," >> "$SCRIPT_DIR/$UPLOAD_JSON_FILE"
echo " \"version\":\"$(_get_package_version)\"," >> "$SCRIPT_DIR/$UPLOAD_JSON_FILE"
echo " \"repositoryId\":\"$REPO_ID\"," >> "$SCRIPT_DIR/$UPLOAD_JSON_FILE"
echo " \"sourceUrl\":\"$UPLOAD_URL\"" >> "$SCRIPT_DIR/$UPLOAD_JSON_FILE"
echo "}" >> "$SCRIPT_DIR/$UPLOAD_JSON_FILE"
}
call_repo_client(){
$SCRIPT_DIR/repoapi_client.sh -addpkg $SCRIPT_DIR/$UPLOAD_JSON_FILE
}
# Extract the package name from the .deb filename
_get_package_name(){
local deb_filename=$(basename $DEB_FILE)
local package_name=${deb_filename%%_*}
echo $package_name
}
# Extract the package version from the .deb filename
_get_package_version(){
local deb_filename=$(basename $DEB_FILE)
local package_version=${deb_filename#*_}
package_version=${package_version%-*}
echo $package_version
}
execute

View file

@ -0,0 +1,189 @@
#!/bin/bash
# This is a VERY basic script for Create/Delete operations on repos and packages
#
# Environment Dependencies:
# $REPO_SERVER
# $REPO_USER
# $REPO_PASS
cmd=$1
urls=urls.txt
defaultPackageFile=new_package.json
defaultRepoFile=new_repo.json
repositoryId=$REPO_ID
server=$REPO_SERVER
user=$REPO_USER
pass=$REPO_PASS
protocol=https
port=443
baseurl="$protocol://$user:$pass@$server:$port"
echo $baseurl
function BailIf
{
if [ $1 -ne 0 ]; then
echo "Failure occurred communicating with $server"
exit 1
fi
}
# List packages, using $1 as a regex to filter results
function ListPackages
{
curl -k "$baseurl/v1/packages" | sed 's/{/\n{/g' | egrep "$1" | sed 's/,/,\n/g' | sed 's/^"/\t"/g'
echo ""
}
# Create a new Repo using the specified JSON file
function AddRepo
{
repoFile=$1
if [ -z $repoFile ]; then
echo "Error: Must specify a JSON-formatted file. Reference $defaultRepoFile.template"
exit 1
fi
if [ ! -f $repoFile ]; then
echo "Error: Cannot create repo - $repoFile does not exist"
exit 1
fi
packageUrl=$(grep "url" $repoFile | head -n 1 | awk '{print $2}' | tr -d ',')
echo "Creating new repo on $server [$packageUrl]"
curl -i -k "$baseurl/v1/repositories" --data @./$repoFile -H "Content-Type: application/json"
BailIf $?
echo ""
}
# Upload a single package using the specified JSON file
function AddPackage
{
packageFile=$1
if [ -z $packageFile ]; then
echo "Error: Must specify a JSON-formatted file. Reference $defaultPackageFile.template"
exit 1
fi
if [ ! -f $packageFile ]; then
echo "Error: Cannot add package - $packageFile does not exist"
exit 1
fi
packageUrl=$(grep "sourceUrl" $packageFile | head -n 1 | awk '{print $2}')
echo "Adding package to $server [$packageUrl]"
curl -i -k "$baseurl/v1/packages" --data @$packageFile -H "Content-Type: application/json"
BailIf $?
echo ""
}
# Upload a single package by dynamically creating a JSON file using a provided URL
function AddPackageByUrl
{
# Parse URL
url=$(echo "$1")
if [ -z $url ]; then
return
fi
escapedUrl=$(echo "$url" | sed 's/\//\\\//g')
set -- "$1"
oldIFS=$IFS
IFS="/"; declare -a splitUrl=($*)
index=${#splitUrl[@]}
let "index -= 1"
filename=${splitUrl[$index]}
set -- "$filename"
IFS="_"; declare -a splitFile=($*)
IFS=$oldIFS
pkgName=${splitFile[0]}
pkgVer=${splitFile[1]}
if [ -z $pkgName ] || [ -z $pkgVer ]; then
echo "ERROR parsing $url"
return
fi
# Create Package .json file
cp $defaultPackageFile.template $defaultPackageFile
sed -i "s/PACKAGENAME/$pkgName/g" $defaultPackageFile
sed -i "s/PACKAGEVERSION/$pkgVer/g" $defaultPackageFile
sed -i "s/PACKAGEURL/$escapedUrl/g" $defaultPackageFile
sed -i "s/REPOSITORYID/$repositoryId/g" $defaultPackageFile
# Test that URL is ok
wget -q --spider "$url"
if [[ $? -eq 0 ]]; then
echo "Ready to upload $pkgName [$pkgVer]"
else
echo "ERROR testing URL $url"
return
fi
# Perform Upload
AddPackage $defaultPackageFile
# Cleanup
# rm $defaultPackageFile
}
# Upload multiple packages by reading urls line-by-line from the specified file
function AddPackages
{
urlFile=$1
if [ -z $urlFile ]; then
echo "Error: Must specify a flat text file containing one or more URLs"
exit 1
fi
if [ ! -f $urlFile ]; then
echo "Error: Cannot add packages. File $urlFile does not exist"
exit 1
fi
for url in $(cat $urlFile); do
AddPackageByUrl "$url"
sleep 5
done
}
# Delete the specified repo
function DeleteRepo
{
repoId=$1
if [ -z $repoId ]; then
echo "Error: Please specify repository ID. Run -listrepos for a list of IDs"
exit 1
fi
curl -I -k -X DELETE "$baseurl/v1/repositories/$repoId"
BailIf $?
}
# Delete the specified package
function DeletePackage
{
packageId=$1
if [ -z $packageId ]; then
echo "Error: Please specify package ID. Run -listpkgs for a list of IDs"
exit 1
fi
echo Removing pkgId $packageId from repo $repositoryId
curl -I -k -X DELETE "$baseurl/v1/packages/$packageId"
BailIf $?
}
if [[ "$1" == "-listrepos" ]]; then
echo "Fetching repo list from $server..."
curl -k "$baseurl/v1/repositories" | sed 's/,/,\n/g' | sed 's/^"/\t"/g'
echo ""
elif [[ "$1" == "-listpkgs" ]]; then
echo "Fetching package list from $server"
ListPackages $2
elif [[ "$1" == "-addrepo" ]]; then
AddRepo $2
elif [[ "$1" == "-addpkg" ]]; then
AddPackage $2
elif [[ "$1" == "-addpkgs" ]]; then
AddPackages $2
elif [[ "$1" == "-delrepo" ]]; then
DeleteRepo $2
elif [[ "$1" == "-delpkg" ]]; then
DeletePackage $2
else
echo "USAGE: ./repotool.sh -OPTION"
echo "-listrepos: Gather a list of repos"
echo "-listpkgs: Gather a list of packages"
echo "-addrepo [FILENAME] : Create a new repo using the specified JSON file"
echo "-addpkg [FILENAME] : Add package to repo using the specified JSON file"
echo "-addpkgs [FILENAME] : Add packages to repo using urls contained in FILENAME"
echo "-delrepo REPOID : Delete the specified repo by ID"
echo "-delpkg PKGID : Delete the specified package by ID"
fi