Move the debian package_tool into our \tools directory.

The package_tool isn't really part of our product, so moving it to the tools directory.
This commit is contained in:
Eric Erhardt 2016-02-09 13:57:43 -06:00
parent 0b27dba299
commit 7b09f3fcac
28 changed files with 2 additions and 1 deletions

View file

@ -0,0 +1,240 @@
#!/usr/bin/python
#
# Copyright (c) .NET Foundation and contributors. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
#
# Parses debian_config.json and generates appropriate templates
# Where optional defaults exist, they are defined in the template_dict
# of the appropriate generation function
import os
import sys
import json
import datetime
FILE_CHANGELOG = 'changelog'
FILE_CONTROL = 'control'
FILE_COPYRIGHT = 'copyright'
FILE_SYMLINK_FORMAT = '{package_name}.links'
PACKAGE_ROOT_FORMAT = "usr/share/{package_name}"
CHANGELOG_DATE_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
# UTC Timezone for Changelog date
class UTC(datetime.tzinfo):
def utcoffset(self, dt):
return datetime.timedelta(0)
def tzname(self, dt):
return "UTC"
def dst(self, dt):
return datetime.timedelta(0)
# Generation Functions
def generate_and_write_all(config_data, template_dir, output_dir, package_version=None):
try:
changelog_contents = generate_changelog(config_data, template_dir, package_version=package_version)
control_contents = generate_control(config_data, template_dir)
copyright_contents = generate_copyright(config_data, template_dir)
symlink_contents = generate_symlinks(config_data)
except Exception as exc:
print exc
help_and_exit("Error: Generation Failed, check your config file.")
write_file(changelog_contents, output_dir, FILE_CHANGELOG)
write_file(control_contents, output_dir, FILE_CONTROL)
write_file(copyright_contents, output_dir, FILE_COPYRIGHT)
# Symlink File is optional
if symlink_contents:
symlink_filename = get_symlink_filename(config_data)
write_file(symlink_contents, output_dir, symlink_filename)
return
def generate_changelog(config_data, template_dir, package_version=None):
template = get_template(template_dir, FILE_CHANGELOG)
release_data = config_data["release"]
# Allow for Version Override
config_package_version = release_data["package_version"]
if package_version is None:
package_version = config_package_version
template_dict = dict(\
PACKAGE_VERSION=package_version,
PACKAGE_REVISION=release_data["package_revision"],
CHANGELOG_MESSAGE=release_data["changelog_message"],
URGENCY=release_data.get("urgency", "low"),
PACKAGE_NAME=config_data["package_name"],
MAINTAINER_NAME=config_data["maintainer_name"],
MAINTAINER_EMAIL=config_data["maintainer_email"],
DATE=datetime.datetime.now(UTC()).strftime(CHANGELOG_DATE_FORMAT)
)
contents = template.format(**template_dict)
return contents
def generate_control(config_data, template_dir):
template = get_template(template_dir, FILE_CONTROL)
dependency_data = config_data.get("debian_dependencies", None)
dependency_str = get_dependendent_packages_string(dependency_data)
conflict_data = config_data.get("package_conflicts", [])
conflict_str = ', '.join(conflict_data)
# Default to empty dict, so we don't explode on nested optional values
control_data = config_data.get("control", dict())
template_dict = dict(\
SHORT_DESCRIPTION=config_data["short_description"],
LONG_DESCRIPTION=config_data["long_description"],
HOMEPAGE=config_data.get("homepage", ""),
SECTION=control_data.get("section", "misc"),
PRIORITY=control_data.get("priority", "low"),
ARCH=control_data.get("architecture", "all"),
DEPENDENT_PACKAGES=dependency_str,
CONFLICT_PACKAGES=conflict_str,
PACKAGE_NAME=config_data["package_name"],
MAINTAINER_NAME=config_data["maintainer_name"],
MAINTAINER_EMAIL=config_data["maintainer_email"]
)
contents = template.format(**template_dict)
return contents
def generate_copyright(config_data, template_dir):
template = get_template(template_dir, FILE_COPYRIGHT)
license_data = config_data["license"]
template_dict = dict(\
COPYRIGHT_TEXT=config_data["copyright"],
LICENSE_NAME=license_data["type"],
LICENSE_TEXT=license_data["full_text"]
)
contents = template.format(**template_dict)
return contents
def generate_symlinks(config_data):
symlink_entries = []
package_root_path = get_package_root(config_data)
symlink_data = config_data.get("symlinks", dict())
for package_rel_path, symlink_path in symlink_data.iteritems():
package_abs_path = os.path.join(package_root_path, package_rel_path)
symlink_entries.append( '%s %s' % (package_abs_path, symlink_path) )
return '\n'.join(symlink_entries)
# Helper Functions
def get_package_root(config_data):
package_name = config_data["package_name"]
return PACKAGE_ROOT_FORMAT.format(package_name=package_name)
def get_symlink_filename(config_data):
package_name = config_data["package_name"]
return FILE_SYMLINK_FORMAT.format(package_name=package_name)
def get_dependendent_packages_string(debian_dependency_data):
if debian_dependency_data is None:
return ""
dependencies = []
for debian_package_name in debian_dependency_data:
dep_str = debian_package_name
if debian_dependency_data[debian_package_name].get("package_version", None):
debian_package_version = debian_dependency_data[debian_package_name].get("package_version")
dep_str += " (>= %s)" % debian_package_version
dependencies.append(dep_str)
# Leading Comma is important here
return ', ' + ', '.join(dependencies)
def load_json(json_path):
json_data = None
with open(json_path, 'r') as json_file:
json_data = json.load(json_file)
return json_data
def get_template(template_dir, name):
path = os.path.join(template_dir, name)
template_contents = None
with open(path, 'r') as template_file:
template_contents = template_file.read()
return template_contents
def write_file(contents, output_dir, name):
path = os.path.join(output_dir, name)
with open(path, 'w') as out_file:
out_file.write(contents)
return
# Tool Functions
def help_and_exit(msg):
print msg
sys.exit(1)
def print_usage():
print "Usage: config_template_generator.py [config file path] [template directory path] [output directory] (package version)"
def parse_and_validate_args():
if len(sys.argv) < 4:
print_usage()
help_and_exit("Error: Invalid Arguments")
config_path = sys.argv[1]
template_dir = sys.argv[2]
output_dir = sys.argv[3]
version_override = None
if len(sys.argv) >= 5:
version_override = sys.argv[4]
if not os.path.isfile(config_path):
help_and_exit("Error: Invalid config file path")
if not os.path.isdir(template_dir):
help_and_exit("Error: Invalid template directory path")
if not os.path.isdir(output_dir):
help_and_exit("Error: Invalid output directory path")
return (config_path, template_dir, output_dir, version_override)
def execute():
config_path, template_dir, output_dir, version_override = parse_and_validate_args()
config_data = load_json(config_path)
generate_and_write_all(config_data, template_dir, output_dir, package_version=version_override)
if __name__ == "__main__":
execute()

View file

@ -0,0 +1,165 @@
#
# Copyright (c) .NET Foundation and contributors. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
#
# This file is not intended to be executed directly
# Import these functions using source
#
# Relies on these environment variables:
# PACKAGE_SOURCE_DIR :: Package Source Staging Directory
# INSTALL_ROOT :: Absolute path of package installation root
# write_debian_install_file
# Summary: Writes the contents of the "install_placement" array to the debian/install
# This array is populated by calls to the "add_system_file_placement" function
# Usage: write_debian_install_file
write_debian_install_file(){
# Remove any existing install file, we need to overwrite it
rm -f ${PACKAGE_SOURCE_DIR}/debian/install
for i in "${install_placement[@]}"
do
echo "${i}" >> "${PACKAGE_SOURCE_DIR}/debian/install"
done
}
# add_system_file_placement
# Summary: Registers a file placement on the filesystem from the package by populating the "install_placement" array
# Usage: add_system_file_placement {local path of file in package} {absolute path of directory to place file in}
add_system_file_placement(){
#Initialize placement_index variable
if [[ -z "$placement_index" ]]; then
placement_index=0
fi
install_placement[${placement_index}]="${1} ${2}"
placement_index=$((${placement_index}+1))
}
# add_system_dir_placement
# Summary: Registers a directory placement on the post-installation package from an in-package path
add_system_dir_placement(){
in_package_dir=$1
abs_installation_dir=$2
dir_files=( $(_get_files_in_dir_tree $PACKAGE_SOURCE_DIR/$in_package_dir) )
# If in_package_dir isn't empty include a slash
if [ ! -z "$in_package_dir" ]; then
in_package_dir="${in_package_dir}/"
fi
for rel_filepath in ${dir_files[@]}
do
local parent_path=$(dirname $rel_filepath)
# If there is no parent, parent_path = "."
if [[ "$parent_path" == "." ]]; then
add_system_file_placement "${in_package_dir}${rel_filepath}" "${abs_installation_dir}"
else
add_system_file_placement "${in_package_dir}${rel_filepath}" "${abs_installation_dir}/${parent_path}"
fi
done
}
# add_file_to_install
# Summary: Adds a file from the local filesystem to the package and installs it rooted at INSTALL_ROOT
# Usage: add_install_file {relative path to local file} {relative path to INSTALL_ROOT to place file}
add_file_to_install(){
copy_from_file=$1
rel_install_path=$2
local filename=$(basename $copy_from_file)
local parent_dir=$(dirname $copy_from_file)
# Create Relative Copy From Path
rel_copy_from_file=${copy_from_file#$parent_dir/}
# Delete any existing file and ensure path exists
rm -f ./${PACKAGE_SOURCE_DIR}/${rel_install_path}/${filename}
mkdir -p ./${PACKAGE_SOURCE_DIR}/${rel_install_path}
dir_files=( "$rel_copy_from_file" )
_copy_files_to_package $parent_dir $rel_install_path "${dir_files[@]}"
add_system_file_placement "${rel_install_path}/${filename}" "${INSTALL_ROOT}/$rel_install_path"
}
# add_dir_to_install
# Summary: Adds contents of a directory on the local filesystem to the package and installs them rooted at INSTALL_ROOT
# Note: Does not install the directory passed, only its contents
# Usage: add_dir_to_install {relative path of directory to copy} {relative path to INSTALL_ROOT to place directory tree}
add_dir_to_install(){
copy_from_dir=$1
rel_install_path=$2
# Delete and Create any existing directory
mkdir -p ${PACKAGE_SOURCE_DIR}/${rel_install_path}
dir_files=( $(_get_files_in_dir_tree $copy_from_dir) )
_copy_files_to_package "$copy_from_dir" "$rel_install_path" "${dir_files[@]}"
add_system_dir_placement "$rel_install_path" "${INSTALL_ROOT}/$rel_install_path"
}
# Usage: _copy_files_to_package {local files root directory} {relative directory in package to copy to} "${filepath_array[@]}"
# Note: The specific syntax on the parameter shows how to pass an array
_copy_files_to_package(){
local_root_dir=$1
package_dest_dir=$2
# Consume the remaining input as an array
shift; shift;
rel_filepath_list=( $@ )
for rel_filepath in ${rel_filepath_list[@]}
do
local parent_dir=$(dirname $rel_filepath)
local filename=$(basename $rel_filepath)
mkdir -p ${PACKAGE_SOURCE_DIR}/${package_dest_dir}/${parent_dir}
# Ignore $parent_dir if it there isn't one
if [[ "$parent_dir" == "." ]]; then
cp "${local_root_dir}/${rel_filepath}" "${PACKAGE_SOURCE_DIR}/${package_dest_dir}"
else
cp "${local_root_dir}/${rel_filepath}" "${PACKAGE_SOURCE_DIR}/${package_dest_dir}/${parent_dir}"
fi
done
}
# Usage: _get_files_in_dir_tree {path of directory}
_get_files_in_dir_tree(){
root_dir=$1
# Use Globstar expansion to enumerate all directories and files in the tree
shopt -s globstar
dir_tree_list=( "${root_dir}/"** )
# Build a new array with only the Files contained in $dir_tree_list
local index=0
for file_path in "${dir_tree_list[@]}"
do
if [ -f $file_path ]; then
dir_tree_file_list[${index}]=$file_path
index=$(($index+1))
fi
done
# Remove $root_dir prefix from each path in dir_tree_file_list
# This is confusing syntax, so here's a reference link (Substring Removal)
# http://wiki.bash-hackers.org/syntax/pe
dir_tree_file_list=( "${dir_tree_file_list[@]#${root_dir}/}" )
# Echo is the return mechanism
echo "${dir_tree_file_list[@]}"
}

View file

@ -0,0 +1,64 @@
#!/usr/bin/python
#
# Copyright (c) .NET Foundation and contributors. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
#
# Extract Json Value
#
# Very simple tool to ease extracting json values from the cmd line.
import os
import sys
import json
def print_usage():
print """
Usage: extract_json_value.py [json file path] [key of value to extract]
For nested keys, use . separator
"""
def help_and_exit(msg=None):
print msg
print_usage()
sys.exit(1)
def parse_and_validate_args():
if len(sys.argv) < 3:
help_and_exit(msg="Error: Invalid Args")
json_path = sys.argv[1]
json_key = sys.argv[2]
if not os.path.isfile(json_path):
help_and_exit("Error: Invalid json file path")
return json_path, json_key
def extract_key(json_path, json_key):
json_data = None
with open(json_path, 'r') as json_file:
json_data = json.load(json_file)
nested_keys = json_key.split('.')
json_context = json_data
for key in nested_keys:
json_context = json_context.get(key, None)
if json_context is None:
help_and_exit("Error: Invalid json key")
return str(json_context)
def execute():
json_path, json_key = parse_and_validate_args()
value = extract_key(json_path, json_key)
return value
if __name__ == "__main__":
print execute()

View file

@ -0,0 +1,303 @@
#!/usr/bin/python
#
# Copyright (c) .NET Foundation and contributors. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
#
# manpage_generator
# Converts top level docs.json format command info to
# nroff manpage format. Done in python for easy json parsing.
#
# Usage: argv[1] = path to docs.json; argv[2] = output path
import sys
import os
import json
import datetime
SECTION_SEPARATOR = "\n.P \n"
MANPAGE_EXTENSION = ".1"
# For now this is a magic number
# See https://www.debian.org/doc/manuals/maint-guide/dother.en.html#manpage
SECTION_NUMBER = 1
def generate_man_pages(doc_path, output_dir):
with open(doc_path) as doc_file:
doc_json = None
try:
doc_json = json.load(doc_file)
except:
raise Exception("Failed to load json file. Check formatting.")
tools = doc_json.get("tools", None)
if tools is None:
raise Exception("No tool sections in doc.json")
for tool_name in tools:
tool_data = tools[tool_name]
man_page_content = generate_man_page(tool_name, tool_data)
man_page_path = get_output_path(tool_name, output_dir)
write_man_page(man_page_path, man_page_content)
def get_output_path(toolname, output_dir):
out_filename = toolname + MANPAGE_EXTENSION
return os.path.join(output_dir, out_filename)
def write_man_page(path, content):
with open(path, 'w') as man_file:
man_file.write(content)
#Build Fails without a final newline
man_file.write('\n')
def generate_man_page(tool_name, tool_data):
sections = [
generate_header_section(tool_name, tool_data),
generate_name_section(tool_name, tool_data),
generate_synopsis_section(tool_name, tool_data),
generate_description_section(tool_name, tool_data),
generate_options_section(tool_name, tool_data),
generate_author_section(tool_name, tool_data),
generate_copyright_section(tool_name, tool_data)
]
return SECTION_SEPARATOR.join(sections)
def generate_header_section(tool_name, tool_data):#
roff_text_builder = []
header_format = ".TH {program_name} {section_number} {center_footer} {left_footer} {center_header}"
today = datetime.date.today()
today_string = today.strftime("%B %d, %Y")
format_args = {
"program_name" : tool_name,
"section_number" : SECTION_NUMBER,
"center_footer" : "", # Omitted
"left_footer" : "", # Omitted
"center_header" : "" # Omitted
}
roff_text_builder.append(header_format.format(**format_args))
return SECTION_SEPARATOR.join(roff_text_builder)
def generate_name_section(tool_name, tool_data):#
roff_text_builder = []
roff_text_builder.append(".SH NAME")
tool_short_description = tool_data.get("short_description", "")
name_format = ".B {program_name} - {short_description}"
name_format_args = {
"program_name": tool_name,
"short_description" : tool_short_description
}
roff_text_builder.append(name_format.format(**name_format_args))
return SECTION_SEPARATOR.join(roff_text_builder)
def generate_synopsis_section(tool_name, tool_data):#
roff_text_builder = []
roff_text_builder.append(".SH SYNOPSIS")
synopsis_format = '.B {program_name} {command_name} \n.RI {options} " "\n.I "{argument_list_name}"'
tool_commands = tool_data.get("commands", [])
for command_name in tool_commands:
command_data = tool_commands[command_name]
# Default options to empty list so the loop doesn't blow up
options = command_data.get("options", [])
argument_list = command_data.get("argumentlist", None)
# Construct Option Strings
option_string_list = []
argument_list_name = ""
for option_name in options:
option_data = options[option_name]
specifier_short = option_data.get("short", None)
specifier_long = option_data.get("long", None)
parameter = option_data.get("parameter", None)
option_string = _option_string_helper(specifier_short, specifier_long, parameter)
option_string_list.append(option_string)
# Populate Argument List Name
if argument_list:
argument_list_name = argument_list.get("name", "")
cmd_format_args = {
'program_name' : tool_name,
'command_name' : command_name,
'options' : '" "'.join(option_string_list),
'argument_list_name' : argument_list_name
}
cmd_string = synopsis_format.format(**cmd_format_args)
roff_text_builder.append(cmd_string)
return SECTION_SEPARATOR.join(roff_text_builder)
def generate_description_section(tool_name, tool_data):#
roff_text_builder = []
roff_text_builder.append(".SH DESCRIPTION")
# Tool Description
long_description = tool_data.get("long_description", "")
roff_text_builder.append(".PP {0}".format(long_description))
# Command Descriptions
cmd_description_format = ".B {program_name} {command_name}\n{command_description}"
tool_commands = tool_data.get("commands", [])
for command_name in tool_commands:
command_data = tool_commands[command_name]
command_description = command_data.get("description", "")
format_args = {
"program_name" : tool_name,
"command_name" : command_name,
"command_description" : command_description
}
cmd_string = cmd_description_format.format(**format_args)
roff_text_builder.append(cmd_string)
return SECTION_SEPARATOR.join(roff_text_builder)
def generate_options_section(tool_name, tool_data):#
roff_text_builder = []
roff_text_builder.append(".SH OPTIONS")
options_format = '.TP\n.B {option_specifiers}\n{option_description}'
tool_commands = tool_data.get("commands", [])
for command_name in tool_commands:
command_data = tool_commands[command_name]
# Default to empty list so the loop doesn't blow up
options = command_data.get("options", [])
for option_name in options:
option_data = options[option_name]
specifier_short = option_data.get("short", None)
specifier_long = option_data.get("long", None)
parameter = option_data.get("parameter", None)
description = option_data.get("description", "")
option_specifiers_string = _option_string_helper(specifier_short,
specifier_long,
parameter,
include_brackets = False,
delimiter=' ", " ')
format_args = {
"option_specifiers": option_specifiers_string,
"option_description" : description
}
roff_text_builder.append(options_format.format(**format_args))
return SECTION_SEPARATOR.join(roff_text_builder)
def generate_author_section(tool_name, tool_data):#
roff_text_builder = []
roff_text_builder.append(".SH AUTHOR")
author_format = '.B "{author_name}" " " \n.RI ( "{author_email}" )'
author_name = tool_data.get("author", "")
author_email = tool_data.get("author_email", "")
format_args = {
"author_name" : author_name,
"author_email" : author_email
}
roff_text_builder.append(author_format.format(**format_args))
return SECTION_SEPARATOR.join(roff_text_builder)
def generate_copyright_section(tool_name, tool_data):#
roff_text_builder = []
roff_text_builder.append(".SH COPYRIGHT")
copyright_data = tool_data.get("copyright")
roff_text_builder.append('.B "{0}"'.format(copyright_data))
return SECTION_SEPARATOR.join(roff_text_builder)
def _option_string_helper(specifier_short, specifier_long, parameter, include_brackets = True, delimiter = " | "):
option_string = ""
if include_brackets:
option_string = " [ "
if specifier_short:
option_string += ' "{0}" '.format(specifier_short)
if specifier_short and specifier_long:
option_string += delimiter
if specifier_long:
option_string += ' "{0}" '.format(specifier_long)
if parameter:
option_string += ' " " '
option_string += ' "{0}" '.format(parameter)
if include_brackets:
option_string += " ] "
return option_string
def print_usage():
print "Usage: argv[1] = path to docs.json; argv[2] = output path"
print "Example: manpage_generator.py ../docs.json ./dotnet-1.0/debian"
def parse_args():
doc_path = sys.argv[1]
output_dir = sys.argv[2]
return (doc_path, output_dir)
def validate_args(doc_path, output_dir):
if not os.path.isfile(doc_path):
raise Exception("Docs.json path is not valid.")
if not os.path.isdir(output_dir):
raise Exception("Output Directory Path is not valid.")
def execute_command_line():
try:
doc_path, output_dir = parse_args()
validate_args(doc_path, output_dir)
generate_man_pages(doc_path, output_dir)
except Exception as exc:
print "Error: ", exc
print_usage()
if __name__ == "__main__":
execute_command_line()