From eaa6b1b274a5b74ea2859a78658fdbc3a0bbb2e1 Mon Sep 17 00:00:00 2001 From: Chelsea Mafrica Date: Tue, 21 Nov 2023 17:41:49 -0800 Subject: [PATCH 1/6] tests: move static checks and dependencies from tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move static checks scripts and dependencies from tests to kata-containers repo. Fixes #8187 Signed-off-by: Amulyam24 Signed-off-by: Archana Shinde Signed-off-by: Bin Liu Signed-off-by: Carlos Venegas Signed-off-by: Chao Wu Signed-off-by: Chelsea Mafrica Signed-off-by: Dan Middleton Signed-off-by: David Gibson Signed-off-by: Derek Lee Signed-off-by: Dov Murik Signed-off-by: Eric Ernst Signed-off-by: Fabiano Fidêncio Signed-off-by: Fupan Li Signed-off-by: Gabriela Cervantes Signed-off-by: Ganesh Maharaj Mahalingam Signed-off-by: Graham Whaley Signed-off-by: Jakob Naucke Signed-off-by: James O. D. Hunt Signed-off-by: Jeremi Piotrowski Signed-off-by: Jon Olson Signed-off-by: Jose Carlos Venegas Munoz Signed-off-by: Julio Montes Signed-off-by: Liu Jiang Signed-off-by: Manohar Castelino Signed-off-by: Marco Vedovati Signed-off-by: Nitesh Konkar Signed-off-by: Peng Tao Signed-off-by: Salvador Fuentes Signed-off-by: Sebastien Boeuf Signed-off-by: Shiming Zhang Signed-off-by: Snir Sheriber Signed-off-by: stevenhorsman Signed-off-by: Wainer dos Santos Moschetta Signed-off-by: Xu Wang Signed-off-by: Yang Bo Signed-off-by: Zvonko Kaiser --- tests/.gitignore | 2 + tests/.golangci.yml | 33 + tests/go.mod | 27 + tests/kata-doc-to-script.sh | 229 ++++++ tests/static-checks.sh | 1334 +++++++++++++++++++++++++++++++++++ 5 files changed, 1625 insertions(+) create mode 100644 tests/.golangci.yml create mode 100644 tests/go.mod create mode 100755 tests/kata-doc-to-script.sh create mode 100755 tests/static-checks.sh diff --git a/tests/.gitignore b/tests/.gitignore index 122d160715..38a895f0e4 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1 +1,3 @@ +cmd/check-markdown/kata-check-markdown +cmd/github-labels/kata-github-labels integration/kubernetes/runtimeclass_workloads_work/ diff --git a/tests/.golangci.yml b/tests/.golangci.yml new file mode 100644 index 0000000000..5608aedbae --- /dev/null +++ b/tests/.golangci.yml @@ -0,0 +1,33 @@ +# Copyright (c) 2017 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +run: + concurrency: 4 + deadline: 600s + skip-dirs: + - vendor +# Ignore auto-generated protobuf code. + skip-files: + - ".*\\.pb\\.go$" + +linters: + disable-all: true + enable: + - gocyclo + - gofmt + - gosimple + - govet + - ineffassign + - misspell + - staticcheck + - typecheck + - unused + +linters-settings: + gocyclo: + min_complexity: 15 + unused: + check-exported: true + govet: + enable: diff --git a/tests/go.mod b/tests/go.mod new file mode 100644 index 0000000000..61a26671c1 --- /dev/null +++ b/tests/go.mod @@ -0,0 +1,27 @@ +module github.com/kata-containers/tests + +go 1.19 + +require ( + github.com/BurntSushi/toml v0.3.1 + github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb + github.com/olekukonko/tablewriter v0.0.6-0.20210304033056-74c60be0ef68 + github.com/sirupsen/logrus v1.8.1 + github.com/stretchr/testify v1.7.1 + github.com/urfave/cli v1.22.0 + gopkg.in/russross/blackfriday.v2 v2.0.0-00010101000000-000000000000 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/cpuguy83/go-md2man v1.0.10 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/russross/blackfriday v1.6.0 // indirect + golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect +) + +replace gopkg.in/russross/blackfriday.v2 => github.com/russross/blackfriday/v2 v2.1.0 diff --git a/tests/kata-doc-to-script.sh b/tests/kata-doc-to-script.sh new file mode 100755 index 0000000000..126073fd57 --- /dev/null +++ b/tests/kata-doc-to-script.sh @@ -0,0 +1,229 @@ +#!/bin/bash +license=" +# +# Copyright (c) 2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +" + +set -e + +[ -n "$DEBUG" ] && set -x + +script_name="${0##*/}" + +typeset -r warning="WARNING: Do *NOT* run the generated script without reviewing it carefully first!" + +# github markdown markers used to surround a code block. All text within the +# markers is rendered in a fixed font. +typeset -r bash_block_open="\`\`\`bash" +typeset -r block_open="\`\`\`" +typeset -r block_close="\`\`\`" + +# GitHub issue templates have a special metadata section at the top delimited +# by this string. See: +# +# https://raw.githubusercontent.com/kata-containers/.github/master/.github/ISSUE_TEMPLATE/bug_report.md +typeset -r metadata_block='---' + +# Used to delimit inline code blocks +typeset -r backtick="\`" + +# convention used in all documentation to represent a non-privileged users +# shell prompt. All lines starting with this value inside a code block are +# commands the user should run. +typeset -r code_prompt="\$ " + +# files are expected to match this regular expression +typeset -r extension_regex="\.md$" + +strict="no" +require_commands="no" +check_only="no" +invert="no" +verbose="no" + +usage() +{ + cat < [ []] + +This script will convert a github-flavoured markdown document file into a +bash(1) script to stdout by extracting the bash code blocks. + +Options: + + -c : check the file but don't create the script (sets exit code). + -h : show this usage. + -i : invert output (remove code blocks and inline code, displaying the + remaining parts of the document). Incompatible with '-c'. + -r : require atleast one command block to be found. + -s : strict mode - perform extra checks. + -v : verbose mode. + +Example usage: + + $ ${script_name} foo.md foo.md.sh + +Notes: + +- If a description is specified, it will be added to the script as a + comment. +- may be specified as '-' meaning send output to stdout. + +Limitations: + +- The script is unable to handle embedded code blocks like this: + + \`\`\` + + \`\`\`bash + \$ echo code in an embedded set of backticks + \`\`\` + + \`\`\` + + To overcome this issue, ensure that the outer set of backticks are replaced + with an HTML PRE tag: + + 
+
+      \`\`\`bash
+      \$ echo code in an embedded set of backticks
+      \`\`\`
+
+
+ + This will both render correctly on GitHub and allow this script to remove + the code block. + + Note: this solves one problem but introduces another - this script will not + remove the HTML tags. + +${warning} + +EOF + + exit 0 +} + +die() +{ + local msg="$*" + + echo "ERROR: $msg" >&2 + exit 1 +} + +script_header() +{ + local -r description="$1" + + cat <<-EOF + #!/bin/bash + ${license} + #---------------------------------------------- + # WARNING: Script auto-generated from '$file'. + # + # ${warning} + #---------------------------------------------- + + #---------------------------------------------- + # Description: $description + #---------------------------------------------- + + # fail the entire script if any simple command fails + set -e + +EOF +} + +# Convert the specified github-flavoured markdown format file +# into a bash script by extracting the bash blocks. +doc_to_script() +{ + file="$1" + outfile="$2" + description="$3" + invert="$4" + + [ -n "$file" ] || die "need file" + + [ "${check_only}" = "no" ] && [ -z "$outfile" ] && die "need output file" + [ "$outfile" = '-' ] && outfile="/dev/stdout" + + if [ "$invert" = "yes" ] + then + # First, remove code blocks. + # Next, remove inline code in backticks. + # Finally, remove a metadata block as used in GitHub issue + # templates. + cat "$file" |\ + sed -e "/^[ \>]*${block_open}/,/^[ \>]*${block_close}/d" \ + -e "s/${backtick}[^${backtick}]*${backtick}//g" \ + -e "/^${metadata_block}$/,/^${metadata_block}$/d" \ + > "$outfile" + return + fi + + all=$(mktemp) + body=$(mktemp) + + cat "$file" |\ + sed -n "/^ *${bash_block_open}/,/^ *${block_close}/ p" |\ + sed -e "/^ *${block_close}/ d" \ + -e "s/^ *${code_prompt}//g" \ + -e 's/^ *//g' > "$body" + + [ "$require_commands" = "yes" ] && [ ! -s "$body" ] && die "no commands found in file '$file'" + + script_header "$description" > "$all" + cat "$body" >> "$all" + + # sanity check + [ "$check_only" = "yes" ] && redirect="1>/dev/null 2>/dev/null" + + { local ret; eval bash -n "$all" $redirect; ret=$?; } || true + [ "$ret" -ne 0 ] && die "shell code in file '$file' is not valid" + + # create output file + [ "$check_only" = "no" ] && cp "$all" "$outfile" + + # clean up + rm -f "$body" "$all" +} + +main() +{ + while getopts "chirsv" opt + do + case $opt in + c) check_only="yes" ;; + h) usage ;; + i) invert="yes" ;; + r) require_commands="yes" ;; + s) strict="yes" ;; + v) verbose="yes" ;; + esac + done + + shift $(($OPTIND - 1)) + + file="$1" + outfile="$2" + description="$3" + + [ -n "$file" ] || die "need file" + + [ "$verbose" = "yes" ] && echo "INFO: processing file '$file'" + + if [ "$strict" = "yes" ] + then + echo "$file"|grep -q "$extension_regex" ||\ + die "file '$file' doesn't match pattern '$extension_regex'" + fi + + doc_to_script "$file" "$outfile" "$description" "$invert" +} + +main "$@" diff --git a/tests/static-checks.sh b/tests/static-checks.sh new file mode 100755 index 0000000000..8f747198c1 --- /dev/null +++ b/tests/static-checks.sh @@ -0,0 +1,1334 @@ +#!/usr/bin/env bash + +# Copyright (c) 2017-2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Description: Central script to run all static checks. +# This script should be called by all other repositories to ensure +# there is only a single source of all static checks. + +set -e + +[ -n "$DEBUG" ] && set -x + +cidir=$(realpath $(dirname "$0")) +source "${cidir}/lib.sh" + +# By default in Golang >= 1.16 GO111MODULE is set to "on", +# some subprojects in this repo may not support "go modules", +# set GO111MODULE to "auto" to enable module-aware mode only when +# a go.mod file is present in the current directory. +export GO111MODULE="auto" +export tests_repo="${tests_repo:-github.com/kata-containers/tests}" +export tests_repo_dir="${GOPATH}/src/${tests_repo}" + +# List of files to delete on exit +files_to_remove=() + +script_name=${0##*/} + +# Static check functions must follow the following naming conventions: +# + +# All static check function names must match this pattern. +typeset -r check_func_regex="^static_check_" + +# All architecture-specific static check functions must match this pattern. +typeset -r arch_func_regex="_arch_specific$" + +repo="" +specific_branch="false" +force="false" +branch=${branch:-main} + +# Which static check functions to consider. +handle_funcs="all" + +single_func_only="false" +list_only="false" + +# number of seconds to wait for curl to check a URL +typeset url_check_timeout_secs="${url_check_timeout_secs:-60}" + +# number of attempts that will be made to check an individual URL. +typeset url_check_max_tries="${url_check_max_tries:-3}" + +typeset -A long_options + +# Generated code +ignore_clh_generated_code="virtcontainers/pkg/cloud-hypervisor/client" + +paths_to_skip=( + "${ignore_clh_generated_code}" + "vendor" +) + +# Skip paths that are not statically checked +# $1 : List of paths to check, space separated list +# If you have a list in a bash array call in this way: +# list=$(skip_paths "${list[@]}") +# If you still want to use it as an array do: +# list=(${list}) +skip_paths(){ + local list_param="${1}" + [ -z "$list_param" ] && return + local list=(${list_param}) + + for p in "${paths_to_skip[@]}"; do + new_list=() + for l in "${list[@]}"; do + if echo "${l}" | grep -qv "${p}"; then + new_list=("${new_list[@]}" "${l}") + fi + done + list=("${new_list[@]}") + done + echo "${list[@]}" +} + + +long_options=( + [all]="Force checking of all changes, including files in the base branch" + [branch]="Specify upstream branch to compare against (default '$branch')" + [docs]="Check document files" + [dockerfiles]="Check dockerfiles" + [files]="Check files" + [force]="Force a skipped test to run" + [golang]="Check '.go' files" + [help]="Display usage statement" + [json]="Check JSON files" + [labels]="Check labels databases" + [licenses]="Check licenses" + [list]="List tests that would run" + [no-arch]="Run/list all tests except architecture-specific ones" + [only-arch]="Only run/list architecture-specific tests" + [repo:]="Specify GitHub URL of repo to use (github.com/user/repo)" + [scripts]="Check script files" + [vendor]="Check vendor files" + [versions]="Check versions files" + [xml]="Check XML files" +) + +yamllint_cmd="yamllint" +have_yamllint_cmd=$(command -v "$yamllint_cmd" || true) + +chronic=chronic + +# Disable chronic on OSX to avoid having to update the Travis config files +# for additional packages on that platform. +[ "$(uname -s)" == "Darwin" ] && chronic= + +usage() +{ + cat </dev/null || die "function '$name' does not exist" +} + +# Calls die() if the specified function is not valid or not a check function. +ensure_func_is_check_func() { + local name="$1" + + func_is_valid "$name" + + { echo "$name" | grep -q "${check_func_regex}"; ret=$?; } + + [ "$ret" = 0 ] || die "function '$name' is not a check function" +} + +# Returns "yes" if the specified function needs to run on all architectures, +# else "no". +func_is_arch_specific() { + local name="$1" + + ensure_func_is_check_func "$name" + + { echo "$name" | grep -q "${arch_func_regex}"; ret=$?; } + + if [ "$ret" = 0 ]; then + echo "yes" + else + echo "no" + fi +} + +function remove_tmp_files() { + rm -rf "${files_to_remove[@]}" +} + +# Convert a golang package to a full path +pkg_to_path() +{ + local pkg="$1" + + go list -f '{{.Dir}}' "$pkg" +} + +# Check that chronic is installed, otherwise die. +need_chronic() { + local first_word + [ -z "$chronic" ] && return + first_word="${chronic%% *}" + command -v chronic &>/dev/null || \ + die "chronic command not found. You must have it installed to run this check." \ + "Usually it is distributed with the 'moreutils' package of your Linux distribution." +} + + +static_check_go_arch_specific() +{ + local go_packages + local submodule_packages + local all_packages + + # List of all golang packages found in all submodules + # + # These will be ignored: since they are references to other + # repositories, we assume they are tested independently in their + # repository so do not need to be re-tested here. + submodule_packages=$(mktemp) + git submodule -q foreach "go list ./..." | sort > "$submodule_packages" || true + + # all packages + all_packages=$(mktemp) + go list ./... | sort > "$all_packages" || true + + files_to_remove+=("$submodule_packages" "$all_packages") + + # List of packages to consider which is defined as: + # + # "all packages" - "submodule packages" + # + # Note: the vendor filtering is required for versions of go older than 1.9 + go_packages=$(comm -3 "$all_packages" "$submodule_packages" || true) + go_packages=$(skip_paths "${go_packages[@]}") + + # No packages to test + [ -z "$go_packages" ] && return + + local linter="golangci-lint" + + # Run golang checks + if [ ! "$(command -v $linter)" ] + then + info "Installing ${linter}" + + local linter_url=$(get_test_version "externals.golangci-lint.url") + local linter_version=$(get_test_version "externals.golangci-lint.version") + + info "Forcing ${linter} version ${linter_version}" + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin "${linter_version}" + command -v $linter &>/dev/null || \ + die "$linter command not found. Ensure that \"\$GOPATH/bin\" is in your \$PATH." + fi + + local linter_args="run -c ${cidir}/.golangci.yml" + + # Non-option arguments other than "./..." are + # considered to be directories by $linter, not package names. + # Hence, we need to obtain a list of package directories to check, + # excluding any that relate to submodules. + local dirs + + for pkg in $go_packages + do + path=$(pkg_to_path "$pkg") + + makefile="${path}/Makefile" + + # perform a basic build since some repos generate code which + # is required for the package to be buildable (and thus + # checkable). + [ -f "$makefile" ] && (cd "$path" && make) + + dirs+=" $path" + done + + info "Running $linter checks on the following packages:\n" + echo "$go_packages" + echo + info "Package paths:\n" + echo "$dirs" | sed 's/^ *//g' | tr ' ' '\n' + for d in ${dirs};do + info "Running $linter on $d" + (cd $d && GO111MODULE=auto eval "$linter" "${linter_args}" ".") + done + +} + +# Install yamllint in the different Linux distributions +install_yamllint() +{ + source /etc/os-release || source /usr/lib/os-release + + package="yamllint" + + case "$ID" in + centos|rhel) sudo yum -y install $package ;; + ubuntu) sudo apt-get -y install $package ;; + fedora) sudo dnf -y install $package ;; + *) die "Please install yamllint on $ID" ;; + esac + + have_yamllint_cmd=$(command -v "$yamllint_cmd" || true) + + if [ -z "$have_yamllint_cmd" ]; then + info "Cannot install $package" && return + fi +} + +# Check the "versions database". +# +# Some repositories use a versions database to maintain version information +# about non-golang dependencies. If found, check it for validity. +static_check_versions() +{ + local db="versions.yaml" + + if [ -z "$have_yamllint_cmd" ]; then + info "Installing yamllint" + install_yamllint + fi + + [ ! -e "$db" ] && return + + if [ -n "$have_yamllint_cmd" ]; then + eval "$yamllint_cmd" "$db" + else + info "Cannot check versions as $yamllint_cmd not available" + fi +} + +static_check_labels() +{ + [ $(uname -s) != Linux ] && info "Can only check labels under Linux" && return + + # Handle SLES which doesn't provide the required command. + [ -z "$have_yamllint_cmd" ] && info "Cannot check labels as $yamllint_cmd not available" && return + + # Since this script is called from another repositories directory, + # ensure the utility is built before the script below (which uses it) is run. + (cd "${tests_repo_dir}" && make github-labels) + + tmp=$(mktemp) + + files_to_remove+=("${tmp}") + + info "Checking labels for repo ${repo} using temporary combined database ${tmp}" + + bash -f "${tests_repo_dir}/cmd/github-labels/github-labels.sh" "generate" "${repo}" "${tmp}" +} + +# Ensure all files (where possible) contain an SPDX license header +static_check_license_headers() +{ + # The branch is the baseline - ignore it. + [ "$specific_branch" = "true" ] && return + + # See: https://spdx.org/licenses/Apache-2.0.html + local -r spdx_tag="SPDX-License-Identifier" + local -r spdx_license="Apache-2.0" + local -r license_pattern="${spdx_tag}: ${spdx_license}" + local -r copyright_pattern="Copyright" + + local header_checks=() + + header_checks+=("SPDX license header::${license_pattern}") + header_checks+=("Copyright header:-i:${copyright_pattern}") + + files=$(get_pr_changed_file_details || true) + + # Strip off status + files=$(echo "$files"|awk '{print $NF}') + + # no files were changed + [ -z "$files" ] && info "No files found" && return + + local header_check + + for header_check in "${header_checks[@]}" + do + local desc=$(echo "$header_check"|cut -d: -f1) + local extra_args=$(echo "$header_check"|cut -d: -f2) + local pattern=$(echo "$header_check"|cut -d: -f3-) + + info "Checking $desc" + + local missing=$(egrep \ + --exclude=".git/*" \ + --exclude=".gitignore" \ + --exclude=".dockerignore" \ + --exclude="Gopkg.lock" \ + --exclude="*.gpl.c" \ + --exclude="*.ipynb" \ + --exclude="*.jpg" \ + --exclude="*.json" \ + --exclude="LICENSE*" \ + --exclude="THIRD-PARTY" \ + --exclude="*.md" \ + --exclude="*.pb.go" \ + --exclude="*pb_test.go" \ + --exclude="*.bin" \ + --exclude="*.png" \ + --exclude="*.pub" \ + --exclude="*.service" \ + --exclude="*.svg" \ + --exclude="*.drawio" \ + --exclude="*.toml" \ + --exclude="*.txt" \ + --exclude="*.dtd" \ + --exclude="vendor/*" \ + --exclude="VERSION" \ + --exclude="kata_config_version" \ + --exclude="tools/packaging/kernel/configs/*" \ + --exclude="virtcontainers/pkg/firecracker/*" \ + --exclude="${ignore_clh_generated_code}*" \ + --exclude="*.xml" \ + --exclude="*.yaml" \ + --exclude="*.yml" \ + --exclude="go.mod" \ + --exclude="go.sum" \ + --exclude="*.lock" \ + --exclude="grpc-rs/*" \ + --exclude="target/*" \ + --exclude="*.patch" \ + --exclude="*.diff" \ + --exclude="tools/packaging/static-build/qemu.blacklist" \ + --exclude="tools/packaging/qemu/default-configs/*" \ + --exclude="src/libs/protocols/protos/gogo/*.proto" \ + --exclude="src/libs/protocols/protos/google/*.proto" \ + --exclude="src/libs/*/test/texture/*" \ + -EL $extra_args "\<${pattern}\>" \ + $files || true) + + if [ -n "$missing" ]; then + cat >&2 <<-EOF + ERROR: Required $desc check ('$pattern') failed for the following files: + + $missing + +EOF + exit 1 + fi + done +} + +check_url() +{ + local url="$1" + local invalid_urls_dir="$2" + + local curl_out=$(mktemp) + files_to_remove+=("${curl_out}") + + info "Checking URL $url" + + # Process specific file to avoid out-of-order writes + local invalid_file=$(printf "%s/%d" "$invalid_urls_dir" "$$") + + local ret + local user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36" + + # Authenticate for github to increase threshold for rate limiting + local curl_args=() + if [[ "$url" =~ github\.com && -n "$GITHUB_USER" && -n "$GITHUB_TOKEN" ]]; then + curl_args+=("-u ${GITHUB_USER}:${GITHUB_TOKEN}") + fi + + # Some endpoints return 403 to HEAD but 200 for GET, so perform a GET but only read headers. + { curl ${curl_args[*]} -sIL -X GET -c - -A "${user_agent}" -H "Accept-Encoding: zstd, none, gzip, deflate" --max-time "$url_check_timeout_secs" \ + --retry "$url_check_max_tries" "$url" &>"$curl_out"; ret=$?; } || true + + # A transitory error, or the URL is incorrect, + # but capture either way. + if [ "$ret" -ne 0 ]; then + echo "$url" >> "${invalid_file}" + + die "check failed for URL $url after $url_check_max_tries tries" + fi + + local http_statuses + + http_statuses=$(grep -E "^HTTP" "$curl_out" | awk '{print $2}' || true) + if [ -z "$http_statuses" ]; then + echo "$url" >> "${invalid_file}" + die "no HTTP status codes for URL $url" + fi + + local status + + for status in $http_statuses + do + # Ignore the following ranges of status codes: + # + # - 1xx: Informational codes. + # - 2xx: Success codes. + # - 3xx: Redirection codes. + # - 405: Specifically to handle some sites + # which get upset by "curl -L" when the + # redirection is not required. + # + # Anything else is considered an error. + # + # See https://en.wikipedia.org/wiki/List_of_HTTP_status_codes + + if ! echo "$status" | grep -qE "^(1[0-9][0-9]|2[0-9][0-9]|3[0-9][0-9]|405)"; then + echo "$url" >> "$invalid_file" + die "found HTTP error status codes for URL $url ($status)" + fi + done +} + +# Perform basic checks on documentation files +static_check_docs() +{ + local cmd="xurls" + + if [ ! "$(command -v $cmd)" ] + then + info "Installing $cmd utility" + + local version + local url + + version=$(get_test_version "externals.xurls.version") + url=$(get_test_version "externals.xurls.url") + + # xurls is very fussy about how it's built. + go install "${url}@${version}" + + command -v xurls &>/dev/null || + die 'xurls not found. Ensure that "$GOPATH/bin" is in your $PATH' + fi + + info "Checking documentation" + + local doc + local all_docs + local docs + local docs_status + local new_docs + local new_urls + local url + + all_docs=$(git ls-files "*.md" | grep -Ev "(grpc-rs|target)/" | sort || true) + all_docs=$(skip_paths "${all_docs[@]}") + + if [ "$specific_branch" = "true" ] + then + info "Checking all documents in $branch branch" + docs="$all_docs" + else + info "Checking local branch for changed documents only" + + docs_status=$(get_pr_changed_file_details || true) + docs_status=$(echo "$docs_status" | grep "\.md$" || true) + + docs=$(echo "$docs_status" | awk '{print $NF}' | sort) + docs=$(skip_paths "${docs[@]}") + + # Newly-added docs + new_docs=$(echo "$docs_status" | awk '/^A/ {print $NF}' | sort) + new_docs=$(skip_paths "${new_docs[@]}") + + for doc in $new_docs + do + # A new document file has been added. If that new doc + # file is referenced by any files on this PR, checking + # its URL will fail since the PR hasn't been merged + # yet. We could construct the URL based on the users + # original PR branch and validate that. But it's + # simpler to just construct the URL that the "pending + # document" *will* result in when the PR has landed + # and then check docs for that new URL and exclude + # them from the real URL check. + url="https://${repo}/blob/${branch}/${doc}" + + new_urls+=" ${url}" + done + fi + + [ -z "$docs" ] && info "No documentation to check" && return + + local urls + local url_map=$(mktemp) + local invalid_urls=$(mktemp) + local md_links=$(mktemp) + files_to_remove+=("${url_map}" "${invalid_urls}" "${md_links}") + + info "Checking document markdown references" + + local md_docs_to_check + + # All markdown docs are checked (not just those changed by a PR). This + # is necessary to guarantee that all docs are referenced. + md_docs_to_check="$all_docs" + + (cd "${tests_repo_dir}" && make check-markdown) + + command -v kata-check-markdown &>/dev/null || \ + die 'kata-check-markdown command not found. Ensure that "$GOPATH/bin" is in your $PATH.' + + for doc in $md_docs_to_check + do + kata-check-markdown check "$doc" + + # Get a link of all other markdown files this doc references + kata-check-markdown list links --format tsv --no-header "$doc" |\ + grep "external-link" |\ + awk '{print $3}' |\ + sort -u >> "$md_links" + done + + # clean the list of links + local tmp + tmp=$(mktemp) + + sort -u "$md_links" > "$tmp" + mv "$tmp" "$md_links" + + # A list of markdown files that do not have to be referenced by any + # other markdown file. + exclude_doc_regexs+=() + + exclude_doc_regexs+=(^CODE_OF_CONDUCT\.md$) + exclude_doc_regexs+=(^CONTRIBUTING\.md$) + + # Magic github template files + exclude_doc_regexs+=(^\.github/.*\.md$) + + # The top level README doesn't need to be referenced by any other + # since it displayed by default when visiting the repo. + exclude_doc_regexs+=(^README\.md$) + + local exclude_pattern + + # Convert the list of files into an egrep(1) alternation pattern. + exclude_pattern=$(echo "${exclude_doc_regexs[@]}"|sed 's, ,|,g') + + # Every document in the repo (except a small handful of exceptions) + # should be referenced by another document. + for doc in $md_docs_to_check + do + # Check the ignore list for markdown files that do not need to + # be referenced by others. + echo "$doc"|egrep -q "(${exclude_pattern})" && continue + + grep -q "$doc" "$md_links" || die "Document $doc is not referenced" + done + + info "Checking document code blocks" + + local doc_to_script_cmd="${cidir}/kata-doc-to-script.sh" + + for doc in $docs + do + bash "${doc_to_script_cmd}" -csv "$doc" + + # Look for URLs in the document + urls=$("${doc_to_script_cmd}" -i "$doc" - | "$cmd") + + # Gather URLs + for url in $urls + do + printf "%s\t%s\n" "${url}" "${doc}" >> "$url_map" + done + done + + # Get unique list of URLs + urls=$(awk '{print $1}' "$url_map" | sort -u) + + info "Checking all document URLs" + local invalid_urls_dir=$(mktemp -d) + files_to_remove+=("${invalid_urls_dir}") + + for url in $urls + do + if [ "$specific_branch" != "true" ] + then + # If the URL is new on this PR, it cannot be checked. + echo "$new_urls" | egrep -q "\<${url}\>" && \ + info "ignoring new (but correct) URL: $url" && continue + fi + + # Ignore local URLs. The only time these are used is in + # examples (meaning these URLs won't exist). + echo "$url" | grep -q "^file://" && continue + echo "$url" | grep -q "^http://localhost" && continue + + # Ignore the install guide URLs that contain a shell variable + echo "$url" | grep -q "\\$" && continue + + # This prefix requires the client to be logged in to github, so ignore + echo "$url" | grep -q 'https://github.com/pulls' && continue + + # Sigh. + echo "$url"|grep -q 'https://example.com' && continue + + # Google APIs typically require an auth token. + echo "$url"|grep -q 'https://www.googleapis.com' && continue + + # Git repo URL check + if echo "$url"|grep -q '^https.*git' + then + timeout "${KATA_NET_TIMEOUT}" git ls-remote "$url" > /dev/null 2>&1 && continue + fi + + # Check the URL, saving it if invalid + # + # Each URL is checked in a separate process as each unique URL + # requires us to hit the network. + check_url "$url" "$invalid_urls_dir" & + done + + # Synchronisation point + wait + + # Combine all the separate invalid URL files into one + local invalid_files=$(ls "$invalid_urls_dir") + + if [ -n "$invalid_files" ]; then + pushd "$invalid_urls_dir" &>/dev/null + cat $(echo "$invalid_files"|tr '\n' ' ') > "$invalid_urls" + popd &>/dev/null + fi + + if [ -s "$invalid_urls" ] + then + local files + + cat "$invalid_urls" | while read url + do + files=$(grep "^${url}" "$url_map" | awk '{print $2}' | sort -u) + echo >&2 -e "ERROR: Invalid URL '$url' found in the following files:\n" + + for file in $files + do + echo >&2 "$file" + done + done + + exit 1 + fi + + # Now, spell check the docs + cmd="${tests_repo_dir}/cmd/check-spelling/kata-spell-check.sh" + + local docs_failed=0 + for doc in $docs + do + "$cmd" check "$doc" || { info "spell check failed for document $doc" && docs_failed=1; } + + static_check_eof "$doc" + done + + [ $docs_failed -eq 0 ] || die "spell check failed, See https://github.com/kata-containers/kata-containers/blob/main/docs/Documentation-Requirements.md#spelling for more information." +} + +static_check_eof() +{ + local file="$1" + local anchor="EOF" + + + [ -z "$file" ] && info "No files to check" && return + + # Skip the itself + [ "$file" == "$script_name" ] && return + + # Skip the Vagrantfile + [ "$file" == "Vagrantfile" ] && return + + local invalid=$(cat "$file" |\ + egrep -o '<<-* *\w*' |\ + sed -e 's/^<<-*//g' |\ + tr -d ' ' |\ + sort -u |\ + egrep -v '^$' |\ + egrep -v "$anchor" || true) + [ -z "$invalid" ] || die "Expected '$anchor' here anchor, in $file found: $invalid" +} + +# Tests to apply to all files. +# +# Currently just looks for TODO/FIXME comments that should be converted to +# (or annotated with) an Issue URL. +static_check_files() +{ + local file + local files + + if [ "$force" = "false" ] + then + info "Skipping check_files: see https://github.com/kata-containers/tests/issues/469" + return + else + info "Force override of check_files skip" + fi + + info "Checking files" + + if [ "$specific_branch" = "true" ] + then + info "Checking all files in $branch branch" + + files=$(git ls-files | egrep -v "/(.git|vendor|grpc-rs|target)/" || true) + else + info "Checking local branch for changed files only" + + files=$(get_pr_changed_file_details || true) + + # Strip off status + files=$(echo "$files"|awk '{print $NF}') + fi + + [ -z "$files" ] && info "No files changed" && return + + local matches="" + + for file in $files + do + local match + + # Look for files containing the specified comment tags but + # which do not include a github URL. + match=$(egrep -H "\|\" "$file" |\ + grep -v "https://github.com/.*/issues/[0-9]" |\ + cut -d: -f1 |\ + sort -u || true) + + [ -z "$match" ] && continue + + # Don't fail if this script contains the patterns + # (as it is guaranteed to ;) + echo "$file" | grep -q "${script_name}$" && info "Ignoring special file $file" && continue + + # We really only care about comments in code. But to avoid + # having to hard-code the list of file extensions to search, + # invert the problem by simply ignoring document files and + # considering all other file types. + echo "$file" | grep -q ".md$" && info "Ignoring comment tag in document $file" && continue + + matches+=" $match" + done + + [ -z "$matches" ] && return + + echo >&2 -n \ + "ERROR: The following files contain TODO/FIXME's that need " + echo >&2 -e "converting to issues:\n" + + for file in $matches + do + echo >&2 "$file" + done + + # spacer + echo >&2 + + exit 1 +} + +# Perform vendor checks: +# +# - Ensure that changes to vendored code are accompanied by an update to the +# vendor tooling config file. If not, the user simply hacked the vendor files +# rather than following the correct process: +# +# https://github.com/kata-containers/community/blob/main/VENDORING.md +# +# - Ensure vendor metadata is valid. +static_check_vendor() +{ + local files + local vendor_files + local result + + # Check if repo has been changed to use go modules + if [ -f "go.mod" ]; then + info "go.mod file found, running go mod verify instead" + # This verifies the integrity of modules in the local cache. + # This does not really verify the integrity of vendored code: + # https://github.com/golang/go/issues/27348 + # Once that is added we need to add an extra step to verify vendored code. + go mod verify + return + fi +} + +static_check_xml() +{ + local all_xml + local files + + need_chronic + + all_xml=$(git ls-files "*.xml" | grep -Ev "/(vendor|grpc-rs|target)/" | sort || true) + + if [ "$specific_branch" = "true" ] + then + info "Checking all XML files in $branch branch" + files="$all_xml" + else + info "Checking local branch for changed XML files only" + + local xml_status + + xml_status=$(get_pr_changed_file_details || true) + xml_status=$(echo "$xml_status" | grep "\.xml$" || true) + + files=$(echo "$xml_status" | awk '{print $NF}') + fi + + [ -z "$files" ] && info "No XML files to check" && return + + local file + + for file in $files + do + info "Checking XML file '$file'" + + local contents + + # Most XML documents are specified as XML 1.0 since, with the + # advent of XML 1.0 (Fifth Edition), XML 1.1 is "almost + # redundant" due to XML 1.0 providing the majority of XML 1.1 + # features. xmllint doesn't support XML 1.1 seemingly for this + # reason, so the only check we can do is to (crudely) force + # the document to be an XML 1.0 one since XML 1.1 documents + # can mostly be represented as XML 1.0. + # + # This is only really required since Jenkins creates XML 1.1 + # documents. + contents=$(sed "s/xml version='1.1'/xml version='1.0'/g" "$file") + + local ret + + { $chronic xmllint -format - <<< "$contents"; ret=$?; } || true + + [ "$ret" -eq 0 ] || die "failed to check XML file '$file'" + done +} + +static_check_shell() +{ + local all_scripts + local scripts + + need_chronic + + all_scripts=$(git ls-files "*.sh" "*.bash" | grep -Ev "/(vendor|grpc-rs|target)/" | sort || true) + + if [ "$specific_branch" = "true" ] + then + info "Checking all scripts in $branch branch" + scripts="$all_scripts" + else + info "Checking local branch for changed scripts only" + + local scripts_status + scripts_status=$(get_pr_changed_file_details || true) + scripts_status=$(echo "$scripts_status" | grep -E "\.(sh|bash)$" || true) + + scripts=$(echo "$scripts_status" | awk '{print $NF}') + fi + + [ -z "$scripts" ] && info "No scripts to check" && return 0 + + local script + + for script in $scripts + do + info "Checking script file '$script'" + + local ret + + { $chronic bash -n "$script"; ret=$?; } || true + + [ "$ret" -eq 0 ] || die "check for script '$script' failed" + + static_check_eof "$script" + done +} + +static_check_json() +{ + local all_json + local json_files + + need_chronic + + all_json=$(git ls-files "*.json" | grep -Ev "/(vendor|grpc-rs|target)/" | sort || true) + + if [ "$specific_branch" = "true" ] + then + info "Checking all JSON in $branch branch" + json_files="$all_json" + else + info "Checking local branch for changed JSON only" + + local json_status + json_status=$(get_pr_changed_file_details || true) + json_status=$(echo "$json_status" | grep "\.json$" || true) + + json_files=$(echo "$json_status" | awk '{print $NF}') + fi + + [ -z "$json_files" ] && info "No JSON files to check" && return 0 + + local json + + for json in $json_files + do + info "Checking JSON file '$json'" + + local ret + + { $chronic jq -S . "$json"; ret=$?; } || true + + [ "$ret" -eq 0 ] || die "failed to check JSON file '$json'" + done +} + +# The dockerfile checker relies on the hadolint tool. This function handle its +# installation if it is not found on PATH. +# Note that we need a specific version of the tool as it seems to not have +# backward/forward compatibility between versions. +has_hadolint_or_install() +{ + # Global variable set by the caller. It might be overwritten here. + linter_cmd=${linter_cmd:-"hadolint"} + local linter_version=$(get_test_version "externals.hadolint.version") + local linter_url=$(get_test_version "externals.hadolint.url") + local linter_dest="${GOPATH}/bin/hadolint" + + local has_linter=$(command -v "$linter_cmd") + if [[ -z "$has_linter" && "$KATA_DEV_MODE" == "yes" ]]; then + # Do not install if it is in development mode. + die "$linter_cmd command not found. You must have the version $linter_version installed to run this check." + elif [ -n "$has_linter" ]; then + # Check if the expected linter version + if $linter_cmd --version | grep -v "$linter_version" &>/dev/null; then + warn "$linter_cmd command found but not the required version $linter_version" + has_linter="" + fi + fi + + if [ -z "$has_linter" ]; then + local download_url="${linter_url}/releases/download/v${linter_version}/hadolint-Linux-x86_64" + info "Installing $linter_cmd $linter_version at $linter_dest" + + curl -sfL "$download_url" -o "$linter_dest" || \ + die "Failed to download $download_url" + chmod +x "$linter_dest" + + # Overwrite in case it cannot be found in PATH. + linter_cmd="$linter_dest" + fi +} + +static_check_dockerfiles() +{ + local all_files + local files + local ignore_files + # Put here a list of files which should be ignored. + local ignore_files=( + ) + local linter_cmd="hadolint" + + all_files=$(git ls-files "*/Dockerfile*" | grep -Ev "/(vendor|grpc-rs|target)/" | sort || true) + + if [ "$specific_branch" = "true" ]; then + info "Checking all Dockerfiles in $branch branch" + files="$all_files" + else + info "Checking local branch for changed Dockerfiles only" + + local files_status + files_status=$(get_pr_changed_file_details || true) + files_status=$(echo "$files_status" | grep -E "Dockerfile.*$" || true) + + files=$(echo "$files_status" | awk '{print $NF}') + fi + + [ -z "$files" ] && info "No Dockerfiles to check" && return 0 + + # As of this writing hadolint is only distributed for x86_64 + if [ "$(uname -m)" != "x86_64" ]; then + info "Skip checking as $linter_cmd is not available for $(uname -m)" + return 0 + fi + has_hadolint_or_install + + linter_cmd+=" --no-color" + + # Let's not fail with INFO rules. + linter_cmd+=" --failure-threshold warning" + + # Some rules we don't want checked, below we ignore them. + # + # "DL3008 warning: Pin versions in apt get install" + linter_cmd+=" --ignore DL3008" + # "DL3041 warning: Specify version with `dnf install -y -`" + linter_cmd+=" --ignore DL3041" + # "DL3033 warning: Specify version with `yum install -y -`" + linter_cmd+=" --ignore DL3033" + # "DL3018 warning: Pin versions in apk add. Instead of `apk add ` use `apk add =`" + linter_cmd+=" --ignore DL3018" + # "DL3003 warning: Use WORKDIR to switch to a directory" + # See https://github.com/hadolint/hadolint/issues/70 + linter_cmd+=" --ignore DL3003" + # "DL3048 style: Invalid label key" + linter_cmd+=" --ignore DL3048" + # DL3037 warning: Specify version with `zypper install -y =`. + linter_cmd+=" --ignore DL3037" + + local file + for file in $files; do + if echo "${ignore_files[@]}" | grep -q $file ; then + info "Ignoring Dockerfile '$file'" + continue + fi + + info "Checking Dockerfile '$file'" + local ret + # The linter generates an Abstract Syntax Tree (AST) from the + # dockerfile. Some of our dockerfiles are actually templates + # with special syntax, thus the linter might fail to build + # the AST. Here we handle Dockerfile templates. + if [[ "$file" =~ Dockerfile.*\.(in|template)$ ]]; then + # In our templates, text with marker as @SOME_NAME@ is + # replaceable. Usually it is used to replace in a + # FROM command (e.g. `FROM @UBUNTU_REGISTRY@/ubuntu`) + # but also to add an entire block of commands. Example + # of later: + # ``` + # RUN apt-get install -y package1 + # @INSTALL_MUSL@ + # @INSTALL_RUST@ + # ``` + # It's known that the linter will fail to parse lines + # started with `@`. Also it might give false-positives + # on some cases. Here we remove all markers as a best + # effort approach. If the template file is still + # unparseable then it should be added in the + # `$ignore_files` list. + { sed -e 's/^@[A-Z_]*@//' -e 's/@\([a-zA-Z_]*\)@/\1/g' "$file" | $linter_cmd -; ret=$?; }\ + || true + else + # Non-template Dockerfile. + { $linter_cmd "$file"; ret=$?; } || true + fi + + [ "$ret" -eq 0 ] || die "failed to check Dockerfile '$file'" + done +} + +# Run the specified function (after first checking it is compatible with the +# users architectural preferences), or simply list the function name if list +# mode is active. +run_or_list_check_function() +{ + local name="$1" + + func_is_valid "$name" + + local arch_func + local handler + + arch_func=$(func_is_arch_specific "$name") + + handler="info" + + # If the user requested only a single function to run, we should die + # if the function cannot be run due to the other options specified. + # + # Whereas if this script is running all functions, just display an + # info message if a function cannot be run. + [ "$single_func_only" = "true" ] && handler="die" + + if [ "$handle_funcs" = "arch-agnostic" ] && [ "$arch_func" = "yes" ]; then + if [ "$list_only" != "true" ]; then + "$handler" "Not running '$func' as requested no architecture-specific functions" + fi + + return 0 + fi + + if [ "$handle_funcs" = "arch-specific" ] && [ "$arch_func" = "no" ]; then + if [ "$list_only" != "true" ]; then + "$handler" "Not running architecture-agnostic function '$func' as requested only architecture specific functions" + fi + + return 0 + fi + + if [ "$list_only" = "true" ]; then + echo "$func" + return 0 + fi + + info "Running '$func' function" + eval "$func" +} + +main() +{ + trap remove_tmp_files EXIT + + local long_option_names="${!long_options[@]}" + + local args + + args=$(getopt \ + -n "$script_name" \ + -a \ + --options="h" \ + --longoptions="$long_option_names" \ + -- "$@") + [ $? -eq 0 ] || { usage >&2; exit 1; } + + eval set -- "$args" + + local func= + + while [ $# -gt 1 ] + do + case "$1" in + --all) specific_branch="true" ;; + --branch) branch="$2"; shift ;; + --commits) func=static_check_commits ;; + --docs) func=static_check_docs ;; + --dockerfiles) func=static_check_dockerfiles ;; + --files) func=static_check_files ;; + --force) force="true" ;; + --golang) func=static_check_go_arch_specific ;; + -h|--help) usage; exit 0 ;; + --json) func=static_check_json ;; + --labels) func=static_check_labels;; + --licenses) func=static_check_license_headers ;; + --list) list_only="true" ;; + --no-arch) handle_funcs="arch-agnostic" ;; + --only-arch) handle_funcs="arch-specific" ;; + --repo) repo="$2"; shift ;; + --scripts) func=static_check_shell ;; + --vendor) func=static_check_vendor;; + --versions) func=static_check_versions ;; + --xml) func=static_check_xml ;; + --) shift; break ;; + esac + + shift + done + + # Consume getopt cruft + [ "$1" = "--" ] && shift + + [ "$1" = "help" ] && usage && exit 0 + + # Set if not already set by options + [ -z "$repo" ] && repo="$1" + [ "$specific_branch" = "false" ] && specific_branch="$2" + + if [ -z "$repo" ] + then + if [ -n "$KATA_DEV_MODE" ] + then + # No repo param provided so assume it's the current + # one to avoid developers having to specify one now + # (backwards compatability). + repo=$(git config --get remote.origin.url |\ + sed 's!https://!!g' || true) + + info "Auto-detected repo as $repo" + else + if [ "$list_only" != "true" ]; then + echo >&2 "ERROR: need repo" && usage && exit 1 + fi + fi + fi + + local all_check_funcs=$(typeset -F|awk '{print $3}'|grep "${check_func_regex}"|sort) + + # Run user-specified check and quit + if [ -n "$func" ]; then + single_func_only="true" + run_or_list_check_function "$func" + exit 0 + fi + + for func in $all_check_funcs + do + run_or_list_check_function "$func" + done +} + +main "$@" From 8ad433d4ad3021d491db861223b104c55cfbd26b Mon Sep 17 00:00:00 2001 From: Chelsea Mafrica Date: Tue, 21 Nov 2023 17:45:32 -0800 Subject: [PATCH 2/6] tests: move markdown check tool to main repo Move the tool as a dependency for static checks migration. Fixes #8187 Signed-off-by: Bin Liu Signed-off-by: Chelsea Mafrica Signed-off-by: Gabriela Cervantes Signed-off-by: Ganesh Maharaj Mahalingam Signed-off-by: James O. D. Hunt Signed-off-by: Julio Montes --- tests/cmd/check-markdown/Makefile | 32 +++ tests/cmd/check-markdown/README.md | 57 ++++ tests/cmd/check-markdown/VERSION | 1 + tests/cmd/check-markdown/add.go | 135 +++++++++ tests/cmd/check-markdown/add_test.go | 191 +++++++++++++ tests/cmd/check-markdown/check.go | 118 ++++++++ tests/cmd/check-markdown/display.go | 102 +++++++ tests/cmd/check-markdown/display_text.go | 57 ++++ tests/cmd/check-markdown/display_tsv.go | 72 +++++ tests/cmd/check-markdown/doc.go | 76 +++++ tests/cmd/check-markdown/extract.go | 93 ++++++ tests/cmd/check-markdown/hack.go | 69 +++++ tests/cmd/check-markdown/heading.go | 36 +++ tests/cmd/check-markdown/heading_test.go | 65 +++++ tests/cmd/check-markdown/link.go | 122 ++++++++ tests/cmd/check-markdown/link_test.go | 209 ++++++++++++++ tests/cmd/check-markdown/main.go | 348 +++++++++++++++++++++++ tests/cmd/check-markdown/node.go | 115 ++++++++ tests/cmd/check-markdown/parse.go | 100 +++++++ tests/cmd/check-markdown/record.go | 43 +++ tests/cmd/check-markdown/search.go | 29 ++ tests/cmd/check-markdown/stats.go | 41 +++ tests/cmd/check-markdown/toc.go | 75 +++++ tests/cmd/check-markdown/types.go | 159 +++++++++++ tests/cmd/check-markdown/utils.go | 97 +++++++ tests/cmd/check-markdown/utils_test.go | 149 ++++++++++ 26 files changed, 2591 insertions(+) create mode 100644 tests/cmd/check-markdown/Makefile create mode 100644 tests/cmd/check-markdown/README.md create mode 100644 tests/cmd/check-markdown/VERSION create mode 100644 tests/cmd/check-markdown/add.go create mode 100644 tests/cmd/check-markdown/add_test.go create mode 100644 tests/cmd/check-markdown/check.go create mode 100644 tests/cmd/check-markdown/display.go create mode 100644 tests/cmd/check-markdown/display_text.go create mode 100644 tests/cmd/check-markdown/display_tsv.go create mode 100644 tests/cmd/check-markdown/doc.go create mode 100644 tests/cmd/check-markdown/extract.go create mode 100644 tests/cmd/check-markdown/hack.go create mode 100644 tests/cmd/check-markdown/heading.go create mode 100644 tests/cmd/check-markdown/heading_test.go create mode 100644 tests/cmd/check-markdown/link.go create mode 100644 tests/cmd/check-markdown/link_test.go create mode 100644 tests/cmd/check-markdown/main.go create mode 100644 tests/cmd/check-markdown/node.go create mode 100644 tests/cmd/check-markdown/parse.go create mode 100644 tests/cmd/check-markdown/record.go create mode 100644 tests/cmd/check-markdown/search.go create mode 100644 tests/cmd/check-markdown/stats.go create mode 100644 tests/cmd/check-markdown/toc.go create mode 100644 tests/cmd/check-markdown/types.go create mode 100644 tests/cmd/check-markdown/utils.go create mode 100644 tests/cmd/check-markdown/utils_test.go diff --git a/tests/cmd/check-markdown/Makefile b/tests/cmd/check-markdown/Makefile new file mode 100644 index 0000000000..bb35e6ca6a --- /dev/null +++ b/tests/cmd/check-markdown/Makefile @@ -0,0 +1,32 @@ +# +# Copyright (c) 2017-2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +TARGET = kata-check-markdown +SOURCES = $(shell find . -type f 2>&1 | grep -E '.*\.go$$') + +VERSION := ${shell cat ./VERSION} +COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true) +COMMIT := $(if $(shell git status --porcelain --untracked-files=no),"${COMMIT_NO}-dirty","${COMMIT_NO}") + +BINDIR := $(GOPATH)/bin +DESTTARGET := $(abspath $(BINDIR)/$(TARGET)) + +default: install + +check: $(SOURCES) + go test -v ./... + +$(TARGET): $(SOURCES) + go build -o "$(TARGET)" -ldflags "-X main.name=${TARGET} -X main.commit=${COMMIT} -X main.version=${VERSION}" . + +install: $(TARGET) + install -d $(shell dirname $(DESTTARGET)) + install $(TARGET) $(DESTTARGET) + +clean: + rm -f $(TARGET) + +.PHONY: install clean diff --git a/tests/cmd/check-markdown/README.md b/tests/cmd/check-markdown/README.md new file mode 100644 index 0000000000..c8e3e4fc8d --- /dev/null +++ b/tests/cmd/check-markdown/README.md @@ -0,0 +1,57 @@ +# Overview + +The Kata Project comprises +[a number of GitHub repositories](https://github.com/kata-containers). +All these repositories contain documents written in +[GitHub-Flavoured Markdown](https://github.github.com/gfm) +format. + +[Linking in documents is strongly encouraged](https://github.com/kata-containers/kata-containers/blob/main/docs/Documentation-Requirements.md) +but due to the number of internal and external document links, it is easy for +mistakes to be made. Also, links can become stale when one document is updated +but the documents it depends on are not. + +# Tool summary + +The `kata-check-markdown` tool checks a markdown document to ensure all links +within it are valid. All internal links are checked and by default all +external links are also checked. The tool is able to suggest corrections for +some errors it finds. It can also generate a TOC (table of contents). + +# Usage + +## Basic + +```sh +$ kata-check-markdown check README.md +``` + +## Generate a TOC + +```sh +$ kata-check-markdown toc README.md +``` + +## List headings + +To list the document headings in the default `text` format: + +```sh +$ kata-check-markdown list headings README.md +``` + +## List links + +To list the links in a document in tab-separated format: + +```sh +$ kata-check-markdown list links --format tsv README.md +``` + +## Full details + +Lists all available options: + +```sh +$ kata-check-markdown -h +``` diff --git a/tests/cmd/check-markdown/VERSION b/tests/cmd/check-markdown/VERSION new file mode 100644 index 0000000000..8acdd82b76 --- /dev/null +++ b/tests/cmd/check-markdown/VERSION @@ -0,0 +1 @@ +0.0.1 diff --git a/tests/cmd/check-markdown/add.go b/tests/cmd/check-markdown/add.go new file mode 100644 index 0000000000..182b3593b0 --- /dev/null +++ b/tests/cmd/check-markdown/add.go @@ -0,0 +1,135 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "errors" + "fmt" + "path/filepath" + "strings" + + "github.com/sirupsen/logrus" +) + +// linkAddrToPath converts a link address into a path name. +func (d *Doc) linkAddrToPath(address string) (string, error) { + if address == "" { + return "", errors.New("need address") + } + + dir := filepath.Dir(d.Name) + + var file string + + // An "absolute link path" like this has been specified: + // + // [Foo](/absolute-link.md) + if strings.HasPrefix(address, absoluteLinkPrefix) { + if !fileExists(docRoot) { + return "", fmt.Errorf("document root %q does not exist", docRoot) + } + + file = filepath.Join(docRoot, address) + } else { + file = filepath.Join(dir, address) + } + + return file, nil +} + +// addHeading adds the specified heading to the document. +// +// Note that headings must be unique. +func (d *Doc) addHeading(heading Heading) error { + name := heading.Name + + if name == "" { + return d.Errorf("heading name cannot be blank: %+v", heading) + } + + if heading.LinkName == "" { + return d.Errorf("heading link name cannot be blank: %q (%+v)", + name, heading) + } + + if heading.Level <= 0 { + return d.Errorf("heading level must be atleast 1: %q (%+v)", + name, heading) + } + + if _, ok := d.Headings[name]; ok { + return d.Errorf("duplicate heading: %q (heading: %+v)", + name, heading) + } + + // Potentially change the ID to handle strange characters + // supported in links by GitHub. + id, err := createHeadingID(heading.Name) + if err != nil { + return err + } + + heading.LinkName = id + + d.Logger.WithField("heading", fmt.Sprintf("%+v", heading)).Debug("adding heading") + + d.Headings[name] = heading + + return nil +} + +// addLink potentially adds the specified link to the document. +// +// Note that links do not need to be unique: a document can contain +// multiple links with: +// +// - the same description and the same address. +// - the same description but with different addresses. +// - different descriptions but with the same address. +func (d *Doc) addLink(link Link) error { + addr := link.Address + + if link.ResolvedPath != "" { + addr = link.ResolvedPath + } + + if addr == "" { + return d.Errorf("link address cannot be blank: %+v", link) + } + + if link.Type == unknownLink { + return d.Errorf("BUG: link type invalid: %+v", link) + } + + // Not checked by default as magic "build status" / go report / godoc + // links don't have a description - they have a image only. + if strict && link.Description == "" { + return d.Errorf("link description cannot be blank: %q (%+v)", + addr, link) + } + + fields := logrus.Fields{ + "link": fmt.Sprintf("%+v", link), + } + + links := d.Links[addr] + + for _, l := range links { + if l.Type == link.Type { + d.Logger.WithFields(fields).Debug("not adding duplicate link") + + return nil + } + } + + d.Logger.WithFields(fields).Debug("adding link") + + links = append(links, link) + d.Links[addr] = links + + return nil +} diff --git a/tests/cmd/check-markdown/add_test.go b/tests/cmd/check-markdown/add_test.go new file mode 100644 index 0000000000..3e5866a0ae --- /dev/null +++ b/tests/cmd/check-markdown/add_test.go @@ -0,0 +1,191 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +const ( + testFileMode = os.FileMode(0640) + testDirMode = os.FileMode(0750) + readmeName = "README.md" +) + +func createFile(file, contents string) error { + return os.WriteFile(file, []byte(contents), testFileMode) +} + +// makeDirs creates two directories below the specified base directory: one is +// an empty director named emptyDirName and the other is named readmeDirName +// and contains a markdown file called "README.md". +func makeDirs(assert *assert.Assertions, baseDir string, readmeDirName, emptyDirName string) { + readmeDir := filepath.Join(baseDir, readmeDirName) + err := os.MkdirAll(readmeDir, testDirMode) + assert.NoError(err) + + readme := filepath.Join(readmeDir, "README.md") + + err = createFile(readme, "# hello") + assert.NoError(err) + + emptyDir := filepath.Join(baseDir, emptyDirName) + err = os.MkdirAll(emptyDir, testDirMode) + assert.NoError(err) +} + +func TestDocAddHeading(t *testing.T) { + assert := assert.New(t) + + type testData struct { + heading Heading + expectError bool + } + + data := []testData{ + {Heading{"", "", "", -1}, true}, + {Heading{"Foo", "", "", -1}, true}, + {Heading{"Foo", "", "", 0}, true}, + {Heading{"Foo", "", "", 1}, true}, + {Heading{"Foo", "", "foo", -1}, true}, + {Heading{"Foo", "", "foo", 0}, true}, + + {Heading{"Foo", "", "foo", 1}, false}, + {Heading{"`Foo`", "`Foo`", "foo", 1}, false}, + } + + logger := logrus.WithField("test", "true") + + for i, d := range data { + doc := newDoc("foo", logger) + + assert.Empty(doc.Headings) + + msg := fmt.Sprintf("test[%d]: %+v\n", i, d) + + err := doc.addHeading(d.heading) + if d.expectError { + assert.Error(err, msg) + continue + } + + assert.NoError(err, msg) + assert.NotEmpty(doc.Headings, msg) + + name := d.heading.Name + + result, ok := doc.Headings[name] + assert.True(ok, msg) + + assert.Equal(d.heading, result, msg) + } +} + +func TestDocAddLink(t *testing.T) { + assert := assert.New(t) + + type testData struct { + link Link + expectError bool + } + + data := []testData{ + {Link{nil, "", "", "", -1}, true}, + {Link{nil, "foo", "", "", unknownLink}, true}, + + {Link{nil, "foo", "", "", internalLink}, false}, + {Link{nil, "http://google.com", "", "", urlLink}, false}, + {Link{nil, "https://google.com", "", "", urlLink}, false}, + {Link{nil, "mailto:me@somewhere.com", "", "", mailLink}, false}, + } + + logger := logrus.WithField("test", "true") + + for i, d := range data { + doc := newDoc("foo", logger) + + assert.Empty(doc.Links) + + msg := fmt.Sprintf("test[%d]: %+v\n", i, d) + + err := doc.addLink(d.link) + if d.expectError { + assert.Error(err, msg) + continue + } + + assert.NoError(err, msg) + assert.NotEmpty(doc.Links, msg) + addr := d.link.Address + + result := doc.Links[addr][0] + assert.Equal(result, d.link) + } +} + +func TestDocLinkAddrToPath(t *testing.T) { + assert := assert.New(t) + + dir, err := os.MkdirTemp("", "") + assert.NoError(err) + + cwd, err := os.Getwd() + assert.NoError(err) + defer os.Chdir(cwd) + + err = os.Chdir(dir) + assert.NoError(err) + defer os.RemoveAll(dir) + + savedDocRoot := docRoot + docRoot = dir + + defer func() { + docRoot = savedDocRoot + + }() + + mdFile := "bar.md" + mdPath := filepath.Join("/", mdFile) + actualMDPath := filepath.Join(dir, mdFile) + + type testData struct { + linkAddr string + expectedPath string + expectError bool + } + + data := []testData{ + {"", "", true}, + {"bar", "bar", false}, + {"bar.md", "bar.md", false}, + {mdPath, actualMDPath, false}, + } + + logger := logrus.WithField("test", "true") + doc := newDoc("foo", logger) + + for i, d := range data { + msg := fmt.Sprintf("test[%d]: %+v\n", i, d) + + result, err := doc.linkAddrToPath(d.linkAddr) + + if d.expectError { + assert.Error(err, msg) + continue + } + + assert.NoError(err, msg) + assert.Equal(d.expectedPath, result) + } +} diff --git a/tests/cmd/check-markdown/check.go b/tests/cmd/check-markdown/check.go new file mode 100644 index 0000000000..1bc038f88b --- /dev/null +++ b/tests/cmd/check-markdown/check.go @@ -0,0 +1,118 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "errors" + "fmt" +) + +// checkLink checks the validity of the specified link. If checkOtherDoc is +// true and the link is an external one, validate the link by considering the +// external document too. +func (d *Doc) checkLink(address string, link Link, checkOtherDoc bool) error { + if address == "" { + return errors.New("link address not set") + } + + switch link.Type { + case externalFile: + fallthrough + case externalLink: + // Check to ensure that referenced file actually exists + + var file string + + if link.ResolvedPath != "" { + file = link.ResolvedPath + } else { + file, _, err := splitLink(address) + if err != nil { + return err + } + + file, err = d.linkAddrToPath(file) + if err != nil { + return err + } + + if !fileExists(file) { + return d.Errorf("link type %v invalid: %q does not exist", + link.Type, + file) + } + } + + if link.Type == externalFile { + break + } + + // Check the other document + other, err := getDoc(file, d.Logger) + if err != nil { + return err + } + + if !checkOtherDoc { + break + } + + _, section, err := splitLink(address) + if err != nil { + return err + } + + if section == "" { + break + } + + if !other.hasHeading(section) { + return other.Errorf("invalid link %v", address) + } + + case internalLink: + // must be a link to an existing heading + + // search for a heading whose LinkName == name + found := d.headingByLinkName(address) + if found == nil { + msg := fmt.Sprintf("failed to find heading for link %q (%+v)", address, link) + + // There is a chance the link description matches the + // correct heading the link address refers to. In + // which case, we can derive the correct link address! + suggestion, err2 := createHeadingID(link.Description) + + if err2 == nil && suggestion != link.Address { + found = d.headingByLinkName(suggestion) + if found != nil { + msg = fmt.Sprintf("%s - correct link name is %q", msg, suggestion) + } + } + + return d.Errorf("%s", msg) + } + case urlLink: + // NOP - handled by xurls + } + + return nil +} + +// check performs all checks on the document. +func (d *Doc) check() error { + for name, linkList := range d.Links { + for _, link := range linkList { + err := d.checkLink(name, link, false) + if err != nil { + return err + } + } + } + + return nil +} diff --git a/tests/cmd/check-markdown/display.go b/tests/cmd/check-markdown/display.go new file mode 100644 index 0000000000..a6d2f7f1d0 --- /dev/null +++ b/tests/cmd/check-markdown/display.go @@ -0,0 +1,102 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + "os" + "sort" + + "github.com/sirupsen/logrus" +) + +var outputFile = os.Stdout + +// displayHandler is an interface that all output display handlers +// (formatters) must implement. +type DisplayHandler interface { + DisplayHeadings(d *Doc) error + DisplayLinks(d *Doc) error +} + +// DisplayHandlers encapsulates the list of available display handlers. +type DisplayHandlers struct { + handlers map[string]DisplayHandler +} + +// handlers is a map of the available output format display handling +// implementations. +var handlers map[string]DisplayHandler + +// NewDisplayHandlers create a new DisplayHandler. +func NewDisplayHandlers(tsvSeparator string, disableHeader bool) *DisplayHandlers { + separator := rune('\t') + + if tsvSeparator != "" { + separator = rune(tsvSeparator[0]) + } + + if handlers == nil { + handlers = make(map[string]DisplayHandler) + + handlers[textFormat] = NewDisplayText(outputFile) + handlers[tsvFormat] = NewDisplayTSV(outputFile, separator, disableHeader) + } + + h := &DisplayHandlers{ + handlers: handlers, + } + + return h +} + +// find looks for a display handler corresponding to the specified format +func (d *DisplayHandlers) find(format string) DisplayHandler { + for f, handler := range d.handlers { + if f == format { + return handler + } + } + + return nil +} + +// Get returns a list of the available formatters (display handler names). +func (d *DisplayHandlers) Get() []string { + var formats []string + + for f := range d.handlers { + formats = append(formats, f) + } + + sort.Strings(formats) + + return formats +} + +func show(inputFilename string, logger *logrus.Entry, handler DisplayHandler, what DataToShow) error { + var fn func(*Doc) error + + switch what { + case showHeadings: + fn = handler.DisplayHeadings + case showLinks: + fn = handler.DisplayLinks + default: + return fmt.Errorf("unknown show option: %v", what) + } + + doc := newDoc(inputFilename, logger) + doc.ListMode = true + + err := doc.parse() + if err != nil { + return err + } + + return fn(doc) +} diff --git a/tests/cmd/check-markdown/display_text.go b/tests/cmd/check-markdown/display_text.go new file mode 100644 index 0000000000..e7e5f87b1e --- /dev/null +++ b/tests/cmd/check-markdown/display_text.go @@ -0,0 +1,57 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + "os" +) + +type displayText struct { + file *os.File +} + +func NewDisplayText(file *os.File) DisplayHandler { + return &displayText{ + file: file, + } +} + +func (d *displayText) DisplayLinks(doc *Doc) error { + for _, linkList := range doc.Links { + for _, link := range linkList { + err := d.displayLink(link) + if err != nil { + return err + } + } + } + + return nil +} + +func (d *displayText) displayLink(l Link) error { + _, err := fmt.Fprintf(d.file, "%+v\n", l) + + return err +} + +func (d *displayText) DisplayHeadings(doc *Doc) error { + for _, h := range doc.Headings { + err := d.displayHeading(h) + if err != nil { + return err + } + } + + return nil +} + +func (d *displayText) displayHeading(h Heading) error { + _, err := fmt.Fprintf(d.file, "%+v\n", h) + + return err +} diff --git a/tests/cmd/check-markdown/display_tsv.go b/tests/cmd/check-markdown/display_tsv.go new file mode 100644 index 0000000000..f71ea91f3d --- /dev/null +++ b/tests/cmd/check-markdown/display_tsv.go @@ -0,0 +1,72 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "encoding/csv" + "os" +) + +type displayTSV struct { + writer *csv.Writer + disableHeader bool +} + +func NewDisplayTSV(file *os.File, separator rune, disableHeader bool) DisplayHandler { + tsv := &displayTSV{ + disableHeader: disableHeader, + } + + tsv.writer = csv.NewWriter(file) + + tsv.writer.Comma = separator + + return tsv +} + +func (d *displayTSV) DisplayLinks(doc *Doc) error { + if !d.disableHeader { + record := linkHeaderRecord() + if err := d.writer.Write(record); err != nil { + return err + } + } + + for _, linkList := range doc.Links { + for _, link := range linkList { + record := linkToRecord(link) + + if err := d.writer.Write(record); err != nil { + return err + } + } + } + + d.writer.Flush() + + return d.writer.Error() +} + +func (d *displayTSV) DisplayHeadings(doc *Doc) error { + if !d.disableHeader { + record := headingHeaderRecord() + if err := d.writer.Write(record); err != nil { + return err + } + } + + for _, l := range doc.Headings { + record := headingToRecord(l) + + if err := d.writer.Write(record); err != nil { + return err + } + } + + d.writer.Flush() + + return d.writer.Error() +} diff --git a/tests/cmd/check-markdown/doc.go b/tests/cmd/check-markdown/doc.go new file mode 100644 index 0000000000..a4d0efead3 --- /dev/null +++ b/tests/cmd/check-markdown/doc.go @@ -0,0 +1,76 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "errors" + "fmt" + + "github.com/sirupsen/logrus" +) + +// Details of the main document, and all other documents it references. +// Key: document name. +var docs map[string]*Doc + +func init() { + docs = make(map[string]*Doc) +} + +// newDoc creates a new document. +func newDoc(name string, logger *logrus.Entry) *Doc { + d := &Doc{ + Name: name, + Headings: make(map[string]Heading), + Links: make(map[string][]Link), + Parsed: false, + ShowTOC: false, + Logger: logger, + } + + d.Logger = logger.WithField("file", d.Name) + + // add to the hash + docs[name] = d + + return d +} + +// getDoc returns the Doc structure represented by the specified name, +// creating it and adding to the docs map if necessary. +func getDoc(name string, logger *logrus.Entry) (*Doc, error) { + if name == "" { + return &Doc{}, errors.New("need doc name") + } + + doc, ok := docs[name] + if ok { + return doc, nil + } + + return newDoc(name, logger), nil +} + +// hasHeading returns true if the specified heading exists for the document. +func (d *Doc) hasHeading(name string) bool { + return d.heading(name) != nil +} + +// Errorf is a convenience function to generate an error for this particular +// document. +func (d *Doc) Errorf(format string, args ...interface{}) error { + s := fmt.Sprintf(format, args...) + + return fmt.Errorf("file=%q: %s", d.Name, s) +} + +// String "pretty-prints" the specified document +// +// Just display the name as that is enough in text output. +func (d *Doc) String() string { + return d.Name +} diff --git a/tests/cmd/check-markdown/extract.go b/tests/cmd/check-markdown/extract.go new file mode 100644 index 0000000000..247bbdfbd3 --- /dev/null +++ b/tests/cmd/check-markdown/extract.go @@ -0,0 +1,93 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + + bf "gopkg.in/russross/blackfriday.v2" +) + +// linkDescription extracts the description from the specified link node. +func linkDescription(l *bf.Node) (string, error) { + if err := checkNode(l, bf.Link); err != nil { + return "", err + } + + // A link description can be comprised of various elements so scan + // through them to build up the final value. + + text := "" + node := l.FirstChild + + for node != nil { + switch node.Type { + case bf.Code: + text += string(node.Literal) + case bf.Text: + text += string(node.Literal) + default: + logger.WithField("node", node).Debug("ignoring node") + } + + if node == l.LastChild { + break + } + + node = node.Next + } + + return text, nil +} + +// headingName extracts the heading name from the specified Heading node in +// plain text, and markdown. The latter is used for creating TOC's which need +// to include the original markdown value. +func headingName(h *bf.Node) (name, mdName string, err error) { + if err = checkNode(h, bf.Heading); err != nil { + return "", "", err + } + + // A heading can be comprised of various elements so scan + // through them to build up the final value. + + node := h.FirstChild + + for node != nil { + switch node.Type { + case bf.Code: + value := string(node.Literal) + + name += value + mdName += fmt.Sprintf("`%s`", value) + case bf.Text: + value := string(node.Literal) + + name += value + mdName += value + case bf.Link: + // yep, people do crazy things like adding links into titles! + descr, err := linkDescription(node) + if err != nil { + return "", "", err + } + + name += descr + mdName += descr + default: + logger.WithField("node", node).Debug("ignoring node") + } + + if node == h.LastChild { + break + } + + node = node.Next + } + + return name, mdName, nil +} diff --git a/tests/cmd/check-markdown/hack.go b/tests/cmd/check-markdown/hack.go new file mode 100644 index 0000000000..ee65cc3a9a --- /dev/null +++ b/tests/cmd/check-markdown/hack.go @@ -0,0 +1,69 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "strings" + + bf "gopkg.in/russross/blackfriday.v2" +) + +// forceCreateHeadings extracts "missed" headings from the specified node, +// returning a slice of the newly headings created (which need to be added by the +// caller). +// +// Alas, Black Friday isn't 100% reliable... +func (d *Doc) forceCreateHeadings(node *bf.Node) ([]Heading, error) { + if err := checkNode(node, bf.Text); err != nil { + return []Heading{}, err + } + + chunk := string(node.Literal) + + if chunk == "" { + // No text in this node + return []Heading{}, nil + } + + lines := strings.Split(chunk, "\n") + if len(lines) <= 1 { + // No headings lurking in this text node + return []Heading{}, nil + } + + var headings []Heading + + for _, line := range lines { + if !strings.HasPrefix(line, anchorPrefix) { + continue + } + + fields := strings.Split(line, anchorPrefix) + name := strings.Join(fields, "") + name = strings.TrimSpace(name) + + count := strings.Count(line, anchorPrefix) + + heading := Heading{ + Name: name, + Level: count, + } + + id, err := createHeadingID(heading.Name) + if err != nil { + return []Heading{}, err + } + + heading.LinkName = id + + headings = append(headings, heading) + + extraHeadings++ + } + + return headings, nil +} diff --git a/tests/cmd/check-markdown/heading.go b/tests/cmd/check-markdown/heading.go new file mode 100644 index 0000000000..31eec470e6 --- /dev/null +++ b/tests/cmd/check-markdown/heading.go @@ -0,0 +1,36 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import "fmt" + +// newHeading creates a new Heading. +func newHeading(name, mdName string, level int) (Heading, error) { + if name == "" { + return Heading{}, fmt.Errorf("heading name cannot be blank") + } + + if mdName == "" { + return Heading{}, fmt.Errorf("heading markdown name cannot be blank") + } + + linkName, err := createHeadingID(name) + if err != nil { + return Heading{}, err + } + + if level < 1 { + return Heading{}, fmt.Errorf("level needs to be atleast 1") + } + + return Heading{ + Name: name, + MDName: mdName, + LinkName: linkName, + Level: level, + }, nil +} diff --git a/tests/cmd/check-markdown/heading_test.go b/tests/cmd/check-markdown/heading_test.go new file mode 100644 index 0000000000..2d5d1a1296 --- /dev/null +++ b/tests/cmd/check-markdown/heading_test.go @@ -0,0 +1,65 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewHeading(t *testing.T) { + assert := assert.New(t) + + type testData struct { + headingName string + mdName string + expectedLinkName string + level int + expectError bool + } + + data := []testData{ + {"", "", "", -1, true}, + {"a", "", "", -1, true}, + {"a", "a", "", -1, true}, + {"a", "a", "", 0, true}, + {"a", "", "", 1, true}, + + {"a", "a", "a", 1, false}, + {"a-b", "`a-b`", "`a-b`", 1, false}, + {"a_b", "`a_b`", "`a_b`", 1, false}, + {"foo (json) bar", "foo `(json)` bar", "foo-json-bar", 1, false}, + {"func(json)", "`func(json)`", "funcjson", 1, false}, + {"?", "?", "", 1, false}, + {"a b", "a b", "a-b", 1, false}, + {"a - b", "a - b", "a---b", 1, false}, + {"a - b?", "a - b?", "a---b", 1, false}, + {"a - b.", "a - b.", "a---b", 1, false}, + {"a:b", "a:b", "ab", 1, false}, + {"a;b", "a;b", "ab", 1, false}, + {"a@b", "a@b", "ab", 1, false}, + {"a+b", "a+b", "ab", 1, false}, + {"a,b", "a,b", "ab", 1, false}, + } + + for i, d := range data { + msg := fmt.Sprintf("test[%d]: %+v\n", i, d) + + h, err := newHeading(d.headingName, d.mdName, d.level) + if d.expectError { + assert.Error(err, msg) + continue + } + + assert.Equal(h.Name, d.headingName, msg) + assert.Equal(h.MDName, d.mdName, msg) + assert.Equal(h.Level, d.level, msg) + assert.Equal(h.LinkName, d.expectedLinkName, msg) + } +} diff --git a/tests/cmd/check-markdown/link.go b/tests/cmd/check-markdown/link.go new file mode 100644 index 0000000000..fe6e5cca4c --- /dev/null +++ b/tests/cmd/check-markdown/link.go @@ -0,0 +1,122 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "errors" + "os" + "path/filepath" + "regexp" + "strings" +) + +// newLink creates a new Link. +func newLink(doc *Doc, address, description string) (Link, error) { + l := Link{ + Doc: doc, + Address: address, + Description: description, + } + + err := l.categorise() + if err != nil { + return Link{}, err + } + + return l, nil +} + +// categorise determines the type of Link. +func (l *Link) categorise() error { + address := l.Address + + // markdown file extension with optional link name ("#...") + const re = `\.md#*.*$` + + pattern := regexp.MustCompile(re) + + matched := pattern.MatchString(address) + + if strings.HasPrefix(address, "http:") { + l.Type = urlLink + } else if strings.HasPrefix(address, "https:") { + l.Type = urlLink + } else if strings.HasPrefix(address, "mailto:") { + l.Type = mailLink + } else if strings.HasPrefix(address, anchorPrefix) { + l.Type = internalLink + + // Remove the prefix to make a valid link address + address = strings.TrimPrefix(address, anchorPrefix) + l.Address = address + } else if matched { + l.Type = externalLink + + file, _, err := splitLink(address) + if err != nil { + return err + } + + file, err = l.Doc.linkAddrToPath(file) + if err != nil { + return err + } + + l.ResolvedPath = file + } else { + isREADME, err := l.handleImplicitREADME() + if err != nil { + return err + } + + if !isREADME { + // Link must be an external file, but not a markdown file. + l.Type = externalFile + } + } + + return nil +} + +// handleImplicitREADME determines if the specified link is an implicit link +// to a README document. +func (l *Link) handleImplicitREADME() (isREADME bool, err error) { + const readme = "README.md" + + address := l.Address + if address == "" { + return false, errors.New("need link address") + } + + file, err := l.Doc.linkAddrToPath(address) + if err != nil { + return false, err + } + + // The resolved path should exist as this is a local file. + st, err := os.Stat(file) + if err != nil { + return false, err + } + + if !st.IsDir() { + return false, nil + } + + // The file is a directory so try appending the implicit README file + // and see if that exists. + resolvedPath := filepath.Join(file, readme) + + success := fileExists(resolvedPath) + + if success { + l.Type = externalLink + l.ResolvedPath = resolvedPath + } + + return success, nil +} diff --git a/tests/cmd/check-markdown/link_test.go b/tests/cmd/check-markdown/link_test.go new file mode 100644 index 0000000000..c3651a2a0b --- /dev/null +++ b/tests/cmd/check-markdown/link_test.go @@ -0,0 +1,209 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +// createLinkAndCategorise will create a link and categorise it. If +// createLinkManually is set, the link will be created "manually" (without the +// constructor) and categorise() called. If not set, the constructor will be +// used. +func createLinkAndCategorise(assert *assert.Assertions, createLinkManually bool) { + dir, err := os.MkdirTemp("", "") + assert.NoError(err) + + cwd, err := os.Getwd() + assert.NoError(err) + defer os.Chdir(cwd) + + err = os.Chdir(dir) + assert.NoError(err) + defer os.RemoveAll(dir) + + readmeDirName := "dir-with-readme" + emptyDirName := "empty" + makeDirs(assert, dir, readmeDirName, emptyDirName) + + readmeDirPath := filepath.Join(readmeDirName, readmeName) + + topLevelReadmeName := "top-level.md" + topLevelReadmeLink := filepath.Join("/", topLevelReadmeName) + + topLevelReadmePath := filepath.Join(dir, topLevelReadmeName) + + type testData struct { + linkAddress string + + expectedPath string + + expectedType LinkType + expectError bool + + // Set if expectedPath should be checked + checkPath bool + } + + docRoot = dir + + data := []testData{ + {"", "", -1, true, false}, + {"a", "", -1, true, false}, + {"a.b", "", -1, true, false}, + {"a#b", "", -1, true, false}, + + {"htt://foo", "", -1, true, false}, + {"HTTP://foo", "", -1, true, false}, + {"moohttp://foo", "", -1, true, false}, + {"mailto", "", -1, true, false}, + {"http", "", -1, true, false}, + {"https", "", -1, true, false}, + + {"http://foo", "", urlLink, false, false}, + {"https://foo/", "", urlLink, false, false}, + {"https://foo/bar", "", urlLink, false, false}, + {"mailto:me", "", mailLink, false, false}, + + {".", "", externalFile, false, false}, + {"/", "", externalFile, false, false}, + {emptyDirName, "", externalFile, false, false}, + + {readmeDirName, readmeDirPath, externalLink, false, true}, + {"foo.md", "foo.md", externalLink, false, true}, + {"foo.md#bar", "foo.md", externalLink, false, true}, + {topLevelReadmeLink, topLevelReadmePath, externalLink, false, true}, + } + + logger := logrus.WithField("test", "true") + description := "" + + for i, d := range data { + var link Link + var err error + + doc := newDoc("foo", logger) + + if createLinkManually { + link = Link{ + Doc: doc, + Address: d.linkAddress, + Description: description, + } + + err = link.categorise() + } else { + link, err = newLink(doc, d.linkAddress, description) + } + + msg := fmt.Sprintf("test[%d] manual-link: %v: %+v, link: %+v\n", i, createLinkManually, d, link) + + if d.expectError { + assert.Error(err, msg) + continue + } + + assert.NoError(err, msg) + + assert.Equal(link.Doc, doc) + assert.Equal(link.Address, d.linkAddress) + assert.Equal(link.Description, description) + assert.Equal(link.Type, d.expectedType) + + if d.checkPath { + assert.Equal(d.expectedPath, link.ResolvedPath) + } + } +} + +func TestNewLink(t *testing.T) { + assert := assert.New(t) + + createLinkAndCategorise(assert, false) +} + +func TestLinkCategorise(t *testing.T) { + assert := assert.New(t) + + createLinkAndCategorise(assert, true) +} + +func TestLinkHandleImplicitREADME(t *testing.T) { + assert := assert.New(t) + + dir, err := os.MkdirTemp("", "") + assert.NoError(err) + defer os.RemoveAll(dir) + + cwd, err := os.Getwd() + assert.NoError(err) + defer os.Chdir(cwd) + + err = os.Chdir(dir) + assert.NoError(err) + defer os.RemoveAll(dir) + + readmeDirName := "dir-with-readme" + emptyDirName := "empty" + makeDirs(assert, dir, readmeDirName, emptyDirName) + + readmePath := filepath.Join(readmeDirName, readmeName) + + emptyFileName := "empty-file" + + err = createFile(emptyFileName, "") + assert.NoError(err) + + type testData struct { + linkAddr string + expectedPath string + expectedType LinkType + isREADME bool + expectError bool + } + + data := []testData{ + {"", "", unknownLink, false, true}, + {"foo", "", unknownLink, false, true}, + {emptyFileName, "", unknownLink, false, false}, + {emptyDirName, "", unknownLink, false, false}, + {readmeDirName, readmePath, externalLink, true, false}, + } + + logger := logrus.WithField("test", "true") + + for i, d := range data { + doc := newDoc("foo", logger) + + link := Link{ + Doc: doc, + Address: d.linkAddr, + } + + msg := fmt.Sprintf("test[%d]: %+v\n", i, d) + + isREADME, err := link.handleImplicitREADME() + + if d.expectError { + assert.Error(err, msg) + continue + } + + assert.NoError(err, msg) + assert.Equal(isREADME, d.isREADME) + assert.Equal(isREADME, d.isREADME) + assert.Equal(link.Address, d.linkAddr) + assert.Equal(link.Type, d.expectedType) + assert.Equal(link.ResolvedPath, d.expectedPath) + } +} diff --git a/tests/cmd/check-markdown/main.go b/tests/cmd/check-markdown/main.go new file mode 100644 index 0000000000..e626a95d9d --- /dev/null +++ b/tests/cmd/check-markdown/main.go @@ -0,0 +1,348 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "errors" + "fmt" + "os" + "time" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +type DataToShow int + +const ( + // Character used (after an optional filename) before a heading ID. + anchorPrefix = "#" + + // Character used to signify an "absolute link path" which should + // expand to the value of the document root. + absoluteLinkPrefix = "/" + + showLinks DataToShow = iota + showHeadings DataToShow = iota + + textFormat = "text" + tsvFormat = "tsv" + defaultOutputFormat = textFormat + defaultSeparator = "\t" +) + +var ( + // set by the build + name = "" + version = "" + commit = "" + + strict = false + + // list entry character to use when generating TOCs + listPrefix = "*" + + logger *logrus.Entry + + errNeedFile = errors.New("need markdown file") +) + +// Black Friday sometimes chokes on markdown (I know!!), so record how many +// extra headings we found. +var extraHeadings int + +// Root directory used to handle "absolute link paths" that start with a slash +// to denote the "top directory", like this: +// +// [Foo](/absolute-link.md) +var docRoot string + +var notes = fmt.Sprintf(` + +NOTES: + +- The document root is used to handle markdown references that begin with %q, + denoting that the path that follows is an "absolute path" from the specified + document root path. + +- The order the document nodes are parsed internally is not known to + this program. This means that if multiple errors exist in the document, + running this tool multiple times will error one *one* of the errors, but not + necessarily the same one as last time. + +LIMITATIONS: + +- The default document root only works if this tool is run from the top-level + of a repository. + +`, absoluteLinkPrefix) + +var formatFlag = cli.StringFlag{ + Name: "format", + Usage: "display in specified format ('help' to show all)", + Value: defaultOutputFormat, +} + +var separatorFlag = cli.StringFlag{ + Name: "separator", + Usage: fmt.Sprintf("use the specified separator character (%s format only)", tsvFormat), + Value: defaultSeparator, +} + +var noHeaderFlag = cli.BoolFlag{ + Name: "no-header", + Usage: "disable display of header (if format supports one)", +} + +func init() { + logger = logrus.WithFields(logrus.Fields{ + "name": name, + "source": "check-markdown", + "version": version, + "commit": commit, + "pid": os.Getpid(), + }) + + logger.Logger.Formatter = &logrus.TextFormatter{ + TimestampFormat: time.RFC3339Nano, + //DisableColors: true, + } + + // Write to stdout to avoid upsetting CI systems that consider stderr + // writes as indicating an error. + logger.Logger.Out = os.Stdout +} + +func handleLogging(c *cli.Context) { + logLevel := logrus.InfoLevel + + if c.GlobalBool("debug") { + logLevel = logrus.DebugLevel + } + + logger.Logger.SetLevel(logLevel) +} + +func handleDoc(c *cli.Context, createTOC bool) error { + handleLogging(c) + + if c.NArg() == 0 { + return errNeedFile + } + + fileName := c.Args().First() + if fileName == "" { + return errNeedFile + } + + singleDocOnly := c.GlobalBool("single-doc-only") + + doc := newDoc(fileName, logger) + doc.ShowTOC = createTOC + + if createTOC { + // Only makes sense to generate a single TOC! + singleDocOnly = true + } + + // Parse the main document first + err := doc.parse() + if err != nil { + return err + } + + if singleDocOnly && len(docs) > 1 { + doc.Logger.Debug("Not checking referenced files at user request") + return nil + } + + // Now handle all other docs that the main doc references. + // This requires care to avoid recursion. + for { + count := len(docs) + parsed := 0 + for _, doc := range docs { + if doc.Parsed { + // Document has already been handled + parsed++ + continue + } + + if err := doc.parse(); err != nil { + return err + } + } + + if parsed == count { + break + } + } + + err = handleIntraDocLinks() + if err != nil { + return err + } + + if !createTOC { + doc.Logger.Info("Checked file") + doc.showStats() + } + + count := len(docs) + + if count > 1 { + // Update to ignore main document + count-- + + doc.Logger.WithField("reference-document-count", count).Info("Checked referenced files") + + for _, d := range docs { + if d.Name == doc.Name { + // Ignore main document + continue + } + + fmt.Printf("\t%q\n", d.Name) + } + } + + // Highlight blackfriday deficiencies + if !doc.ShowTOC && extraHeadings > 0 { + doc.Logger.WithField("extra-heading-count", extraHeadings).Debug("Found extra headings") + } + + return nil +} + +// commonListHandler is used to handle all list operations. +func commonListHandler(context *cli.Context, what DataToShow) error { + handleLogging(context) + + handlers := NewDisplayHandlers(context.String("separator"), context.Bool("no-header")) + + format := context.String("format") + if format == "help" { + availableFormats := handlers.Get() + + for _, format := range availableFormats { + fmt.Fprintf(outputFile, "%s\n", format) + } + + return nil + } + + handler := handlers.find(format) + if handler == nil { + return fmt.Errorf("no handler for format %q", format) + } + + if context.NArg() == 0 { + return errNeedFile + } + + file := context.Args().Get(0) + + return show(file, logger, handler, what) +} + +func realMain() error { + cwd, err := os.Getwd() + if err != nil { + return err + } + + docRoot = cwd + + cli.VersionPrinter = func(c *cli.Context) { + fmt.Fprintln(os.Stdout, c.App.Version) + } + + cli.AppHelpTemplate = fmt.Sprintf(`%s%s`, cli.AppHelpTemplate, notes) + + app := cli.NewApp() + app.Name = name + app.Version = fmt.Sprintf("%s %s (commit %v)", name, version, commit) + app.Description = "Tool to check GitHub-Flavoured Markdown (GFM) format documents" + app.Usage = app.Description + app.UsageText = fmt.Sprintf("%s [options] file ...", app.Name) + app.Flags = []cli.Flag{ + cli.BoolFlag{ + Name: "debug, d", + Usage: "display debug information", + }, + cli.StringFlag{ + Name: "doc-root, r", + Usage: "specify document root", + Value: docRoot, + }, + cli.BoolFlag{ + Name: "single-doc-only, o", + Usage: "only check primary (specified) document", + }, + cli.BoolFlag{ + Name: "strict, s", + Usage: "enable strict mode", + }, + } + + app.Commands = []cli.Command{ + { + Name: "check", + Usage: "perform tests on the specified document", + Description: "Exit code denotes success", + Action: func(c *cli.Context) error { + return handleDoc(c, false) + }, + }, + { + Name: "toc", + Usage: "display a markdown Table of Contents", + Action: func(c *cli.Context) error { + return handleDoc(c, true) + }, + }, + { + Name: "list", + Usage: "display particular parts of the document", + Subcommands: []cli.Command{ + { + Name: "headings", + Usage: "display headings", + Flags: []cli.Flag{ + formatFlag, + noHeaderFlag, + separatorFlag, + }, + Action: func(c *cli.Context) error { + return commonListHandler(c, showHeadings) + }, + }, + { + Name: "links", + Usage: "display links", + Flags: []cli.Flag{ + formatFlag, + noHeaderFlag, + separatorFlag, + }, + Action: func(c *cli.Context) error { + return commonListHandler(c, showLinks) + }, + }, + }, + }, + } + + return app.Run(os.Args) +} + +func main() { + err := realMain() + if err != nil { + logger.Fatalf("%v", err) + } +} diff --git a/tests/cmd/check-markdown/node.go b/tests/cmd/check-markdown/node.go new file mode 100644 index 0000000000..5d8d104a36 --- /dev/null +++ b/tests/cmd/check-markdown/node.go @@ -0,0 +1,115 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + bf "gopkg.in/russross/blackfriday.v2" +) + +// handleNode processes the specified node. +func (d *Doc) handleNode(node *bf.Node) error { + var err error + + switch node.Type { + case bf.Heading: + err = d.handleHeading(node) + case bf.Link: + err = d.handleLink(node) + case bf.Text: + // handle blackfriday deficiencies + headings, err := d.forceCreateHeadings(node) + if err != nil { + return err + } + + for _, heading := range headings { + err := d.addHeading(heading) + if err != nil { + return err + } + } + + default: + return nil + } + + return err +} + +// makeHeading creates a heading from the specified node. +func (d *Doc) makeHeading(node *bf.Node) (Heading, error) { + if err := checkNode(node, bf.Heading); err != nil { + return Heading{}, err + } + + name, mdName, err := headingName(node) + if err != nil { + return Heading{}, d.Errorf("failed to get heading name: %v", err) + } + + data := node.HeadingData + + heading, err := newHeading(name, mdName, data.Level) + if err != nil { + return Heading{}, err + } + + return heading, nil +} + +// handleHeading processes the heading represented by the specified node. +func (d *Doc) handleHeading(node *bf.Node) error { + if err := checkNode(node, bf.Heading); err != nil { + return err + } + + heading, err := d.makeHeading(node) + if err != nil { + return err + } + + return d.addHeading(heading) +} + +func (d *Doc) handleLink(node *bf.Node) error { + if err := checkNode(node, bf.Link); err != nil { + return err + } + + address := string(node.Destination) + + description, err := linkDescription(node) + if err != nil { + return d.Errorf("failed to get link name: %v", err) + } + + link, err := newLink(d, address, description) + if err != nil { + return err + } + + return d.addLink(link) +} + +// handleIntraDocLinks checks the links between documents are correct. +// +// For example, if a document refers to "foo.md#section-bar", this function +// will ensure that "section-bar" exists in external file "foo.md". +func handleIntraDocLinks() error { + for _, doc := range docs { + for addr, linkList := range doc.Links { + for _, link := range linkList { + err := doc.checkLink(addr, link, true) + if err != nil { + return doc.Errorf("intra-doc link invalid: %v", err) + } + } + } + } + + return nil +} diff --git a/tests/cmd/check-markdown/parse.go b/tests/cmd/check-markdown/parse.go new file mode 100644 index 0000000000..930a3e8926 --- /dev/null +++ b/tests/cmd/check-markdown/parse.go @@ -0,0 +1,100 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + "os" + "strings" + + bf "gopkg.in/russross/blackfriday.v2" +) + +// List of errors found by visitor. Used as the visitor cannot return an error +// directly. +var errorList []error + +func (d *Doc) parse() error { + if !d.ShowTOC && !d.ListMode { + d.Logger.Info("Checking file") + } + + err := d.parseMarkdown() + if err != nil { + return err + } + + // mark document as having been handled + d.Parsed = true + + return nil +} + +// parseMarkdown parses the documents markdown. +func (d *Doc) parseMarkdown() error { + bytes, err := os.ReadFile(d.Name) + if err != nil { + return err + } + + md := bf.New(bf.WithExtensions(bf.CommonExtensions)) + + root := md.Parse(bytes) + + root.Walk(makeVisitor(d, d.ShowTOC)) + + errorCount := len(errorList) + if errorCount > 0 { + extra := "" + if errorCount != 1 { + extra = "s" + } + + var msg []string + + for _, err := range errorList { + msg = append(msg, err.Error()) + } + + return fmt.Errorf("found %d parse error%s:\n%s", + errorCount, + extra, + strings.Join(msg, "\n")) + } + + return d.check() +} + +// makeVisitor returns a function that is used to visit all document nodes. +// +// If createTOC is false, the visitor will check all nodes, but if true, the +// visitor will only display a table of contents for the document. +func makeVisitor(doc *Doc, createTOC bool) func(node *bf.Node, entering bool) bf.WalkStatus { + f := func(node *bf.Node, entering bool) bf.WalkStatus { + if !entering { + return bf.GoToNext + } + + var err error + + if createTOC { + err = doc.displayTOC(node) + } else { + err = doc.handleNode(node) + } + + if err != nil { + // The visitor cannot return an error, so collect up all parser + // errors for dealing with later. + errorList = append(errorList, err) + } + + return bf.GoToNext + } + + return f +} diff --git a/tests/cmd/check-markdown/record.go b/tests/cmd/check-markdown/record.go new file mode 100644 index 0000000000..72be75a7b1 --- /dev/null +++ b/tests/cmd/check-markdown/record.go @@ -0,0 +1,43 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import "fmt" + +func linkHeaderRecord() []string { + return []string{ + "Document", + "Address", + "Path", + "Description", + "Type", + } +} + +func linkToRecord(l Link) (record []string) { + record = append(record, l.Doc.Name) + record = append(record, l.Address) + record = append(record, l.ResolvedPath) + record = append(record, l.Description) + record = append(record, l.Type.String()) + + return record +} + +func headingHeaderRecord() []string { + return []string{ + "Name", + "Link", + "Level", + } +} +func headingToRecord(h Heading) (record []string) { + record = append(record, h.Name) + record = append(record, h.LinkName) + record = append(record, fmt.Sprintf("%d", h.Level)) + + return record +} diff --git a/tests/cmd/check-markdown/search.go b/tests/cmd/check-markdown/search.go new file mode 100644 index 0000000000..d310748a4b --- /dev/null +++ b/tests/cmd/check-markdown/search.go @@ -0,0 +1,29 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +// headingByLinkName returns the heading associated with the specified link name. +func (d *Doc) headingByLinkName(linkName string) *Heading { + for _, heading := range d.Headings { + if heading.LinkName == linkName { + return &heading + } + } + + return nil +} + +// heading returns the heading with the name specified. +func (d *Doc) heading(name string) *Heading { + for _, heading := range d.Headings { + if name == heading.LinkName { + return &heading + } + } + + return nil +} diff --git a/tests/cmd/check-markdown/stats.go b/tests/cmd/check-markdown/stats.go new file mode 100644 index 0000000000..a73a4a09bd --- /dev/null +++ b/tests/cmd/check-markdown/stats.go @@ -0,0 +1,41 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + + "github.com/sirupsen/logrus" +) + +func (d *Doc) showStats() { + var counters [LinkTypeCount]int + + linkCount := 0 + + for _, linkList := range d.Links { + for _, link := range linkList { + counters[link.Type]++ + linkCount++ + } + } + + fields := logrus.Fields{ + "headings-count": len(d.Headings), + "links-count": linkCount, + } + + for i, count := range counters { + name := LinkType(i).String() + + fieldName := fmt.Sprintf("link-type-%s-count", name) + + fields[fieldName] = count + } + + d.Logger.WithFields(fields).Info("Statistics") +} diff --git a/tests/cmd/check-markdown/toc.go b/tests/cmd/check-markdown/toc.go new file mode 100644 index 0000000000..40eb5f8e39 --- /dev/null +++ b/tests/cmd/check-markdown/toc.go @@ -0,0 +1,75 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + "strings" + + bf "gopkg.in/russross/blackfriday.v2" +) + +// displayTOC displays a table of contents entry for the specified node. +func (d *Doc) displayTOC(node *bf.Node) error { + switch node.Type { + case bf.Heading: + return d.displayTOCEntryFromNode(node) + case bf.Text: + // handle blackfriday deficiencies + headings, err := d.forceCreateHeadings(node) + if err != nil { + return err + } + + for _, heading := range headings { + err := d.displayTOCEntryFromHeading(heading) + if err != nil { + return err + } + } + } + + return nil +} + +// displayTOCEntryFromHeading displays a table of contents entry +// for the specified heading. +func (d *Doc) displayTOCEntryFromHeading(heading Heading) error { + const indentSpaces = 4 + + prefix := "" + + level := heading.Level + + // Indent needs to be zero for top level headings + level-- + + if level > 0 { + prefix = strings.Repeat(" ", level*indentSpaces) + } + + entry := fmt.Sprintf("[%s](%s%s)", heading.MDName, anchorPrefix, heading.LinkName) + + fmt.Printf("%s%s %s\n", prefix, listPrefix, entry) + + return nil +} + +// displayTOCEntryFromHeading displays a table of contents entry +// for the specified heading. +func (d *Doc) displayTOCEntryFromNode(node *bf.Node) error { + if err := checkNode(node, bf.Heading); err != nil { + return err + } + + heading, err := d.makeHeading(node) + if err != nil { + return err + } + + return d.displayTOCEntryFromHeading(heading) +} diff --git a/tests/cmd/check-markdown/types.go b/tests/cmd/check-markdown/types.go new file mode 100644 index 0000000000..61b9a503ca --- /dev/null +++ b/tests/cmd/check-markdown/types.go @@ -0,0 +1,159 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import "github.com/sirupsen/logrus" + +// LinkType represents the type of a link in a markdown document. +type LinkType int + +const ( + unknownLink LinkType = iota + internalLink LinkType = iota + externalLink LinkType = iota // External ".md" file + externalFile LinkType = iota // External non-".md" file + urlLink LinkType = iota + mailLink LinkType = iota + LinkTypeCount LinkType = iota +) + +func (t LinkType) String() string { + var name string + + switch t { + case unknownLink: + name = "unknown" + case internalLink: + name = "internal-link" + case externalLink: + name = "external-link" + case externalFile: + name = "external-file" + case urlLink: + name = "url-link" + case mailLink: + name = "mail-link" + } + + return name +} + +// Heading is a markdown heading, which might be the destination +// for a link. +// +// Example: A heading like this: +// +// ### This is a `verbatim` heading +// +// ... would be described as: +// +// ```go +// +// Heading{ +// Name: "This is a verbatim heading", +// MDName "This is a `verbatim` heading", +// LinkName: "this-is-a-verbatim-heading", +// Level: 3, +// } +// +// ``` +type Heading struct { + // Not strictly necessary since the name is used as a hash key. + // However, storing here too makes the code simpler ;) + Name string + + // Name including any markdown syntax + MDName string + + // The encoded value of Name. + LinkName string + + // Heading level (1 for top level) + Level int +} + +// Link is a reference to another part of this document +// (or another document). +// +// Example: A link like this: +// +// [internal link](#internal-section-name) +// +// ... would be described as: +// +// ```go +// +// Link{ +// Address: "internal-section-name", +// ResolvedPath: "", +// Description: "internal link", +// Type: internalLink, +// } +// +// And a link like this: +// +// [external link](/foo.md#section-name) +// +// ... would be described as: +// +// ```go +// +// Link{ +// Address: "foo.md#section-name", +// ResolvedPath: "/docroot/foo.md", +// Description: "external link", +// Type: externalLink, +// } +// +// ``` +type Link struct { + // Document this link refers to. + Doc *Doc + + // Original address from document. + // + // Must be a valid Heading.LinkName. + // + // Not strictly necessary since the address is used as a hash key. + // However, storing here too makes the code simpler ;) + Address string + + // The fully expanded address, without any anchor and heading suffix. + // + // Only applies to certain link types. + ResolvedPath string + + // The text the user sees for the hyperlink address + Description string + + Type LinkType +} + +// Doc represents a markdown document. +type Doc struct { + Logger *logrus.Entry + + // Key: heading name + // Value: Heading + Headings map[string]Heading + + // Key: link address + // Value: *list* of links. Required since you can have multiple links with + // the same _address_, but of a different type. + Links map[string][]Link + + // Filename + Name string + + // true when this document has been fully parsed + Parsed bool + + // if true, only show the Table Of Contents + ShowTOC bool + + ListMode bool +} diff --git a/tests/cmd/check-markdown/utils.go b/tests/cmd/check-markdown/utils.go new file mode 100644 index 0000000000..bd6d415412 --- /dev/null +++ b/tests/cmd/check-markdown/utils.go @@ -0,0 +1,97 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "errors" + "fmt" + "os" + "strings" + "unicode" + + bf "gopkg.in/russross/blackfriday.v2" +) + +// fileExists returns true if the specified file exists, else false. +func fileExists(path string) bool { + if _, err := os.Stat(path); os.IsNotExist(err) { + return false + } + + return true +} + +// splitLink splits a link like "foo.md#section-name" into a filename +// ("foo.md") and a section name ("section-name"). +func splitLink(linkName string) (fileName, sectionName string, err error) { + if linkName == "" { + return "", "", errors.New("need linkName") + } + + if !strings.Contains(linkName, anchorPrefix) { + return linkName, "", nil + } + + fields := strings.Split(linkName, anchorPrefix) + + expectedFields := 2 + foundFields := len(fields) + if foundFields != expectedFields { + + return "", "", fmt.Errorf("invalid link %s: expected %d fields, found %d", linkName, expectedFields, foundFields) + } + + fileName = fields[0] + sectionName = fields[1] + + return fileName, sectionName, nil +} + +// validHeadingIDChar is a strings.Map() function used to determine which characters +// can appear in a heading ID. +func validHeadingIDChar(r rune) rune { + if unicode.IsLetter(r) || + unicode.IsNumber(r) || + unicode.IsSpace(r) || + r == '-' || r == '_' { + return r + } + + // Remove all other chars from destination string + return -1 +} + +// createHeadingID creates an HTML anchor name for the specified heading +func createHeadingID(headingName string) (id string, err error) { + if headingName == "" { + return "", fmt.Errorf("need heading name") + } + + // Munge the original heading into an id by: + // + // - removing invalid characters. + // - lower-casing. + // - replace spaces + id = strings.Map(validHeadingIDChar, headingName) + + id = strings.ToLower(id) + id = strings.Replace(id, " ", "-", -1) + + return id, nil +} + +func checkNode(node *bf.Node, expectedType bf.NodeType) error { + if node == nil { + return errors.New("node cannot be nil") + } + + if node.Type != expectedType { + return fmt.Errorf("expected %v node, found %v", expectedType, node.Type) + } + + return nil +} diff --git a/tests/cmd/check-markdown/utils_test.go b/tests/cmd/check-markdown/utils_test.go new file mode 100644 index 0000000000..c9899fcae5 --- /dev/null +++ b/tests/cmd/check-markdown/utils_test.go @@ -0,0 +1,149 @@ +// +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSplitLink(t *testing.T) { + assert := assert.New(t) + + type testData struct { + linkName string + file string + section string + valid bool + } + + data := []testData{ + {"", "", "", false}, + + {"foo.md", "foo.md", "", true}, + {"#bar", "", "bar", true}, + {"foo.md#bar", "foo.md", "bar", true}, + {"foo.md%%bar", "foo.md%%bar", "", true}, + } + + for i, d := range data { + file, section, err := splitLink(d.linkName) + + if d.valid { + assert.NoErrorf(err, "test[%d]: %+v", i, d) + assert.Equal(file, d.file, "test[%d]: %+v", i, d) + assert.Equal(section, d.section, "test[%d]: %+v", i, d) + } else { + assert.Errorf(err, "test[%d]: %+v", i, d) + } + } +} + +func TestValidHeadingIDChar(t *testing.T) { + assert := assert.New(t) + + type testData struct { + ch rune + valid bool + } + + data := []testData{ + {' ', true}, + {'\t', true}, + {'\n', true}, + + {'a', true}, + {'z', true}, + {'A', true}, + {'Z', true}, + + {'0', true}, + {'9', true}, + + {'-', true}, + {'_', true}, + + {'\000', false}, + {'\001', false}, + } + + for i, d := range data { + result := validHeadingIDChar(d.ch) + + var outcome bool + + if d.valid { + outcome = result != -1 + } else { + outcome = result == -1 + } + + assert.Truef(outcome, "test[%d]: %+v", i, d) + } + + // the main list of invalid chars to test + invalid := "!@#$%^&*()+=[]{}\\|:\";'<>?,./" + + for i, ch := range invalid { + result := validHeadingIDChar(ch) + + outcome := result == -1 + + assert.Truef(outcome, "invalid[%d]: %+v", i, ch) + } +} + +func TestCreateHeadingID(t *testing.T) { + assert := assert.New(t) + + type testData struct { + heading string + id string + expecteError bool + } + + data := []testData{ + {"", "", true}, + {"a", "a", false}, + {"a.b/c:d", "abcd", false}, + {"a ?", "a-", false}, + {"a !?!", "a-", false}, + {"foo", "foo", false}, + {"foo bar", "foo-bar", false}, + {"foo_bar", "foo_bar", false}, + {"foo_bar()", "foo_bar", false}, + {"`foo_bar()`", "foo_bar", false}, + {"foo_bar()baz", "foo_barbaz", false}, + {"Stability or Performance?", "stability-or-performance", false}, + {"Hello - World", "hello---world", false}, + {"metrics_json_init()", "metrics_json_init", false}, + {"metrics_json_add_array_element(json)", "metrics_json_add_array_elementjson", false}, + {"What is it ?", "what-is-it-", false}, + {"Sandbox `DeviceInfo`", "sandbox-deviceinfo", false}, + {"Build a custom QEMU for aarch64/arm64 - REQUIRED", "build-a-custom-qemu-for-aarch64arm64---required", false}, + {"docker --net=host", "docker---nethost", false}, + {"Containerd Runtime V2 API (Shim V2 API)", "containerd-runtime-v2-api-shim-v2-api", false}, + {"Containerd Runtime V2 API: Shim V2 API", "containerd-runtime-v2-api-shim-v2-api", false}, + {"Launch i3.metal instance", "launch-i3metal-instance", false}, + {"Deploy!", "deploy", false}, + } + + for i, d := range data { + id, err := createHeadingID(d.heading) + + msg := fmt.Sprintf("test[%d]: %+v, id: %q\n", i, d, id) + + if d.expecteError { + assert.Error(err) + continue + } + + assert.Equal(id, d.id, msg) + } +} From 7f3c12f1ddde070a664b385e9843ecdd8e581dda Mon Sep 17 00:00:00 2001 From: Chelsea Mafrica Date: Tue, 21 Nov 2023 17:47:16 -0800 Subject: [PATCH 3/6] tests: move spell check tool to main repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move tool as part of static checks migration. Fixes #8187 Signed-off-by: Bo Chen Signed-off-by: Carlos Venegas Signed-off-by: Chao Wu Signed-off-by: Chelsea Mafrica Signed-off-by: Dan Middleton Signed-off-by: Derek Lee Signed-off-by: Eric Ernst Signed-off-by: Fabiano Fidêncio Signed-off-by: Gabriela Cervantes Signed-off-by: Graham Whaley Signed-off-by: Hui Zhu Signed-off-by: James O. D. Hunt Signed-off-by: Jimmy Xu Signed-off-by: Liu Xiaodong Signed-off-by: Mikko Ylinen Signed-off-by: Shiming Zhang Signed-off-by: Snir Sheriber Signed-off-by: Wainer dos Santos Moschetta --- tests/cmd/check-spelling/README.md | 178 +++++++++ tests/cmd/check-spelling/data/acronyms.txt | 123 ++++++ tests/cmd/check-spelling/data/arches.txt | 21 + tests/cmd/check-spelling/data/distros.txt | 18 + tests/cmd/check-spelling/data/files.txt | 25 ++ tests/cmd/check-spelling/data/hunspell.txt | 13 + tests/cmd/check-spelling/data/main.txt | 135 +++++++ tests/cmd/check-spelling/data/projects.txt | 101 +++++ tests/cmd/check-spelling/data/rules.aff | 36 ++ tests/cmd/check-spelling/kata-dictionary.aff | 36 ++ tests/cmd/check-spelling/kata-dictionary.dic | 384 +++++++++++++++++++ tests/cmd/check-spelling/kata-spell-check.sh | 336 ++++++++++++++++ 12 files changed, 1406 insertions(+) create mode 100644 tests/cmd/check-spelling/README.md create mode 100644 tests/cmd/check-spelling/data/acronyms.txt create mode 100644 tests/cmd/check-spelling/data/arches.txt create mode 100644 tests/cmd/check-spelling/data/distros.txt create mode 100644 tests/cmd/check-spelling/data/files.txt create mode 100644 tests/cmd/check-spelling/data/hunspell.txt create mode 100644 tests/cmd/check-spelling/data/main.txt create mode 100644 tests/cmd/check-spelling/data/projects.txt create mode 100644 tests/cmd/check-spelling/data/rules.aff create mode 100644 tests/cmd/check-spelling/kata-dictionary.aff create mode 100644 tests/cmd/check-spelling/kata-dictionary.dic create mode 100755 tests/cmd/check-spelling/kata-spell-check.sh diff --git a/tests/cmd/check-spelling/README.md b/tests/cmd/check-spelling/README.md new file mode 100644 index 0000000000..b7b114f752 --- /dev/null +++ b/tests/cmd/check-spelling/README.md @@ -0,0 +1,178 @@ +# Spell check tool + +## Overview + +The `kata-spell-check.sh` tool is used to check a markdown file for +typographical (spelling) mistakes. + +## Approach + +The spell check tool is based on +[`hunspell`](https://github.com/hunspell/hunspell). It uses standard Hunspell +English dictionaries and supplements these with a custom Hunspell dictionary. +The document is cleaned of several entities before the spell-check begins. +These entities include the following: + +- URLs +- Email addresses +- Code blocks +- Most punctuation +- GitHub userids + +## Custom words + +A custom dictionary is required to accept specific words that are either well +understood by the community or are defined in various document files, but do +not appear in standard dictionaries. The custom dictionaries allow those words +to be accepted as correct. The following lists common examples of such words: + +- Abbreviations +- Acronyms +- Company names +- Product names +- Project names +- Technical terms + +## Spell check a document file + +```sh +$ ./kata-spell-check.sh check /path/to/file +``` + +> **Note:** If you have made local edits to the dictionaries, you may +> [re-create the master dictionary files](#create-the-master-dictionary-files) +> as documented in the [Adding a new word](#adding-a-new-word) section, +> in order for your local edits take effect. + +## Other options + +Lists all available options and commands: + +```sh +$ ./kata-spell-check.sh -h +``` + +## Technical details + +### Hunspell dictionary format + +A Hunspell dictionary comprises two text files: + +- A word list file + + This file defines a list of words (one per line). The list includes optional + references to one or more rules defined in the rules file as well as optional + comments. Specify fixed words (e.g. company names) verbatim. Enter “normal” + words in their root form. + + The root form of a "normal" word is the simplest and shortest form of that + word. For example, the following list of words are all formed from the root + word "computer": + + - Computers + - Computer’s + - Computing + - Computed + + Each word in the previous list is an example of using the word "computer" to + construct said word through a combination of applying the following + manipulations: + + - Remove one or more characters from the end of the word. + - Add a new ending. + + Therefore, you list the root word "computer" in the word list file. + +- A rules file + + This file defines named manipulations to apply to root words to form new + words. For example, rules that make a root word plural. + +### Source files + +The rules file and the the word list file for the custom dictionary generate +from "source" fragment files in the [`data`](data/) directory. + +All the fragment files allow comments using the hash (`#`) comment +symbol and all files contain a comment header explaining their content. + +#### Word list file fragments + +The `*.txt` files are word list file fragments. Splitting the word list +into fragments makes updates easier and clearer as each fragment is a +grouping of related terms. The name of the file gives a clue as to the +contents but the comments at the top of each file provide further +detail. + +Every line that does not start with a comment symbol contains a single +word. An optional comment for a word may appear after the word and is +separated from the word by whitespace followed by the comment symbol: + +``` +word # This is a comment explaining this particular word list entry. +``` + +You *may* suffix each word by a forward slash followed by one or more +upper-case letters. Each letter refers to a rule name in the rules file: + +``` +word/AC # This word references the 'A' and 'C' rules. +``` + +#### Rules file + +The [rules file](data/rules.aff) contains a set of general rules that can be +applied to one or more root words in the word list files. You can make +comments in the rules file. + +For an explanation of the format of this file see +[`man 5 hunspell`](http://www.manpagez.com/man/5/hunspell) +([source](https://github.com/hunspell/hunspell/blob/master/man/hunspell.5)). + +## Adding a new word + +### Update the word list fragment + +If you want to allow a new word to the dictionary, + +- Check to ensure you do need to add the word + + Is the word valid and correct? If the word is a project, product, + or company name, is the capitalization correct? + +- Add the new word to the appropriate [word list fragment file](data). + + Specifically, if it is a general word, add the *root* of the word to + the appropriate fragment file. + +- Add a `/` suffix along with the letters for each rule to apply in order to + add rules references. + +### Optionally update the rules file + +It should not generally be necessary to update the rules file since it +already contains rules for most scenarios. However, if you need to +update the file, [read the documentation carefully](#rules-file). + +### Create the master dictionary files + +Every time you change the dictionary files you must recreate the master +dictionary files: + +```sh +$ ./kata-spell-check.sh make-dict +``` + +As a convenience, [checking a file](#spell-check-a-document-file) will +automatically create the database. + +### Test the changes + +You must test any changes to the [word list file +fragments](#word-list-file-fragments) or the [rules file](#rules-file) +by doing the following: + +1. Recreate the [master dictionary files](#create-the-master-dictionary-files). + +1. [Run the spell checker](#spell-check-a-document-file) on a file containing the + words you have added to the dictionary. diff --git a/tests/cmd/check-spelling/data/acronyms.txt b/tests/cmd/check-spelling/data/acronyms.txt new file mode 100644 index 0000000000..3be0907cd3 --- /dev/null +++ b/tests/cmd/check-spelling/data/acronyms.txt @@ -0,0 +1,123 @@ +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Description: List of acronyms and abbreviations. + +ACPI/AB +ACS/AB +API/AB +AUFS # Another Union FS +AWS/AB +BDF/AB +CFS/AB +CLI/AB +CNI/AB +CNM/AB +CPUID/AB +CRI/AB +CVE/AB +DAX/AB +DinD/B # Docker in Docker +dind/B +DMA/AB +DPDK/AB +FaaS/B # Function as a Service +FS/AB +fs/B # For terms like "virtio-fs" +GCE/AB +GOPATH/AB +GPG/AB +GPU/AB +gRPC/AB +GSC/AB +GVT/AB +IaaS/B # Infrastructure as a Service +IOMMU/AB +IoT/AB # Internet of Things +IOV/AB +JSON/AB +k8s/B +KCSA/AB +KSM/AB +KVM/AB +LTS/AB +MACVTAP/AB +mem/B # For terms like "virtio-mem" +memdisk/B +MDEV/AB +NEMU/AB +NIC/AB +NVDIMM/AB +OCI/AB +OVMF/AB +OverlayFS/B +PaaS/B # Platform as a Service +PCDIMM/AB +PCI/AB +PCIe/AB +PID/AB +pmem/B # persistent memory +PNG/AB +POD/AB +PR/AB +PSS/AB +QA/AB +QAT/AB +QEMU/AB +RBAC/AB +RDMA/AB +RNG/AB +SaaS/B # Software as a Service +SCSI/AB +SDK/AB +seccomp # secure computing mode +SHA/AB +SPDX/AB +SRIOV/AB +SVG/AB +TBD/AB +TOC/AB +TOML/AB +TTY/AB +UI/AB +UTS/AB +UUID/AB +vCPU/AB +VETH/AB +VF/AB +VFIO/AB +VGPU/AB +vhost/AB +VHOST/AB +virtio/AB +VirtIO/AB +Virtio-fs/AB +Virtio-mem/AB +VLAN/AB +VM/AB +VMCache/AB +vmm +VMM/AB +VMX/AB +VPP/AB +VSOCK/AB +VSS/AB +WIP/AB # Work In Progress +WRT/AB # With Respect To +XIP/AB +YAML/AB +irq/AB +mmio/AB +APIC +msg/AB +UDS +dbs # Dragonball Sandbox +TDX +tdx +mptable +fdt +gic +msr +cpuid +pio diff --git a/tests/cmd/check-spelling/data/arches.txt b/tests/cmd/check-spelling/data/arches.txt new file mode 100644 index 0000000000..08fa55d850 --- /dev/null +++ b/tests/cmd/check-spelling/data/arches.txt @@ -0,0 +1,21 @@ +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Description: List of architectures. + +# Architectures + +aarch64/B +amd64/B +arm64/B +ppc64el/B +ppc64le/B +s390x/B +x86_64/B +x86/B + +# Micro architecture names + +Haswell/B +Ivybridge/B diff --git a/tests/cmd/check-spelling/data/distros.txt b/tests/cmd/check-spelling/data/distros.txt new file mode 100644 index 0000000000..1edca51f8a --- /dev/null +++ b/tests/cmd/check-spelling/data/distros.txt @@ -0,0 +1,18 @@ +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Description: List of Linux Distributions. + +CentOS/B +Debian/B +EulerOS/B +Fedora/B +macOS/B +MacOS/B +minikube/B +openSUSE/B +OpenSUSE/B +RHEL/B +SLES/B +Ubuntu/B diff --git a/tests/cmd/check-spelling/data/files.txt b/tests/cmd/check-spelling/data/files.txt new file mode 100644 index 0000000000..5fa4fc1168 --- /dev/null +++ b/tests/cmd/check-spelling/data/files.txt @@ -0,0 +1,25 @@ +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Description: Names of commands, files and packages. +# +# Notes: These *should* strictly be placed in backticks but alas this +# doesn't always happen. +# +# References: https://github.com/kata-containers/kata-containers/blob/main/docs/Documentation-Requirements.md#files-and-command-names + +cgroup/AB +coredump/A +cpuset/AB +Dockerfile/AB +init/AB +initramfs/AB +initrd/AB +netns/AB +rootfs/AB +stderr/AB +stdin/AB +stdout/AB +syslog/AB +Vagrantfile/B diff --git a/tests/cmd/check-spelling/data/hunspell.txt b/tests/cmd/check-spelling/data/hunspell.txt new file mode 100644 index 0000000000..feae4b539a --- /dev/null +++ b/tests/cmd/check-spelling/data/hunspell.txt @@ -0,0 +1,13 @@ +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Description: List of words that are missing from Hunspell dictionaries +# on some platforms. + +committer/AB # Not available on Ubuntu 16.04 or CentOS 7 +plugin/AB # Not available on Ubuntu 16.04 +regexp/AB # Not available on Ubuntu 16.04 +screenshot/AB # Not available on Ubuntu 16.04 or CentOS 7 +tarball/AB # Not available on Ubuntu 16.04 +uninstall # Not available on Ubuntu 16.04 diff --git a/tests/cmd/check-spelling/data/main.txt b/tests/cmd/check-spelling/data/main.txt new file mode 100644 index 0000000000..3fcf4e5076 --- /dev/null +++ b/tests/cmd/check-spelling/data/main.txt @@ -0,0 +1,135 @@ +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Description: General word list. + +ack/A +arg # Argument +auditability +backend +backport/ACD +backtick/AB +backtrace +bootloader/AB +centric/B +checkbox/A +chipset/AB +codebase +commandline +config/AB +crypto # Cryptography +cryptoprocessor/AB +DaemonSet/AB +deliverable/AB +dev +devicemapper/B +deploy +dialer +dialog/A +Diffie/B # Diffie–Hellman (cryptography) +distro/AB +emptydir/A +enablement/AB +entrypoint/AB +ethernet +filename/AB +filesystem/AB +freeform +goroutine/AB +hostname/AB +hotplug/ACD +howto/AB +HugePage/AB +hugepage/AB +Hyp +hypercall/A +hypervisor/AB +implementer/A +implementor/A +Infiniband +iodepth/A +ioengine/A +iptables +Itanium/AB +kata +Kat/AB # "Kat Herding Team" :) +keypair/A +lifecycle/A +linter/AB +logfile/A +Longterm +longterm +loopback +memcpy/A +mergeable +metadata +microcontroller/AB +miniOS +mmap/AB +nack/AB +namespace/ABCD +netlink +NVIDIA/A +nvidia/A +onwards +OpenAPI +OS/AB +parallelize/AC +passthrough +patchset/A +pluggable/AB +portmapper/AB +portmapping/A +pre +prefetch/ACD +prestart +programmatically +proxying +Quadro +ramdisk/A +readonly +rebase/ACD +refactor/ACD +remediate +repo/A +runtime/AB +scalability +serverless +signoff/A +stalebot/B +startup +subdirectory/A +swappiness +sysctl/AB +teardown +templating +timestamp/AB +tracability +ttRPC/B +udev/B +uevent/AB +unbootable +uncomment/ACD +unported +unskip/AC +untrusted +untrusting +userid/AB +userspace/B +vendored +vendoring +versioning +vGPU +virtualization +virtualized +webhook/AB +whitespace +workflow/A +Xeon/A +yaml +upcall +Upcall +ioctl/A +struct/A # struct in Rust +Struct/A \ No newline at end of file diff --git a/tests/cmd/check-spelling/data/projects.txt b/tests/cmd/check-spelling/data/projects.txt new file mode 100644 index 0000000000..963de14158 --- /dev/null +++ b/tests/cmd/check-spelling/data/projects.txt @@ -0,0 +1,101 @@ +# Copyright (c) 2019-2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Description: Names of projects, companies and services. + +Ansible/B +AppArmor/B +blogbench/B +BusyBox/B +Cassandra/B +ccloudvm/B +codecov/B +containerd/B +cnn/B +cri-o/B +CRI-O/B +DevStack/B +Django/B +Docker/B +dracut/B +Dragonball/B +Facebook/B +fio/B +Fluentd/B +Frakti/B +Git/B +GitHub/B +GoDoc/B +golang/B +Golang/B +Grafana/B +Gramine/B +Huawei/B +Inclavare/B +iPerf/B +IPerf/B +Istio/B +Jaeger/B +Jenkins/B +Jupyter/B +journald/B +jq/B +Kata/B +Kibana/B +Kubelet/B +Kubernetes/B +Launchpad/B +LevelDB/B +libcontainer/B +libelf/B +libvirt/B +Linkerd/B +LinuxONE/B +Logrus/B +Logstash/B +Mellanox/B +Minikube/B +MITRE/B +musl/B +Netlify/B +Nginx/B +OpenCensus/B +OpenPGP/B +OpenShift/B +OpenSSL/B +OpenStack/B +OpenTelemetry/B +OpenTracing/B +osbuilder/B +packagecloud/B +Pandoc/B +Podman/B +PullApprove/B +Pytorch/B +QuickAssist/B +R/B +raytracer/B +rkt/B/B +runc/B +runV/B +rustlang/B +Rustlang/B +SELinux/B +SemaphoreCI/B +snapcraft/B +snapd/B +SQLite/B +SUSE/B +Sysbench/B +systemd/B +tf/B +TravisCI/B +Tokio/B +Vexxhost/B +virtcontainers/B +VMWare/B +vSphere/B +Yamux/B +yq/B +Zun/B diff --git a/tests/cmd/check-spelling/data/rules.aff b/tests/cmd/check-spelling/data/rules.aff new file mode 100644 index 0000000000..7f37dbf477 --- /dev/null +++ b/tests/cmd/check-spelling/data/rules.aff @@ -0,0 +1,36 @@ +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +SET UTF-8 + +# Add the following characters so they are accepted as part of a word +WORDCHARS 0123456789' + +# Disable hyphenation +BREAK 0 + +# plural +SFX A N 3 +SFX A 0 s [^x] +SFX A 0 es x +SFX A y ies + +# possession +SFX B N 1 +SFX B 0 's + +# past tense +SFX C N 4 +SFX C 0 d e +SFX C 0 ed [rt] +SFX C 0 ped p +SFX C 0 ged g + +# present continuous +SFX D N 3 +SFX D 0 ging g +SFX D 0 ing [rt] +SFX D e ing e diff --git a/tests/cmd/check-spelling/kata-dictionary.aff b/tests/cmd/check-spelling/kata-dictionary.aff new file mode 100644 index 0000000000..7f37dbf477 --- /dev/null +++ b/tests/cmd/check-spelling/kata-dictionary.aff @@ -0,0 +1,36 @@ +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +SET UTF-8 + +# Add the following characters so they are accepted as part of a word +WORDCHARS 0123456789' + +# Disable hyphenation +BREAK 0 + +# plural +SFX A N 3 +SFX A 0 s [^x] +SFX A 0 es x +SFX A y ies + +# possession +SFX B N 1 +SFX B 0 's + +# past tense +SFX C N 4 +SFX C 0 d e +SFX C 0 ed [rt] +SFX C 0 ped p +SFX C 0 ged g + +# present continuous +SFX D N 3 +SFX D 0 ging g +SFX D 0 ing [rt] +SFX D e ing e diff --git a/tests/cmd/check-spelling/kata-dictionary.dic b/tests/cmd/check-spelling/kata-dictionary.dic new file mode 100644 index 0000000000..33d41e37ec --- /dev/null +++ b/tests/cmd/check-spelling/kata-dictionary.dic @@ -0,0 +1,384 @@ +383 +ACPI/AB +ACS/AB +API/AB +APIC +AUFS +AWS/AB +Ansible/B +AppArmor/B +BDF/AB +BusyBox/B +CFS/AB +CLI/AB +CNI/AB +CNM/AB +CPUID/AB +CRI-O/B +CRI/AB +CVE/AB +Cassandra/B +CentOS/B +DAX/AB +DMA/AB +DPDK/AB +DaemonSet/AB +Debian/B +DevStack/B +Diffie/B +DinD/B +Django/B +Docker/B +Dockerfile/AB +Dragonball/B +EulerOS/B +FS/AB +FaaS/B +Facebook/B +Fedora/B +Fluentd/B +Frakti/B +GCE/AB +GOPATH/AB +GPG/AB +GPU/AB +GSC/AB +GVT/AB +Git/B +GitHub/B +GoDoc/B +Golang/B +Grafana/B +Gramine/B +Haswell/B +Huawei/B +HugePage/AB +Hyp +IOMMU/AB +IOV/AB +IPerf/B +IaaS/B +Inclavare/B +Infiniband +IoT/AB +Istio/B +Itanium/AB +Ivybridge/B +JSON/AB +Jaeger/B +Jenkins/B +Jupyter/B +KCSA/AB +KSM/AB +KVM/AB +Kat/AB +Kata/B +Kibana/B +Kubelet/B +Kubernetes/B +LTS/AB +Launchpad/B +LevelDB/B +Linkerd/B +LinuxONE/B +Logrus/B +Logstash/B +Longterm +MACVTAP/AB +MDEV/AB +MITRE/B +MacOS/B +Mellanox/B +Minikube/B +NEMU/AB +NIC/AB +NVDIMM/AB +NVIDIA/A +Netlify/B +Nginx/B +OCI/AB +OS/AB +OVMF/AB +OpenAPI +OpenCensus/B +OpenPGP/B +OpenSSL/B +OpenSUSE/B +OpenShift/B +OpenStack/B +OpenTelemetry/B +OpenTracing/B +OverlayFS/B +PCDIMM/AB +PCI/AB +PCIe/AB +PID/AB +PNG/AB +POD/AB +PR/AB +PSS/AB +PaaS/B +Pandoc/B +Podman/B +PullApprove/B +Pytorch/B +QA/AB +QAT/AB +QEMU/AB +Quadro +QuickAssist/B +R/B +RBAC/AB +RDMA/AB +RHEL/B +RNG/AB +Rustlang/B +SCSI/AB +SDK/AB +SELinux/B +SHA/AB +SLES/B +SPDX/AB +SQLite/B +SRIOV/AB +SUSE/B +SVG/AB +SaaS/B +SemaphoreCI/B +Struct/A# +Sysbench/B +TBD/AB +TDX +TOC/AB +TOML/AB +TTY/AB +Tokio/B +TravisCI/B +UDS +UI/AB +UTS/AB +UUID/AB +Ubuntu/B +Upcall +VETH/AB +VF/AB +VFIO/AB +VGPU/AB +VHOST/AB +VLAN/AB +VM/AB +VMCache/AB +VMM/AB +VMWare/B +VMX/AB +VPP/AB +VSOCK/AB +VSS/AB +Vagrantfile/B +Vexxhost/B +VirtIO/AB +Virtio-fs/AB +Virtio-mem/AB +WIP/AB +WRT/AB +XIP/AB +Xeon/A +YAML/AB +Yamux/B +Zun/B +aarch64/B +ack/A +amd64/B +arg +arm64/B +auditability +backend +backport/ACD +backtick/AB +backtrace +blogbench/B +bootloader/AB +ccloudvm/B +centric/B +cgroup/AB +checkbox/A +chipset/AB +cnn/B +codebase +codecov/B +commandline +committer/AB +config/AB +containerd/B +coredump/A +cpuid +cpuset/AB +cri-o/B +crypto +cryptoprocessor/AB +dbs +deliverable/AB +deploy +dev +devicemapper/B +dialer +dialog/A +dind/B +distro/AB +dracut/B +emptydir/A +enablement/AB +entrypoint/AB +ethernet +fdt +filename/AB +filesystem/AB +fio/B +freeform +fs/B +gRPC/AB +gic +golang/B +goroutine/AB +hostname/AB +hotplug/ACD +howto/AB +hugepage/AB +hypercall/A +hypervisor/AB +iPerf/B +implementer/A +implementor/A +init/AB +initramfs/AB +initrd/AB +ioctl/A +iodepth/A +ioengine/A +iptables +irq/AB +journald/B +jq/B +k8s/B +kata +keypair/A +libcontainer/B +libelf/B +libvirt/B +lifecycle/A +linter/AB +logfile/A +longterm +loopback +macOS/B +mem/B +memcpy/A +memdisk/B +mergeable +metadata +microcontroller/AB +miniOS +minikube/B +mmap/AB +mmio/AB +mptable +msg/AB +msr +musl/B +nack/AB +namespace/ABCD +netlink +netns/AB +nvidia/A +onwards +openSUSE/B +osbuilder/B +packagecloud/B +parallelize/AC +passthrough +patchset/A +pio +pluggable/AB +plugin/AB +pmem/B +portmapper/AB +portmapping/A +ppc64el/B +ppc64le/B +pre +prefetch/ACD +prestart +programmatically +proxying +ramdisk/A +raytracer/B +readonly +rebase/ACD +refactor/ACD +regexp/AB +remediate +repo/A +rkt/B/B +rootfs/AB +runV/B +runc/B +runtime/AB +rustlang/B +s390x/B +scalability +screenshot/AB +seccomp +serverless +signoff/A +snapcraft/B +snapd/B +stalebot/B +startup +stderr/AB +stdin/AB +stdout/AB +struct/A +subdirectory/A +swappiness +sysctl/AB +syslog/AB +systemd/B +tarball/AB +tdx +teardown +templating +tf/B +timestamp/AB +tracability +ttRPC/B +udev/B +uevent/AB +unbootable +uncomment/ACD +uninstall +unported +unskip/AC +untrusted +untrusting +upcall +userid/AB +userspace/B +vCPU/AB +vGPU +vSphere/B +vendored +vendoring +versioning +vhost/AB +virtcontainers/B +virtio/AB +virtualization +virtualized +vmm +webhook/AB +whitespace +workflow/A +x86/B +x86_64/B +yaml +yq/B diff --git a/tests/cmd/check-spelling/kata-spell-check.sh b/tests/cmd/check-spelling/kata-spell-check.sh new file mode 100755 index 0000000000..ca7b2b8f09 --- /dev/null +++ b/tests/cmd/check-spelling/kata-spell-check.sh @@ -0,0 +1,336 @@ +#!/bin/bash +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Description: spell-check utility. + +[ -n "$DEBUG" ] && set -x + +set -o errexit +set -o pipefail +set -o nounset + +# Ensure we spell check in English +LANG=C +LC_ALL=C + +script_name=${0##*/} + +if [ "$(uname -s)" == "Darwin" ] +then + # Hunspell dictionaries are a not easily available + # on this platform it seems. + echo "INFO: $script_name: OSX not supported - exiting" + exit 0 +fi + +self_dir=$(dirname "$(readlink -f "$0")") +cidir="${self_dir}/../../.ci" + +source "${cidir}/lib.sh" + +# Directory containing word lists. +# +# Each file in this directory must: +# +# - Have the ".txt" extension. +# - Contain one word per line. +# +# Additionally, the files may contain blank lines and comments +# (lines beginning with '#'). +KATA_DICT_FRAGMENT_DIR=${KATA_DICT_FRAGMENT_DIR:-data} + +KATA_DICT_NAME="${KATA_DICT_NAME:-kata-dictionary}" + +# Name of dictionary file suitable for using with hunspell(1) +# as a personal dictionary. +KATA_DICT_FILE="${KATA_DICT_FILE:-${KATA_DICT_NAME}.dic}" + +KATA_RULES_FILE="${KATA_RULES_FILE:-${KATA_DICT_FILE/.dic/.aff}}" + +# command to remove code from markdown (inline and blocks) +strip_cmd="${cidir}/kata-doc-to-script.sh" + +fragment_dir="${self_dir}/${KATA_DICT_FRAGMENT_DIR}" + +# Name of file containing dictionary rules that apply to the +# KATA_DICT_FILE word list. +rules_file_name="rules.aff" + +# Command to spell check a file +spell_check_cmd="${KATA_SPELL_CHECK_CMD:-hunspell}" + +# Command to convert a markdown file into plain text +md_convert_tool="${KATA_MARKDOWN_CONVERT_TOOL:-pandoc}" + +KATA_DICT_DIR="${KATA_DICT_DIR:-${self_dir}}" +dict_file="${KATA_DICT_DIR}/${KATA_DICT_FILE}" +rules_file="${KATA_DICT_DIR}/${KATA_RULES_FILE}" + +# Hunspell refers to custom dictionary by their path followed by the name of +# the dictionary (without the file extension). +kata_dict_ref="${KATA_DICT_DIR}/${KATA_DICT_NAME}" + +# All project documentation must be written in English, +# with American English taking priority. +# +# We also use a custom dictionary which has to be specified by its +# "directory and name prefix" and which must also be the first specified +# dictionary. +dict_languages="${kata_dict_ref},en_US,en_GB" + +make_dictionary() +{ + [ -d "$fragment_dir" ] || die "invalid fragment directory" + [ -z "$dict_file" ] && die "missing dictionary output file name" + + # Note: the first field is extracted to allow for inline + # comments in each fragment. For example: + # + # word # this text describes why the word is in the dictionary. + # + local dict + + dict=$(cat "$fragment_dir"/*.txt |\ + grep -v '^\#' |\ + grep -v '^$' |\ + awk '{print $1}' |\ + sort -u || true) + + [ -z "$dict" ] && die "generated dictionary is empty" + + # Now, add in the number of words as a header (required by Hunspell) + local count + + count=$(echo "$dict"| wc -l | awk '{print $1}' || true) + [ -z "$count" ] && die "cannot determine dictionary length" + [ "$count" -eq 0 ] && die "invalid dictionary length" + + # Construct the dictionary + (echo "$count"; echo "$dict") > "$dict_file" + + cp "${fragment_dir}/${rules_file_name}" "${rules_file}" +} + +spell_check_file() +{ + local file="$1" + + [ -z "$file" ] && die "need file to check" + [ -e "$file" ] || die "file does not exist: '$file'" + + [ -e "$dict_file" ] || make_dictionary + + info "Spell checking file '$file'" + + # Determine the pandoc input format. + local pandoc_input_fmts + local pandoc_input_fmt + + local pandoc_input_fmts=$(pandoc --list-input-formats 2>/dev/null || true) + + if [ -z "$pandoc_input_fmts" ] + then + # We're using a very old version of pandoc that doesn't + # support listing its available input formats, so + # specify a default. + pandoc_input_fmt="markdown_github" + else + # Pandoc has multiple names for the gfm parser so find one of them + pandoc_input_fmt=$(echo "$pandoc_input_fmts" |\ + grep -E "gfm|github" |\ + head -1 || true) + fi + + [ -z "$pandoc_input_fmt" ] && die "cannot find usable pandoc input format" + + local stripped_doc + + local pandoc_doc + local utf8_free_doc + local pre_hunspell_doc + local hunspell_results + local final_results + + # First strip out all code blocks and convert all + # "quoted apostrophe's" ('\'') back into a single apostrophe. + stripped_doc=$("$strip_cmd" -i "$file" -) + + # Next, convert the remainder it into plain text to remove the + # remaining markdown syntax. + # + # Before pandoc gets hold of it: + # + # - Replace pipes with spaces. This + # fixes an issue with old versions of pandoc (Ubuntu 16.04) + # which completely mangle tables into nonsense. + # + # - Remove empty reference links. + # + # For example, this markdown + # + # blah [`qemu-lite`][qemu-lite] blah. + # : + # [qemu-lite]: https://... + # + # Gets converted into + # + # blah [][qemu-lite] blah. + # : + # [qemu-lite]: https://... + # + # And the empty set of square brackets confuses pandoc. + # + # After pandoc has processed the data, remove any remaining + # "inline links" in this format: + # + # [link name](#link-address) + # + # This is strictly only required for old versions of pandoc. + + pandoc_doc=$(echo "$stripped_doc" |\ + tr '|' ' ' |\ + sed 's/\[\]\[[^]]*\]//g' |\ + "$md_convert_tool" -f "${pandoc_input_fmt}" -t plain - |\ + sed 's/\[[^]]*\]([^\)]*)//g' || true) + + # Convert the file into "pure ASCII" by removing all awkward + # Unicode characters that won't spell check. + # + # Necessary since pandoc is "clever" and will convert things like + # GitHub's colon emojis (such as ":smile:") into the actual utf8 + # character where possible. + utf8_free_doc=$(echo "$pandoc_doc" | iconv -c -f utf-8 -t ascii) + + # Next, perform the following simplifications: + # + # - Remove URLs. + # - Remove email addresses. + # - Replace most punctuation symbols with a space + # (excluding a dash (aka hyphen!) + # - Carefully remove non-hyphen dashes. + # - Remove GitHub @userids. + pre_hunspell_doc=$(echo "$utf8_free_doc" |\ + sed 's,https*://[^[:space:]()][^[:space:]()]*,,g' |\ + sed -r 's/[a-zA-Z0-9.-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9.-]+//g' |\ + tr '[,\[\]()\*\\/\|=]' ' ' |\ + sed -e 's/^ *-//g' -e 's/- $//g' -e 's/ -//g' |\ + sed 's/@[a-zA-Z0-9][a-zA-Z0-9]*\b//g') + + # Call the spell checker + hunspell_results=$(echo "$pre_hunspell_doc" | $spell_check_cmd -d "${dict_languages}") + + # Finally, post-process the hunspell output: + # + # - Parse the output to ignore: + # - Hunspell banner. + # - Correctly spelt words (lines starting with '*', '+' or '-'). + # - All words containing numbers (like "100MB"). + # - All words that appear to be acronymns / Abbreviations + # (atleast two upper-case letters and which may be plural or + # possessive). + # - All words that appear to be numbers. + # - All possessives and the dreaded isolated "'s" which occurs + # for input like this: + # + # `kata-shim`'s + # + # which gets converted by $strip_cmd into simply: + # + # 's + # + # - Sort output. + + final_results=$(echo "$hunspell_results" |\ + grep -Evi "(ispell|hunspell)" |\ + grep -Ev '^(\*|\+|-)' |\ + grep -Evi "^(&|#) [^ ]*[0-9][^ ]*" |\ + grep -Ev "^. [A-Z][A-Z][A-Z]*(s|'s)*" |\ + grep -Ev "^. 's" |\ + sort -u || true) + + local line + local incorrects + local near_misses + + near_misses=$(echo "$final_results" | grep '^&' || true) + incorrects=$(echo "$final_results" | grep '^\#' | awk '{print $2}' || true) + + local -i failed=0 + + [ -n "$near_misses" ] && failed+=1 + [ -n "$incorrects" ] && failed+=1 + + echo "$near_misses" | while read -r line + do + [ "$line" = "" ] && continue + + local word + local possibles + + word=$(echo "$line" | awk '{print $2}') + possibles=$(echo "$line" | cut -d: -f2- | sed 's/^ *//g') + + warn "Word '${word}': did you mean one of the following?: ${possibles}" + done + + local incorrect + for incorrect in $incorrects + do + warn "Incorrect word: '$incorrect'" + done + + [ "$failed" -gt 0 ] && die "Spell check failed for file: '$file'" + + info "Spell check successful for file: '$file'" +} + +delete_dictionary() +{ + rm -f "${KATA_DICT_FILE}" "${KATA_RULES_FILE}" +} + +setup() +{ + local cmd + + for cmd in "$spell_check_cmd" "$md_convert_tool" + do + command -v "$cmd" &>/dev/null || die "Need $cmd command" + done +} + +usage() +{ + cat <<-EOF + Usage: ${script_name} [arguments] + + Description: Spell-checking utility. + + Commands: + + check : Spell check the specified file + (implies 'make-dict'). + delete-dict : Delete the dictionary. + help : Show this usage. + make-dict : Create the dictionary. +EOF +} + +main() +{ + setup + + [ -z "${1:-}" ] && usage && echo && die "need command" + + case "$1" in + check) shift && spell_check_file "$1" ;; + delete-dict) delete_dictionary ;; + help|-h|--help) usage && exit 0 ;; + make-dict) make_dictionary ;; + *) die "invalid command: '$1'" ;; + esac +} + +main "$@" From 66f3944b52d386cd995c77c9a4fc6f5d311fc16f Mon Sep 17 00:00:00 2001 From: Chelsea Mafrica Date: Tue, 21 Nov 2023 17:48:18 -0800 Subject: [PATCH 4/6] tests: move github-labels to main repo Move tool as part of static checks migration. Fixes #8187 Signed-off-by: Chelsea Mafrica Signed-off-by: Derek Lee Signed-off-by: Gabriela Cervantes Signed-off-by: Graham Whaley Signed-off-by: James O. D. Hunt Signed-off-by: Marco Vedovati Signed-off-by: Peng Tao Signed-off-by: Shiming Zhang Signed-off-by: Snir Sheriber Signed-off-by: Wainer dos Santos Moschetta --- tests/cmd/github-labels/Makefile | 32 + tests/cmd/github-labels/README.md | 71 +++ tests/cmd/github-labels/VERSION | 1 + tests/cmd/github-labels/archive/README.md | 50 ++ ...er-original-labels-kata-containers-ci.yaml | 58 ++ ...inal-labels-kata-containers-community.yaml | 27 + ...abels-kata-containers-kata-containers.yaml | 44 ++ ...original-labels-kata-containers-tests.yaml | 60 ++ tests/cmd/github-labels/check.go | 216 +++++++ tests/cmd/github-labels/clean.go | 62 ++ tests/cmd/github-labels/display.go | 83 +++ tests/cmd/github-labels/display_markdown.go | 75 +++ tests/cmd/github-labels/display_text.go | 101 ++++ tests/cmd/github-labels/display_tsv.go | 66 +++ tests/cmd/github-labels/github-labels.sh | 176 ++++++ tests/cmd/github-labels/labels.yaml | 555 ++++++++++++++++++ tests/cmd/github-labels/labels.yaml.in | 555 ++++++++++++++++++ tests/cmd/github-labels/main.go | 157 +++++ tests/cmd/github-labels/record.go | 102 ++++ tests/cmd/github-labels/types.go | 55 ++ tests/cmd/github-labels/utils.go | 24 + tests/cmd/github-labels/yaml.go | 72 +++ 22 files changed, 2642 insertions(+) create mode 100644 tests/cmd/github-labels/Makefile create mode 100644 tests/cmd/github-labels/README.md create mode 100644 tests/cmd/github-labels/VERSION create mode 100644 tests/cmd/github-labels/archive/README.md create mode 100644 tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-ci.yaml create mode 100644 tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-community.yaml create mode 100644 tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-kata-containers.yaml create mode 100644 tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-tests.yaml create mode 100644 tests/cmd/github-labels/check.go create mode 100644 tests/cmd/github-labels/clean.go create mode 100644 tests/cmd/github-labels/display.go create mode 100644 tests/cmd/github-labels/display_markdown.go create mode 100644 tests/cmd/github-labels/display_text.go create mode 100644 tests/cmd/github-labels/display_tsv.go create mode 100755 tests/cmd/github-labels/github-labels.sh create mode 100644 tests/cmd/github-labels/labels.yaml create mode 100644 tests/cmd/github-labels/labels.yaml.in create mode 100644 tests/cmd/github-labels/main.go create mode 100644 tests/cmd/github-labels/record.go create mode 100644 tests/cmd/github-labels/types.go create mode 100644 tests/cmd/github-labels/utils.go create mode 100644 tests/cmd/github-labels/yaml.go diff --git a/tests/cmd/github-labels/Makefile b/tests/cmd/github-labels/Makefile new file mode 100644 index 0000000000..c0d5e06e9e --- /dev/null +++ b/tests/cmd/github-labels/Makefile @@ -0,0 +1,32 @@ +# +# Copyright (c) 2017-2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +TARGET = kata-github-labels +SOURCES = $(shell find . -type f 2>&1 | grep -E '.*\.go$$') + +VERSION := ${shell cat ./VERSION} +COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true) +COMMIT := $(if $(shell git status --porcelain --untracked-files=no),"${COMMIT_NO}-dirty","${COMMIT_NO}") + +BINDIR := $(GOPATH)/bin +DESTTARGET := $(abspath $(BINDIR)/$(TARGET)) + +default: install + +check: $(SOURCES) + go test -v ./... + +$(TARGET): $(SOURCES) + go build -o "$(TARGET)" -ldflags "-X main.name=${TARGET} -X main.commit=${COMMIT} -X main.version=${VERSION}" . + +install: $(TARGET) + install -d $(shell dirname $(DESTTARGET)) + install $(TARGET) $(DESTTARGET) + +clean: + rm -f $(TARGET) + +.PHONY: install clean diff --git a/tests/cmd/github-labels/README.md b/tests/cmd/github-labels/README.md new file mode 100644 index 0000000000..f602fd9642 --- /dev/null +++ b/tests/cmd/github-labels/README.md @@ -0,0 +1,71 @@ +# Overview + +The Kata Project uses a number of GitHub repositories. To allow issues and PRs +to be handled consistently between repositories a standard set of issue labels +are used. These labels are stored in YAML format in the master +[labels database template](labels.yaml.in). This file is human-readable, +machine-readable, and self-describing (see the file for the introductory +description). + +Each repository can contain a set of additional (repository-specific) labels, +which are stored in a top-level YAML template file called `labels.yaml.in`. + +Expanding the templates and merging the two databases describes the full set +of labels a repository uses. + +# Generating the combined labels database + +You can run the `github_labels.sh` script with the `generate` argument to +create the combined labels database. The additional arguments specify the +repository (in order to generate the combined labels database) and the name of +a file to write the combined database: + +```sh +$ ./github-labels.sh generate github.com/kata-containers/kata-containers /tmp/combined.yaml +``` + +This script validates the combined labels database by performing a number of +checks, including running the `kata-github-labels` tool in checking mode. See +the +[Checking and summarising the labels database](#checking-and-summarising-the-labels-database) +section for more information. + +# Checking and summarising the labels database + +The `kata-github-labels` tool checks and summarizes the labels database for +each repository. + +## Show labels + +Displays a summary of the labels: + +```sh +$ kata-github-labels show labels labels.yaml +``` + +## Show categories + +Shows all information about categories: + +```sh +$ kata-github-labels show categories --with-labels labels.yaml +``` +## Check only + +Performs checks on a specified labels database: + +```sh +$ kata-github-labels check labels.yaml +``` + +## Full details + +Lists all available options: + +```sh +$ kata-github-labels -h +``` + +# Archive of old GitHub labels + +See the [archive documentation](archive). diff --git a/tests/cmd/github-labels/VERSION b/tests/cmd/github-labels/VERSION new file mode 100644 index 0000000000..8acdd82b76 --- /dev/null +++ b/tests/cmd/github-labels/VERSION @@ -0,0 +1 @@ +0.0.1 diff --git a/tests/cmd/github-labels/archive/README.md b/tests/cmd/github-labels/archive/README.md new file mode 100644 index 0000000000..358e28989b --- /dev/null +++ b/tests/cmd/github-labels/archive/README.md @@ -0,0 +1,50 @@ +# GitHub labels archive + +## Overview + +This directory contains one YAML file per repository containing the original +set of GitHub labels before the +[new ones were applied on 2019-06-04](../labels.yaml.in). + +## How the YAML files were created + +This section explains how the YAML files were created. + +The [`labeler`](https://github.com/tonglil/labeler) tool was used to read +the labels and write them to a YAML file. + +### Install and patch the `labeler` tool + +This isn't ideal but our [labels database](../labels.yaml.in) mandates +descriptions for every label. However, at the time of writing, the `labeler` +tool does not support descriptions. But, +[there is a PR](https://github.com/tonglil/labeler/pull/37) +to add in description support. + +To enable description support: + +```sh +$ go get -u github.com/tonglil/labeler +$ cd $GOPATH/src/github.com/tonglil/labeler +$ pr=37 +$ pr_branch="PR${pr}" +$ git fetch origin "refs/pull/${pr}/head:{pr_branch}" +$ git checkout "${pr_branch}" +$ go install -v ./... +``` + +### Save GitHub labels for a repository + +Run the following for reach repository: + +```sh +$ labeler scan -r ${github_repo_slug} ${output_file} +``` + +For example, to save the labels for the `tests` repository: + +```sh +$ labeler scan -r kata-containers/tests tests.yaml + +``` + diff --git a/tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-ci.yaml b/tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-ci.yaml new file mode 100644 index 0000000000..a1e3402967 --- /dev/null +++ b/tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-ci.yaml @@ -0,0 +1,58 @@ +# Scanned and autogenerated by https://github.com/tonglil/labeler +--- +repo: kata-containers/ci +labels: + - name: P1 + color: b60205 + description: Highest priority issue (Critical) + - name: P2 + color: d93f0b + description: Urgent issue + - name: P3 + color: fbca04 + description: Important issue + - name: P4 + color: fef2c0 + description: Noteworthy issue + - name: backlog + color: ededed + - name: bitesize + color: d4c5f9 + description: small/easy task + - name: bug + color: d73a4a + description: Something isn't working + - name: do-not-merge + color: b60205 + - name: duplicate + color: cfd3d7 + description: This issue or pull request already exists + - name: enhancement + color: a2eeef + description: New feature or request + - name: good first issue + color: 7057ff + description: Good for newcomers + - name: help wanted + color: "008672" + description: Extra attention is needed + - name: in progress + color: ededed + - name: invalid + color: e4e669 + description: This doesn't seem right + - name: next + color: ededed + - name: question + color: d876e3 + description: Further information is requested + - name: review + color: ededed + - name: security + color: fbca04 + - name: wip + color: b60205 + description: Work In Progress + - name: wontfix + color: ffffff + description: This will not be worked on diff --git a/tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-community.yaml b/tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-community.yaml new file mode 100644 index 0000000000..45d540ea7a --- /dev/null +++ b/tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-community.yaml @@ -0,0 +1,27 @@ +# Scanned and autogenerated by https://github.com/tonglil/labeler +--- +repo: kata-containers/community +labels: + - name: WIP + color: b60205 + - name: bitesize + color: d4c5f9 + description: small/easy task + - name: bug + color: ee0701 + - name: do-not-merge + color: b60205 + - name: duplicate + color: cccccc + - name: enhancement + color: 84b6eb + - name: good first issue + color: 7057ff + - name: help wanted + color: 33aa3f + - name: invalid + color: e6e6e6 + - name: question + color: cc317c + - name: wontfix + color: ffffff diff --git a/tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-kata-containers.yaml b/tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-kata-containers.yaml new file mode 100644 index 0000000000..d88016b29c --- /dev/null +++ b/tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-kata-containers.yaml @@ -0,0 +1,44 @@ +# Scanned and autogenerated by https://github.com/tonglil/labeler +--- +repo: kata-containers/kata-containers +labels: + - name: P1 + color: b60205 + description: Highest priority issue (Critical) + - name: P2 + color: d93f0b + description: Urgent issue + - name: P3 + color: fbca04 + description: Important issue + - name: P4 + color: fef2c0 + description: Noteworthy issue + - name: bitesize + color: d4c5f9 + description: small/easy task + - name: bug + color: ee0701 + - name: devices + color: 006b75 + description: direct device support + - name: duplicate + color: cccccc + - name: enhancement + color: 84b6eb + - name: feature + color: ef70a3 + - name: good first issue + color: 7057ff + - name: help wanted + color: 33aa3f + - name: invalid + color: e6e6e6 + - name: limitation + color: c2e0c6 + - name: question + color: cc317c + - name: security + color: fbca04 + - name: wontfix + color: ffffff diff --git a/tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-tests.yaml b/tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-tests.yaml new file mode 100644 index 0000000000..3d06cb62bd --- /dev/null +++ b/tests/cmd/github-labels/archive/labeler-original-labels-kata-containers-tests.yaml @@ -0,0 +1,60 @@ +# Scanned and autogenerated by https://github.com/tonglil/labeler +--- +repo: kata-containers/tests +labels: + - name: CI + color: 0052cc + description: Continuous Integration + - name: P1 + color: b60205 + description: Highest priority issue (Critical) + - name: P2 + color: d93f0b + description: Urgent issue + - name: P3 + color: fbca04 + description: Important issue + - name: P4 + color: fef2c0 + description: Noteworthy issue + - name: backlog + color: ededed + - name: bitesize + color: d4c5f9 + description: small/easy task + - name: bug + color: ee0701 + - name: do-not-merge + color: b60205 + - name: duplicate + color: cccccc + - name: enhancement + color: 84b6eb + - name: good first issue + color: 7057ff + - name: hackathon + color: 35bfa1 + description: PR/Issues in hackathon events + - name: help wanted + color: 33aa3f + - name: in progress + color: ededed + - name: invalid + color: e6e6e6 + - name: limitation + color: c2e0c6 + - name: next + color: ededed + - name: question + color: cc317c + - name: review + color: ededed + - name: security + color: fbca04 + - name: stable-candidate + color: bfdadc + description: Candidate to backport to stable branches + - name: wip + color: b60205 + - name: wontfix + color: ffffff diff --git a/tests/cmd/github-labels/check.go b/tests/cmd/github-labels/check.go new file mode 100644 index 0000000000..c462370e84 --- /dev/null +++ b/tests/cmd/github-labels/check.go @@ -0,0 +1,216 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "errors" + "fmt" + "strings" + "unicode" +) + +func containsWhitespace(s string) bool { + for _, ch := range s { + if unicode.IsSpace(ch) { + return true + } + } + + return false +} + +func isLower(s string) bool { + for _, ch := range s { + if !unicode.IsLetter(ch) { + continue + } + + if !unicode.IsLower(ch) { + return false + } + } + + return true +} + +func checkCategory(c Category) error { + if c.Name == "" { + return fmt.Errorf("category name cannot be blank: %+v", c) + } + + if containsWhitespace(c.Name) { + return fmt.Errorf("category name cannot contain whitespace: %+v", c) + } + + if !isLower(c.Name) { + return fmt.Errorf("category name must be all lower case: %+v", c) + } + + if c.Description == "" { + return fmt.Errorf("category description cannot be blank: %+v", c) + } + + first := c.Description[0] + + if !unicode.IsUpper(rune(first)) { + return fmt.Errorf("category description needs initial capital letter: %+v", c) + } + + if !strings.HasSuffix(c.Description, ".") { + return fmt.Errorf("category description needs trailing period: %+v", c) + } + + return nil +} + +func checkLabel(l Label) error { + if l.Name == "" { + return fmt.Errorf("label name cannot be blank: %+v", l) + } + + if !isLower(l.Name) { + return fmt.Errorf("label name must be all lower case: %+v", l) + } + + if containsWhitespace(l.Name) { + return fmt.Errorf("label name cannot contain whitespace: %+v", l) + } + + if l.Description == "" { + return fmt.Errorf("label description cannot be blank: %+v", l) + } + + first := l.Description[0] + + if !unicode.IsUpper(rune(first)) { + return fmt.Errorf("label description needs initial capital letter: %+v", l) + } + + if l.CategoryName == "" { + return fmt.Errorf("label category name cannot be blank: %+v", l) + } + + if l.Colour == "" { + return fmt.Errorf("label colour cannot be blank: %+v", l) + } + + return nil +} + +func checkLabelsAndCategories(lf *LabelsFile) error { + catCount := 0 + + var catNameMap map[string]int + var catDescMap map[string]int + + var labelNameMap map[string]int + var labelDescMap map[string]int + + catNameMap = make(map[string]int) + catDescMap = make(map[string]int) + labelNameMap = make(map[string]int) + labelDescMap = make(map[string]int) + + for _, c := range lf.Categories { + if err := checkCategory(c); err != nil { + return err + } + + catCount++ + + if _, ok := catNameMap[c.Name]; ok { + return fmt.Errorf("duplicate category name: %+v", c) + } + + catNameMap[c.Name] = 0 + + if _, ok := catDescMap[c.Description]; ok { + return fmt.Errorf("duplicate category description: %+v", c) + } + + catDescMap[c.Description] = 0 + } + + if catCount == 0 { + return errors.New("no categories found") + } + + labelCount := 0 + + for _, l := range lf.Labels { + if err := checkLabel(l); err != nil { + return err + } + + if _, ok := labelNameMap[l.Name]; ok { + return fmt.Errorf("duplicate label name: %+v", l) + } + + labelNameMap[l.Name] = 0 + + if _, ok := labelDescMap[l.Description]; ok { + return fmt.Errorf("duplicate label description: %+v", l) + } + + labelDescMap[l.Description] = 0 + + labelCount++ + + catName := l.CategoryName + + var value int + var ok bool + if value, ok = catNameMap[catName]; !ok { + return fmt.Errorf("invalid category %v found for label %+v", catName, l) + } + + // Record category name seen and count of occurrences + value++ + catNameMap[catName] = value + } + + if labelCount == 0 { + return errors.New("no labels found") + } + + if debug { + fmt.Printf("DEBUG: category count: %v\n", catCount) + fmt.Printf("DEBUG: label count: %v\n", labelCount) + } + + for name, count := range catNameMap { + if count == 0 { + return fmt.Errorf("category %v not used", name) + } + + if debug { + fmt.Printf("DEBUG: category %v: label count: %d\n", + name, count) + } + } + + return nil +} + +func check(lf *LabelsFile) error { + if lf.Description == "" { + return errors.New("description cannot be blank") + } + + if lf.Repo == "" { + return errors.New("repo cannot be blank") + } + + if len(lf.Categories) == 0 { + return errors.New("no categories") + } + + if len(lf.Labels) == 0 { + return errors.New("no labels") + } + + return checkLabelsAndCategories(lf) +} diff --git a/tests/cmd/github-labels/clean.go b/tests/cmd/github-labels/clean.go new file mode 100644 index 0000000000..1f834cc1fe --- /dev/null +++ b/tests/cmd/github-labels/clean.go @@ -0,0 +1,62 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import "strings" + +func cleanString(s string) string { + result := strings.Replace(s, "\n", " ", -1) + result = strings.Replace(result, "\t", "\\t", -1) + result = strings.TrimSpace(result) + + return result +} + +func cleanLabel(l Label) Label { + return Label{ + Name: cleanString(l.Name), + Description: cleanString(l.Description), + CategoryName: cleanString(l.CategoryName), + Colour: cleanString(l.Colour), + From: cleanString(l.From), + } +} + +func cleanCategory(c *Category) { + c.Name = cleanString(c.Name) + c.Description = cleanString(c.Description) + c.URL = cleanString(c.URL) +} + +func cleanCategories(lf *LabelsFile) { + var cleaned Categories + + for _, c := range lf.Categories { + cleanCategory(&c) + cleaned = append(cleaned, c) + } + + lf.Categories = cleaned +} + +func cleanLabels(lf *LabelsFile) { + var cleaned Labels + + for _, l := range lf.Labels { + new := cleanLabel(l) + cleaned = append(cleaned, new) + } + + lf.Labels = cleaned +} + +func clean(lf *LabelsFile) { + lf.Description = cleanString(lf.Description) + lf.Repo = cleanString(lf.Repo) + + cleanCategories(lf) + cleanLabels(lf) +} diff --git a/tests/cmd/github-labels/display.go b/tests/cmd/github-labels/display.go new file mode 100644 index 0000000000..888d514230 --- /dev/null +++ b/tests/cmd/github-labels/display.go @@ -0,0 +1,83 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "os" + "sort" +) + +var outputFile = os.Stdout + +// displayHandler is an interface that all output display handlers +// (formatters) must implement. +type DisplayHandler interface { + DisplayLabels(lf *LabelsFile) error + DisplayCategories(lf *LabelsFile, showLabels bool) error +} + +// DisplayHandlers encapsulates the list of available display handlers. +type DisplayHandlers struct { + handlers map[string]DisplayHandler +} + +// handlers is a map of the available output format display handling +// implementations. +var handlers map[string]DisplayHandler + +// NewDisplayHandlers create a new DisplayHandler. +func NewDisplayHandlers() *DisplayHandlers { + if handlers == nil { + handlers = make(map[string]DisplayHandler) + + handlers["md"] = NewDisplayMD(outputFile) + handlers[textFormat] = NewDisplayText(outputFile) + handlers["tsv"] = NewDisplayTSV(outputFile) + } + + h := &DisplayHandlers{ + handlers: handlers, + } + + return h +} + +// find looks for a display handler corresponding to the specified format +func (d *DisplayHandlers) find(format string) DisplayHandler { + for f, handler := range d.handlers { + if f == format { + return handler + } + } + + return nil +} + +// Get returns a list of the available formatters (display handler names). +func (d *DisplayHandlers) Get() []string { + var formats []string + + for f := range d.handlers { + formats = append(formats, f) + } + + sort.Strings(formats) + + return formats +} + +func show(inputFilename string, handler DisplayHandler, what DataToShow, withLabels bool) error { + lf, err := readYAML(inputFilename) + if err != nil { + return err + } + + if what == showLabels { + return handler.DisplayLabels(lf) + } + + return handler.DisplayCategories(lf, withLabels) +} diff --git a/tests/cmd/github-labels/display_markdown.go b/tests/cmd/github-labels/display_markdown.go new file mode 100644 index 0000000000..7ac87e7366 --- /dev/null +++ b/tests/cmd/github-labels/display_markdown.go @@ -0,0 +1,75 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "os" + + "github.com/olekukonko/tablewriter" +) + +type displayMD struct { + writer *tablewriter.Table +} + +func NewDisplayMD(file *os.File) DisplayHandler { + md := &displayMD{} + + md.writer = tablewriter.NewWriter(file) + md.writer.SetCenterSeparator("|") + + md.writer.SetBorders(tablewriter.Border{ + Left: true, + Right: true, + Top: false, + Bottom: false, + }) + + // Critical for GitHub Flavoured Markdown + md.writer.SetAutoWrapText(false) + + return md +} + +func (d *displayMD) render(headerFields []string, records [][]string) { + d.writer.SetHeader(headerFields) + d.writer.AppendBulk(records) + d.writer.Render() +} + +func (d *displayMD) DisplayLabels(lf *LabelsFile) error { + var records [][]string + + for _, l := range lf.Labels { + record := labelToRecord(l, true) + records = append(records, record) + } + + headerFields := labelHeaderRecord() + + d.render(headerFields, records) + + return nil +} + +func (d *displayMD) DisplayCategories(lf *LabelsFile, showLabels bool) error { + headerFields := categoryHeaderRecord(showLabels) + + var records [][]string + + for _, c := range lf.Categories { + record, err := categoryToRecord(lf, c, showLabels, true) + if err != nil { + return err + } + + records = append(records, record) + } + + d.render(headerFields, records) + + return nil +} diff --git a/tests/cmd/github-labels/display_text.go b/tests/cmd/github-labels/display_text.go new file mode 100644 index 0000000000..63211975d5 --- /dev/null +++ b/tests/cmd/github-labels/display_text.go @@ -0,0 +1,101 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + "os" +) + +type displayText struct { + file *os.File +} + +func NewDisplayText(file *os.File) DisplayHandler { + return &displayText{ + file: file, + } +} + +func (d *displayText) DisplayLabels(lf *LabelsFile) error { + _, err := fmt.Fprintf(d.file, "Labels (count: %d):\n", len(lf.Labels)) + if err != nil { + return err + } + + for _, l := range lf.Labels { + err = d.displayLabel(l) + if err != nil { + return err + } + } + + return nil +} + +func (d *displayText) displayLabel(l Label) error { + _, err := fmt.Fprintf(d.file, " %s (%q) [category %q, colour %q, from %q]\n", + l.Name, + l.Description, + l.CategoryName, + l.Colour, + l.From) + + return err +} + +func (d *displayText) DisplayCategories(lf *LabelsFile, showLabels bool) error { + _, err := fmt.Fprintf(d.file, "Categories (count: %d):\n", len(lf.Categories)) + if err != nil { + return err + } + + for _, c := range lf.Categories { + err := d.displayCategory(c, lf, showLabels) + if err != nil { + return err + } + } + + return nil +} + +func (d *displayText) displayCategory(c Category, lf *LabelsFile, showLabels bool) error { + if showLabels { + labels, err := getLabelsByCategory(c.Name, lf) + if err != nil { + return err + } + + _, err = fmt.Fprintf(d.file, " %s (%q, label count: %d, url: %v)\n", + c.Name, + c.Description, + len(labels), + c.URL) + if err != nil { + return err + } + + for _, label := range labels { + _, err := fmt.Fprintf(d.file, " %s (%q)\n", + label.Name, + label.Description) + if err != nil { + return err + } + } + } else { + _, err := fmt.Printf(" %s (%q, url: %v)\n", + c.Name, + c.Description, + c.URL) + if err != nil { + return err + } + } + + return nil +} diff --git a/tests/cmd/github-labels/display_tsv.go b/tests/cmd/github-labels/display_tsv.go new file mode 100644 index 0000000000..e848003696 --- /dev/null +++ b/tests/cmd/github-labels/display_tsv.go @@ -0,0 +1,66 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "encoding/csv" + "os" +) + +type displayTSV struct { + writer *csv.Writer +} + +func NewDisplayTSV(file *os.File) DisplayHandler { + tsv := &displayTSV{} + tsv.writer = csv.NewWriter(file) + + // Tab separator + tsv.writer.Comma = rune('\t') + + return tsv +} + +func (d *displayTSV) DisplayLabels(lf *LabelsFile) error { + record := labelHeaderRecord() + if err := d.writer.Write(record); err != nil { + return err + } + + for _, l := range lf.Labels { + record := labelToRecord(l, false) + + if err := d.writer.Write(record); err != nil { + return err + } + } + + d.writer.Flush() + + return d.writer.Error() +} + +func (d *displayTSV) DisplayCategories(lf *LabelsFile, showLabels bool) error { + record := categoryHeaderRecord(showLabels) + if err := d.writer.Write(record); err != nil { + return err + } + + for _, c := range lf.Categories { + record, err := categoryToRecord(lf, c, showLabels, false) + if err != nil { + return err + } + + if err := d.writer.Write(record); err != nil { + return err + } + } + + d.writer.Flush() + + return d.writer.Error() +} diff --git a/tests/cmd/github-labels/github-labels.sh b/tests/cmd/github-labels/github-labels.sh new file mode 100755 index 0000000000..e5bf4a7359 --- /dev/null +++ b/tests/cmd/github-labels/github-labels.sh @@ -0,0 +1,176 @@ +#!/bin/bash +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Description: Generate the combined GitHub labels database for the +# specified repository. + +set -e + +script_name=${0##*/} + +source "/etc/os-release" || "source /usr/lib/os-release" + +self_dir=$(dirname "$(readlink -f "$0")") +cidir="${self_dir}/../../.ci" +source "${cidir}/lib.sh" + +typeset -r labels_file="labels.yaml" +typeset -r labels_template="${labels_file}.in" + +typeset -r master_labels_file="${self_dir}/${labels_file}" +typeset -r master_labels_template="${self_dir}/${labels_template}" + +# The GitHub labels API requires a colour for each label so +# default to a white background. +typeset -r default_color="ffffff" + +need_yq() { + # install yq if not exist + ${cidir}/install_yq.sh + + command -v yq &>/dev/null || \ + die 'yq command not found. Ensure "$GOPATH/bin" is in your $PATH.' +} + +merge_yaml() +{ + local -r file1="$1" + local -r file2="$2" + local -r out="$3" + + [ -n "$file1" ] || die "need 1st file" + [ -n "$file2" ] || die "need 2nd file" + [ -n "$out" ] || die "need output file" + + need_yq + yq merge "$file1" --append "$file2" > "$out" +} + +check_yaml() +{ + local -r file="$1" + + [ -n "$file" ] || die "need file to check" + + need_yq + yq read "$file" >/dev/null + + [ -z "$(command -v yamllint)" ] && die "need yamllint installed" + + # Deal with different versions of the tool + local opts="" + local has_strict_opt=$(yamllint --help 2>&1|grep -- --strict) + + [ -n "$has_strict_opt" ] && opts+="--strict" + + yamllint $opts "$file" +} + +# Expand the variables in the labels database. +generate_yaml() +{ + local repo="$1" + local template="$2" + local out="$3" + + [ -n "$repo" ] || die "need repo" + [ -n "$template" ] || die "need template" + [ -n "$out" ] || die "need output file" + + local repo_slug=$(echo "${repo}"|sed 's!github.com/!!g') + + sed \ + -e "s|REPO_SLUG|${repo_slug}|g" \ + -e "s|DEFAULT_COLOUR|${default_color}|g" \ + "$template" > "$out" + + check_yaml "$out" +} + +cmd_generate() +{ + local repo="$1" + local out_file="$2" + + [ -n "$repo" ] || die "need repo" + [ -n "$out_file" ] || die "need output file" + + # Create the master database from the template + generate_yaml \ + "${repo}" \ + "${master_labels_template}" \ + "${master_labels_file}" + + local -r repo_labels_template="${GOPATH}/src/${repo}/${labels_template}" + local -r repo_labels_file="${GOPATH}/src/${repo}/${labels_file}" + + # Check for a repo-specific set of labels + if [ -e "${repo_labels_template}" ]; then + info "Found repo-specific labels database" + + # Generate repo-specific labels from template + generate_yaml \ + "${repo}" \ + "${repo_labels_template}" \ + "${repo_labels_file}" + + # Combine the two databases + tmp=$(mktemp) + + merge_yaml \ + "${master_labels_file}" \ + "${repo_labels_file}" \ + "${tmp}" + + mv "${tmp}" "${out_file}" + else + info "No repo-specific labels database" + cp "${master_labels_file}" "${out_file}" + fi + + + info "Generated labels database ${out_file}" + + # Perform checks + kata-github-labels check "${out_file}" +} + +usage() +{ + cat < + +Examples: + + # Generate combined labels database for runtime repo and write to + # specified file + \$ ${script_name} generate github.com/kata-containers/kata-containers /tmp/out.yaml + +EOF +} + +main() +{ + case "$1" in + generate) + shift + cmd_generate "$@" + ;; + + help|"") + usage + exit 0 + ;; + + *) + die "Invalid command: '$1'" + ;; + esac +} + +main "$@" diff --git a/tests/cmd/github-labels/labels.yaml b/tests/cmd/github-labels/labels.yaml new file mode 100644 index 0000000000..c8e69027b6 --- /dev/null +++ b/tests/cmd/github-labels/labels.yaml @@ -0,0 +1,555 @@ +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +--- +description: | + This file contains a list of all the generic GitHub labels used by all Kata + Containers GitHub repositories. + + Each repository can optionally contain a top-level `labels.yaml` that + specifies a list of repository-specific labels (and possibly additional + categories). The labels in the repository-specific labels file plus the + labels defined in this file define the minimum list of labels for the + repository in question. + + Each label must specify: + + - Name (which must be lower-case without spaces) + - Description + - Category + - Colour (explicit colour, or `ffffff`) + + A label may also specify a "From" value. This is used for renaming labels; + if a label has an associated "From" value, an existing label whose name is + specified by the "From" value will be renamed to the label name. + + A category is a collective name used to describe one or more related labels. + Each category must specify: + + - Name (which must be lower-case without spaces) + - Description + + A category may also specify a related URL which points to a document + containing further information. + +categories: + - name: api + description: Change related to an Application Programming Interface. + + - name: architecture-committee + description: Needs input from the Architecture Committee. + url: https://github.com/kata-containers/community#architecture-committee + + - name: area + description: Code component / general part of product affected. + + - name: backport + description: | + Code that needs to be applied to other branches, generally older stable + ones. + + - name: behaviour + description: | + How the issue affect the operation of the system. A more precise version + of regression. + + - name: block + description: | + Stop a PR from being merged. + + - name: cleanup + description: Refactoring, restructuring or general tidy-up needed. + + - name: customer + description: Related to a customer. + + - name: design + description: Requires formal review on the approach to solving the problem. + + - name: detail + description: Need further information from the user or author. + + - name: documentation + description: Needs more documentation. + + - name: environment + description: Related to particular system environment. + + - name: help + description: | + Request for technical help / extra resource. Also used for assisted + workflow. + + - name: label-admin + description: Relates to the administration of labels. + + - name: limitation + description: | + Issue cannot be resolved (too hard/impossible, would be too slow, + insufficient resources, etc). + url: | + https://github.com/kata-containers/kata-containers/blob/main/docs/Documentation-Requirements.md + + - name: new-contributor + description: Small, self-contained tasks suitable for newcomers. + url: | + https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md + + - name: priority + description: | + Relative urgency (time-critical). + + - name: question + description: Needs input from the team. + + - name: rebase + description: Code conflicts need to be resolved. + + - name: related + description: | + Related project. Base set can be generated from + https://github.com/kata-containers/kata-containers/blob/main/versions.yaml. + + - name: release + description: Related to production of new versions. + + - name: resolution + description: | + Issue is not (or no longer) valid for some reason. Label specifies + reason for closing. + + - name: security + description: Potential or actual vulnerability / CVE. + url: https://github.com/kata-containers/community/blob/main/VMT/VMT.md + + - name: severity + description: Relative importance (mission-critical). + + - name: sizing + description: Estimate of the complexity of the task (story points). + + - name: sub-type + description: More specific detail on the type category. + + - name: team + description: Team that needs to analyse the issue. + + - name: test + description: New tests needed. + + - name: type + description: High-level summary of the issue. + + - name: vendor + description: Related to handling imported code. + url: | + https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md#re-vendor-prs + +repo: kata-containers/kata-containers + +labels: + - name: api-breakage + description: API was broken + category: api + color: ff0000 + + - name: api-change + description: API change + category: api + color: ffffff + + - name: architecture-specific + description: Affects subset of architectures + category: environment + color: ffffff + + - name: area/api + description: Application Programming Interface + category: area + color: ffffff + + - name: area/cli + description: Command Line Interface (flags/options and arguments) + category: area + color: ffffff + + - name: area/comms + description: Communications (gRPC, Yamux, etc) + category: area + color: ffffff + + - name: area/config + description: Configuration + category: area + color: ffffff + + - name: area/logging + description: Logging + category: area + color: ffffff + + - name: area/networking + description: Networking + category: area + color: ffffff + + - name: area/storage + description: Storage + category: area + color: ffffff + + - name: area/tracing + description: Tracing + category: area + color: ffffff + + - name: backport + description: Code needs to be applied to older (stable) releases + category: backport + color: ffffff + + - name: bug + description: Incorrect behaviour + category: type + color: ff0000 + + - name: cannot-reproduce + description: Issue cannot be recreated + category: resolution + color: ffffff + + - name: cleanup + description: General tidy-up + category: cleanup + color: ffffff + + - name: crash + description: Causes part of the system to crash + category: behaviour + color: ffffff + + - name: customer + description: Relates to a customer + category: customer + color: ffffff + + - name: data-loss + description: System loses information + category: behaviour + color: ffffff + + - name: deprecate + description: Highlight a feature that will soon be removed + category: cleanup + color: ffffff + + - name: do-not-merge + description: PR has problems or depends on another + category: block + color: ff0000 + + - name: duplicate + description: Same issue as one already reported + category: resolution + color: ffffff + + - name: enhancement + description: Improvement to an existing feature + category: type + color: ffffff + + - name: feature + description: New functionality + category: type + color: ffffff + + - name: good-first-issue + description: Small and simple task for new contributors + category: new-contributor + color: ffffff + + - name: hang + description: System appears to stop operating or freeze + category: behaviour + color: ffffff + + - name: high-priority + description: Very urgent issue (resolve quickly) + category: priority + color: ff7f00 + + - name: high-severity + description: Very important issue + category: severity + color: 00d7ff + + - name: highest-priority + description: Critically urgent issue (must be resolved as soon as possible) + category: priority + color: ff0000 + + - name: highest-severity + description: Extremely important issue + category: severity + color: 00ffff + + - name: invalid + description: Issue does not make sense + category: resolution + color: ffffff + + - name: limitation + description: Issue cannot be resolved + category: limitation + color: ffffff + + - name: medium-priority + description: Urgent issue (resolve before unprioritised issues) + category: priority + color: ffff00 + + - name: medium-severity + description: Important issue + category: severity + color: 0000ff + + - name: needs-decision + description: Requires input from the Architecture Committee + category: architecture-committee + color: ffffff + + - name: needs-design-doc + description: Needs a document explaining the design + category: design + color: ffffff + + - name: needs-design-review + description: Needs a formal design review of the approach + category: design + color: ffffff + + - name: needs-docs + description: Needs some new or updated documentation + category: documentation + color: ffffff + + - name: needs-help + description: Request for extra help (technical, resource, etc) + category: help + color: ffffff + + - name: needs-integration-tests + description: | + Needs new system/integration tests to validate behaviour in the tests + repository + category: test + color: ffffff + + - name: needs-more-info + description: Blocked until user or author provides further details + category: detail + color: ffffff + + - name: needs-new-label + description: New label required to categorise this issue + category: label-admin + color: ffffff + + - name: needs-rebase + description: PR contains conflicts which need resolving + category: rebase + color: ffffff + + - name: needs-revendor + description: Needs imported code to be re-vendored + category: vendor + color: ffffff + + - name: needs-review + description: Needs to be assessed by the team. + category: team + color: 00ff00 + + - name: needs-unit-tests + description: Needs new unit tests to validate behaviour in this repository + category: test + color: ffffff + + - name: os-specific + description: Affects subset of operating system / distro versions + category: environment + color: ffffff + + - name: performance + description: System runs too slowly + category: behaviour + color: ffffff + + - name: question + description: Requires an answer + category: question + color: ffffff + + - name: refactor + description: Remove duplication, improve organisation, etc + category: cleanup + color: ffffff + + - name: regression + description: Behaviour inadvertently reverted to older behaviour + category: sub-type + color: ffffff + + - name: related/containerd + description: Containerd + category: related + color: ffffff + + - name: related/cri + description: CRI + category: related + color: ffffff + + - name: related/crio + description: CRIO + category: related + color: ffffff + + - name: related/docker + description: Docker + category: related + color: ffffff + + - name: related/firecracker + description: Firecracker + category: related + color: ffffff + + - name: related/k8s + description: Kubernetes + category: related + color: ffffff + + - name: related/qemu + description: QEMU + category: related + color: ffffff + + - name: related/runc + description: Runc + category: related + color: ffffff + + - name: release-gating + description: Release must wait for this to be resolved before release + category: release + color: ffffff + + - name: resource-hog + description: System uses too many resources (such as memory) + category: behaviour + color: ffffff + + - name: resource-leak + description: System does not free resources (such as memory) + category: behaviour + color: ffffff + + - name: rfc + description: Requires input from the team + category: question + color: ffffff + + - name: security + description: Potential or actual security issue + category: security + color: ff0000 + + - name: size/huge + description: | + Largest and most complex task (probably needs breaking into small + pieces) + category: sizing + color: ffffff + + - name: size/large + description: Task of significant size + category: sizing + color: ffffff + + - name: size/medium + description: Average sized task + category: sizing + color: ffffff + + - name: size/small + description: Small and simple task + category: sizing + color: ffffff + + - name: size/tiny + description: Smallest and simplest task + category: sizing + color: ffffff + + - name: stale + description: Issue or PR was not updated in a timely fashion + category: resolution + color: ffffff + + - name: team/ci + description: Need Continuous Integration Team input + category: team + color: ffffff + + - name: team/developer + description: Need Developer Team input + category: team + color: ffffff + + - name: team/documentation + description: Need Documentation Team input + category: team + color: ffffff + + - name: team/kernel + description: Need Kernel Team input + category: team + color: ffffff + + - name: team/metrics + description: Need Metrics Team input + category: team + color: ffffff + + - name: team/packaging + description: Need Packaging Team input + category: team + color: ffffff + + - name: team/test + description: Need Test Team input + category: team + color: ffffff + + - name: unreliable + description: Part of the system is not stable + category: behaviour + color: ffffff + + - name: wip + description: Work in Progress (PR incomplete - needs more work or rework) + category: block + color: ff0000 + + - name: wont-fix + description: Issue will not be fixed (not a good use of limited resources) + category: resolution + color: ffffff + + - name: wrong-repo + description: Raised in incorrect repository + category: resolution + color: ffffff diff --git a/tests/cmd/github-labels/labels.yaml.in b/tests/cmd/github-labels/labels.yaml.in new file mode 100644 index 0000000000..65d5ea69f3 --- /dev/null +++ b/tests/cmd/github-labels/labels.yaml.in @@ -0,0 +1,555 @@ +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +--- +description: | + This file contains a list of all the generic GitHub labels used by all Kata + Containers GitHub repositories. + + Each repository can optionally contain a top-level `labels.yaml` that + specifies a list of repository-specific labels (and possibly additional + categories). The labels in the repository-specific labels file plus the + labels defined in this file define the minimum list of labels for the + repository in question. + + Each label must specify: + + - Name (which must be lower-case without spaces) + - Description + - Category + - Colour (explicit colour, or `DEFAULT_COLOUR`) + + A label may also specify a "From" value. This is used for renaming labels; + if a label has an associated "From" value, an existing label whose name is + specified by the "From" value will be renamed to the label name. + + A category is a collective name used to describe one or more related labels. + Each category must specify: + + - Name (which must be lower-case without spaces) + - Description + + A category may also specify a related URL which points to a document + containing further information. + +categories: + - name: api + description: Change related to an Application Programming Interface. + + - name: architecture-committee + description: Needs input from the Architecture Committee. + url: https://github.com/kata-containers/community#architecture-committee + + - name: area + description: Code component / general part of product affected. + + - name: backport + description: | + Code that needs to be applied to other branches, generally older stable + ones. + + - name: behaviour + description: | + How the issue affect the operation of the system. A more precise version + of regression. + + - name: block + description: | + Stop a PR from being merged. + + - name: cleanup + description: Refactoring, restructuring or general tidy-up needed. + + - name: customer + description: Related to a customer. + + - name: design + description: Requires formal review on the approach to solving the problem. + + - name: detail + description: Need further information from the user or author. + + - name: documentation + description: Needs more documentation. + + - name: environment + description: Related to particular system environment. + + - name: help + description: | + Request for technical help / extra resource. Also used for assisted + workflow. + + - name: label-admin + description: Relates to the administration of labels. + + - name: limitation + description: | + Issue cannot be resolved (too hard/impossible, would be too slow, + insufficient resources, etc). + url: | + https://github.com/kata-containers/kata-containers/blob/main/docs/Documentation-Requirements.md + + - name: new-contributor + description: Small, self-contained tasks suitable for newcomers. + url: | + https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md + + - name: priority + description: | + Relative urgency (time-critical). + + - name: question + description: Needs input from the team. + + - name: rebase + description: Code conflicts need to be resolved. + + - name: related + description: | + Related project. Base set can be generated from + https://github.com/kata-containers/kata-containers/blob/main/versions.yaml. + + - name: release + description: Related to production of new versions. + + - name: resolution + description: | + Issue is not (or no longer) valid for some reason. Label specifies + reason for closing. + + - name: security + description: Potential or actual vulnerability / CVE. + url: https://github.com/kata-containers/community/blob/main/VMT/VMT.md + + - name: severity + description: Relative importance (mission-critical). + + - name: sizing + description: Estimate of the complexity of the task (story points). + + - name: sub-type + description: More specific detail on the type category. + + - name: team + description: Team that needs to analyse the issue. + + - name: test + description: New tests needed. + + - name: type + description: High-level summary of the issue. + + - name: vendor + description: Related to handling imported code. + url: | + https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md#re-vendor-prs + +repo: REPO_SLUG + +labels: + - name: api-breakage + description: API was broken + category: api + color: ff0000 + + - name: api-change + description: API change + category: api + color: DEFAULT_COLOUR + + - name: architecture-specific + description: Affects subset of architectures + category: environment + color: DEFAULT_COLOUR + + - name: area/api + description: Application Programming Interface + category: area + color: DEFAULT_COLOUR + + - name: area/cli + description: Command Line Interface (flags/options and arguments) + category: area + color: DEFAULT_COLOUR + + - name: area/comms + description: Communications (gRPC, Yamux, etc) + category: area + color: DEFAULT_COLOUR + + - name: area/config + description: Configuration + category: area + color: DEFAULT_COLOUR + + - name: area/logging + description: Logging + category: area + color: DEFAULT_COLOUR + + - name: area/networking + description: Networking + category: area + color: DEFAULT_COLOUR + + - name: area/storage + description: Storage + category: area + color: DEFAULT_COLOUR + + - name: area/tracing + description: Tracing + category: area + color: DEFAULT_COLOUR + + - name: backport + description: Code needs to be applied to older (stable) releases + category: backport + color: DEFAULT_COLOUR + + - name: bug + description: Incorrect behaviour + category: type + color: ff0000 + + - name: cannot-reproduce + description: Issue cannot be recreated + category: resolution + color: DEFAULT_COLOUR + + - name: cleanup + description: General tidy-up + category: cleanup + color: DEFAULT_COLOUR + + - name: crash + description: Causes part of the system to crash + category: behaviour + color: DEFAULT_COLOUR + + - name: customer + description: Relates to a customer + category: customer + color: DEFAULT_COLOUR + + - name: data-loss + description: System loses information + category: behaviour + color: DEFAULT_COLOUR + + - name: deprecate + description: Highlight a feature that will soon be removed + category: cleanup + color: DEFAULT_COLOUR + + - name: do-not-merge + description: PR has problems or depends on another + category: block + color: ff0000 + + - name: duplicate + description: Same issue as one already reported + category: resolution + color: DEFAULT_COLOUR + + - name: enhancement + description: Improvement to an existing feature + category: type + color: DEFAULT_COLOUR + + - name: feature + description: New functionality + category: type + color: DEFAULT_COLOUR + + - name: good-first-issue + description: Small and simple task for new contributors + category: new-contributor + color: DEFAULT_COLOUR + + - name: hang + description: System appears to stop operating or freeze + category: behaviour + color: DEFAULT_COLOUR + + - name: high-priority + description: Very urgent issue (resolve quickly) + category: priority + color: ff7f00 + + - name: high-severity + description: Very important issue + category: severity + color: 00d7ff + + - name: highest-priority + description: Critically urgent issue (must be resolved as soon as possible) + category: priority + color: ff0000 + + - name: highest-severity + description: Extremely important issue + category: severity + color: 00ffff + + - name: invalid + description: Issue does not make sense + category: resolution + color: DEFAULT_COLOUR + + - name: limitation + description: Issue cannot be resolved + category: limitation + color: DEFAULT_COLOUR + + - name: medium-priority + description: Urgent issue (resolve before unprioritised issues) + category: priority + color: ffff00 + + - name: medium-severity + description: Important issue + category: severity + color: 0000ff + + - name: needs-decision + description: Requires input from the Architecture Committee + category: architecture-committee + color: DEFAULT_COLOUR + + - name: needs-design-doc + description: Needs a document explaining the design + category: design + color: DEFAULT_COLOUR + + - name: needs-design-review + description: Needs a formal design review of the approach + category: design + color: DEFAULT_COLOUR + + - name: needs-docs + description: Needs some new or updated documentation + category: documentation + color: DEFAULT_COLOUR + + - name: needs-help + description: Request for extra help (technical, resource, etc) + category: help + color: DEFAULT_COLOUR + + - name: needs-integration-tests + description: | + Needs new system/integration tests to validate behaviour in the tests + repository + category: test + color: DEFAULT_COLOUR + + - name: needs-more-info + description: Blocked until user or author provides further details + category: detail + color: DEFAULT_COLOUR + + - name: needs-new-label + description: New label required to categorise this issue + category: label-admin + color: DEFAULT_COLOUR + + - name: needs-rebase + description: PR contains conflicts which need resolving + category: rebase + color: DEFAULT_COLOUR + + - name: needs-revendor + description: Needs imported code to be re-vendored + category: vendor + color: DEFAULT_COLOUR + + - name: needs-review + description: Needs to be assessed by the team. + category: team + color: 00ff00 + + - name: needs-unit-tests + description: Needs new unit tests to validate behaviour in this repository + category: test + color: DEFAULT_COLOUR + + - name: os-specific + description: Affects subset of operating system / distro versions + category: environment + color: DEFAULT_COLOUR + + - name: performance + description: System runs too slowly + category: behaviour + color: DEFAULT_COLOUR + + - name: question + description: Requires an answer + category: question + color: DEFAULT_COLOUR + + - name: refactor + description: Remove duplication, improve organisation, etc + category: cleanup + color: DEFAULT_COLOUR + + - name: regression + description: Behaviour inadvertently reverted to older behaviour + category: sub-type + color: DEFAULT_COLOUR + + - name: related/containerd + description: Containerd + category: related + color: DEFAULT_COLOUR + + - name: related/cri + description: CRI + category: related + color: DEFAULT_COLOUR + + - name: related/crio + description: CRIO + category: related + color: DEFAULT_COLOUR + + - name: related/docker + description: Docker + category: related + color: DEFAULT_COLOUR + + - name: related/firecracker + description: Firecracker + category: related + color: DEFAULT_COLOUR + + - name: related/k8s + description: Kubernetes + category: related + color: DEFAULT_COLOUR + + - name: related/qemu + description: QEMU + category: related + color: DEFAULT_COLOUR + + - name: related/runc + description: Runc + category: related + color: DEFAULT_COLOUR + + - name: release-gating + description: Release must wait for this to be resolved before release + category: release + color: DEFAULT_COLOUR + + - name: resource-hog + description: System uses too many resources (such as memory) + category: behaviour + color: DEFAULT_COLOUR + + - name: resource-leak + description: System does not free resources (such as memory) + category: behaviour + color: DEFAULT_COLOUR + + - name: rfc + description: Requires input from the team + category: question + color: DEFAULT_COLOUR + + - name: security + description: Potential or actual security issue + category: security + color: ff0000 + + - name: size/huge + description: | + Largest and most complex task (probably needs breaking into small + pieces) + category: sizing + color: DEFAULT_COLOUR + + - name: size/large + description: Task of significant size + category: sizing + color: DEFAULT_COLOUR + + - name: size/medium + description: Average sized task + category: sizing + color: DEFAULT_COLOUR + + - name: size/small + description: Small and simple task + category: sizing + color: DEFAULT_COLOUR + + - name: size/tiny + description: Smallest and simplest task + category: sizing + color: DEFAULT_COLOUR + + - name: stale + description: Issue or PR was not updated in a timely fashion + category: resolution + color: DEFAULT_COLOUR + + - name: team/ci + description: Need Continuous Integration Team input + category: team + color: DEFAULT_COLOUR + + - name: team/developer + description: Need Developer Team input + category: team + color: DEFAULT_COLOUR + + - name: team/documentation + description: Need Documentation Team input + category: team + color: DEFAULT_COLOUR + + - name: team/kernel + description: Need Kernel Team input + category: team + color: DEFAULT_COLOUR + + - name: team/metrics + description: Need Metrics Team input + category: team + color: DEFAULT_COLOUR + + - name: team/packaging + description: Need Packaging Team input + category: team + color: DEFAULT_COLOUR + + - name: team/test + description: Need Test Team input + category: team + color: DEFAULT_COLOUR + + - name: unreliable + description: Part of the system is not stable + category: behaviour + color: DEFAULT_COLOUR + + - name: wip + description: Work in Progress (PR incomplete - needs more work or rework) + category: block + color: ff0000 + + - name: wont-fix + description: Issue will not be fixed (not a good use of limited resources) + category: resolution + color: DEFAULT_COLOUR + + - name: wrong-repo + description: Raised in incorrect repository + category: resolution + color: DEFAULT_COLOUR diff --git a/tests/cmd/github-labels/main.go b/tests/cmd/github-labels/main.go new file mode 100644 index 0000000000..31a1b9b1d7 --- /dev/null +++ b/tests/cmd/github-labels/main.go @@ -0,0 +1,157 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +// Description: Program to check and summarise the Kata GitHub +// labels YAML file. + +package main + +import ( + "errors" + "fmt" + "os" + + "github.com/urfave/cli" +) + +type DataToShow int + +const ( + showLabels DataToShow = iota + showCategories DataToShow = iota + + textFormat = "text" + defaultOutputFormat = textFormat +) + +var errNeedYAMLFile = errors.New("need YAML file") + +var ( + // set by the build + name = "" + version = "" + commit = "" + + debug = false +) + +var formatFlag = cli.StringFlag{ + Name: "format", + Usage: "display in specified format ('help' to show all)", + Value: defaultOutputFormat, +} + +func commonHandler(context *cli.Context, what DataToShow, withLabels bool) error { + handlers := NewDisplayHandlers() + + format := context.String("format") + if format == "help" { + availableFormats := handlers.Get() + + for _, format := range availableFormats { + fmt.Fprintf(outputFile, "%s\n", format) + } + + return nil + } + + handler := handlers.find(format) + if handler == nil { + return fmt.Errorf("no handler for format %q", format) + } + + if context.NArg() == 0 { + return errNeedYAMLFile + } + + file := context.Args().Get(0) + + return show(file, handler, what, withLabels) +} + +func main() { + app := cli.NewApp() + app.Description = "tool to manipulate Kata GitHub labels" + app.Usage = app.Description + app.Version = fmt.Sprintf("%s %s (commit %v)", name, version, commit) + + app.Flags = []cli.Flag{ + cli.BoolFlag{ + Name: "debug, d", + Usage: "enable debug output", + Destination: &debug, + }, + } + + app.Commands = []cli.Command{ + { + Name: "check", + Usage: "Perform tests on the labels database", + Description: "Exit code denotes success", + Action: func(context *cli.Context) error { + if context.NArg() == 0 { + return errNeedYAMLFile + } + + file := context.Args().Get(0) + + return checkYAML(file) + }, + }, + { + Name: "show", + Usage: "Display labels database details", + Subcommands: []cli.Command{ + { + Name: "categories", + Usage: "Display categories from labels database", + Flags: []cli.Flag{ + formatFlag, + cli.BoolFlag{ + Name: "with-labels", + Usage: "Add labels in each category to output", + }, + }, + Action: func(context *cli.Context) error { + withLabels := context.Bool("with-labels") + return commonHandler(context, showCategories, withLabels) + }, + }, + { + Name: "labels", + Usage: "Display labels from labels database", + Flags: []cli.Flag{ + formatFlag, + }, + Action: func(context *cli.Context) error { + withLabels := context.Bool("with-labels") + return commonHandler(context, showLabels, withLabels) + }, + }, + }, + }, + { + Name: "sort", + Usage: "Sort the specified YAML labels file and write to a new file", + Description: "Can be used to keep the master labels file sorted", + ArgsUsage: " ", + Action: func(context *cli.Context) error { + if context.NArg() != 2 { + return errors.New("need two YAML files: ") + } + + from := context.Args().Get(0) + to := context.Args().Get(1) + return sortYAML(from, to) + }, + }, + } + + err := app.Run(os.Args) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) + os.Exit(1) + } +} diff --git a/tests/cmd/github-labels/record.go b/tests/cmd/github-labels/record.go new file mode 100644 index 0000000000..6bdc12b946 --- /dev/null +++ b/tests/cmd/github-labels/record.go @@ -0,0 +1,102 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + "strings" +) + +const ( + labelNamesSeparator = "," +) + +func labelToRecord(l Label, quote bool) (record []string) { + name := l.Name + category := l.CategoryName + colour := l.Colour + from := l.From + + if quote { + name = fmt.Sprintf("`%s`", l.Name) + category = fmt.Sprintf("`%s`", l.CategoryName) + colour = fmt.Sprintf("`%s`", l.Colour) + if from != "" { + from = fmt.Sprintf("`%s`", l.From) + } + } + + record = append(record, name) + record = append(record, l.Description) + record = append(record, category) + record = append(record, colour) + record = append(record, from) + + return record +} + +func labelHeaderRecord() []string { + return []string{ + "Name", + "Description", + "Category", + "Colour", + "From", + } +} + +func categoryHeaderRecord(showLabels bool) []string { + var fields []string + + fields = append(fields, "Name") + fields = append(fields, "Description") + fields = append(fields, "URL") + + if showLabels { + fields = append(fields, "Labels") + } + + return fields +} + +func categoryToRecord(lf *LabelsFile, c Category, showLabels, quote bool) ([]string, error) { + var record []string + + name := c.Name + + if quote { + name = fmt.Sprintf("`%s`", c.Name) + } + + record = append(record, name) + record = append(record, c.Description) + record = append(record, c.URL) + + if showLabels { + var labelNames []string + + labels, err := getLabelsByCategory(c.Name, lf) + if err != nil { + return nil, err + } + + for _, l := range labels { + labelName := l.Name + + if quote { + labelName = fmt.Sprintf("`%s`", l.Name) + } + + labelNames = append(labelNames, labelName) + } + + result := strings.Join(labelNames, labelNamesSeparator) + + record = append(record, result) + } + + return record, nil +} diff --git a/tests/cmd/github-labels/types.go b/tests/cmd/github-labels/types.go new file mode 100644 index 0000000000..9248f8c68a --- /dev/null +++ b/tests/cmd/github-labels/types.go @@ -0,0 +1,55 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +type Category struct { + Name string + Description string + URL string `yaml:",omitempty"` +} + +type Label struct { + Name string + Description string + CategoryName string `yaml:"category"` + Colour string `yaml:"color"` + From string `yaml:",omitempty"` +} + +type Categories []Category + +func (c Categories) Len() int { + return len(c) +} + +func (c Categories) Swap(i, j int) { + c[i], c[j] = c[j], c[i] +} + +func (c Categories) Less(i, j int) bool { + return c[i].Name < c[j].Name +} + +type Labels []Label + +func (l Labels) Len() int { + return len(l) +} + +func (l Labels) Swap(i, j int) { + l[i], l[j] = l[j], l[i] +} + +func (l Labels) Less(i, j int) bool { + return l[i].Name < l[j].Name +} + +type LabelsFile struct { + Description string + Categories Categories + Repo string + Labels Labels +} diff --git a/tests/cmd/github-labels/utils.go b/tests/cmd/github-labels/utils.go new file mode 100644 index 0000000000..4490d3a0df --- /dev/null +++ b/tests/cmd/github-labels/utils.go @@ -0,0 +1,24 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import "errors" + +func getLabelsByCategory(categoryName string, lf *LabelsFile) ([]Label, error) { + var labels []Label + + if categoryName == "" { + return nil, errors.New("need category name") + } + + for _, label := range lf.Labels { + if label.CategoryName == categoryName { + labels = append(labels, label) + } + } + + return labels, nil +} diff --git a/tests/cmd/github-labels/yaml.go b/tests/cmd/github-labels/yaml.go new file mode 100644 index 0000000000..d1dfb6aff6 --- /dev/null +++ b/tests/cmd/github-labels/yaml.go @@ -0,0 +1,72 @@ +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + "os" + "sort" + + yaml "gopkg.in/yaml.v2" +) + +const fileMode os.FileMode = 0600 + +func readYAML(file string) (*LabelsFile, error) { + bytes, err := os.ReadFile(file) + if err != nil { + return nil, err + } + + lf := LabelsFile{} + + err = yaml.Unmarshal(bytes, &lf) + if err != nil { + return nil, err + } + + sort.Sort(lf.Labels) + sort.Sort(lf.Categories) + + clean(&lf) + + err = check(&lf) + if err != nil { + return nil, fmt.Errorf("file was not in expected format: %v", err) + } + + return &lf, nil +} + +func writeYAML(lf *LabelsFile, file string) error { + bytes, err := yaml.Marshal(lf) + if err != nil { + return err + } + + return os.WriteFile(file, bytes, fileMode) +} + +func checkYAML(file string) error { + // read and check + _, err := readYAML(file) + + if err == nil { + fmt.Printf("Checked file %v\n", file) + } + + return err +} + +func sortYAML(fromFile, toFile string) error { + // read and sort + lf, err := readYAML(fromFile) + if err != nil { + return err + } + + return writeYAML(lf, toFile) +} From 6d9cb9325da2ef7f2201018c9919ec21f68c802a Mon Sep 17 00:00:00 2001 From: Chelsea Mafrica Date: Tue, 21 Nov 2023 17:43:04 -0800 Subject: [PATCH 5/6] tests: update scripts for static checks migration Updates to scripts for static-checks.sh functionality, including common functions location, the move of several common functions to the existing common.bash, adding hadolint and xurls to the versions file, and changes to static checks for running in the main kata containers repo. The changes to the vendor check include searching for existing go.mod files but no other changes to expand the test. Fixes #8187 Signed-off-by: Chelsea Mafrica --- tests/cmd/check-spelling/kata-spell-check.sh | 4 +- tests/cmd/github-labels/github-labels.sh | 4 +- tests/common.bash | 61 +++++++++ tests/static-checks.sh | 125 ++++++++++++++----- versions.yaml | 12 ++ 5 files changed, 171 insertions(+), 35 deletions(-) diff --git a/tests/cmd/check-spelling/kata-spell-check.sh b/tests/cmd/check-spelling/kata-spell-check.sh index ca7b2b8f09..e4b2ce37f1 100755 --- a/tests/cmd/check-spelling/kata-spell-check.sh +++ b/tests/cmd/check-spelling/kata-spell-check.sh @@ -26,9 +26,9 @@ then fi self_dir=$(dirname "$(readlink -f "$0")") -cidir="${self_dir}/../../.ci" +cidir="${self_dir}/../../../tests" -source "${cidir}/lib.sh" +source "${cidir}/common.bash" # Directory containing word lists. # diff --git a/tests/cmd/github-labels/github-labels.sh b/tests/cmd/github-labels/github-labels.sh index e5bf4a7359..b60ca9cc58 100755 --- a/tests/cmd/github-labels/github-labels.sh +++ b/tests/cmd/github-labels/github-labels.sh @@ -15,8 +15,8 @@ script_name=${0##*/} source "/etc/os-release" || "source /usr/lib/os-release" self_dir=$(dirname "$(readlink -f "$0")") -cidir="${self_dir}/../../.ci" -source "${cidir}/lib.sh" +cidir="${self_dir}/../.." +source "${cidir}/common.bash" typeset -r labels_file="labels.yaml" typeset -r labels_template="${labels_file}.in" diff --git a/tests/common.bash b/tests/common.bash index 9e9025bbf4..b7ac98c499 100644 --- a/tests/common.bash +++ b/tests/common.bash @@ -615,3 +615,64 @@ function arch_to_kernel() { *) die "unsupported architecture: ${arch}";; esac } + +# Obtain a list of the files the PR changed. +# Returns the information in format "${filter}\t${file}". +get_pr_changed_file_details_full() +{ + # List of filters used to restrict the types of file changes. + # See git-diff-tree(1) for further info. + local filters="" + + # Added file + filters+="A" + + # Copied file + filters+="C" + + # Modified file + filters+="M" + + # Renamed file + filters+="R" + + git diff-tree \ + -r \ + --name-status \ + --diff-filter="${filters}" \ + "origin/${branch}" HEAD +} + +# Obtain a list of the files the PR changed, ignoring vendor files. +# Returns the information in format "${filter}\t${file}". +get_pr_changed_file_details() +{ + get_pr_changed_file_details_full | grep -v "vendor/" +} + +function get_dep_from_yaml_db(){ + local versions_file="$1" + local dependency="$2" + + [ ! -f "$versions_file" ] && die "cannot find $versions_file" + + "${repo_root_dir}/ci/install_yq.sh" >&2 + + result=$("${GOPATH}/bin/yq" r -X "$versions_file" "$dependency") + [ "$result" = "null" ] && result="" + echo "$result" +} + +function get_test_version(){ + local dependency="$1" + + local db + local cidir + + # directory of this script, not the caller + local cidir=$(dirname "${BASH_SOURCE[0]}") + + db="${cidir}/../versions.yaml" + + get_dep_from_yaml_db "${db}" "${dependency}" +} diff --git a/tests/static-checks.sh b/tests/static-checks.sh index 8f747198c1..925306c7b2 100755 --- a/tests/static-checks.sh +++ b/tests/static-checks.sh @@ -13,15 +13,15 @@ set -e [ -n "$DEBUG" ] && set -x cidir=$(realpath $(dirname "$0")) -source "${cidir}/lib.sh" +source "${cidir}/common.bash" # By default in Golang >= 1.16 GO111MODULE is set to "on", # some subprojects in this repo may not support "go modules", # set GO111MODULE to "auto" to enable module-aware mode only when # a go.mod file is present in the current directory. export GO111MODULE="auto" -export tests_repo="${tests_repo:-github.com/kata-containers/tests}" -export tests_repo_dir="${GOPATH}/src/${tests_repo}" +export test_path="${test_path:-github.com/kata-containers/kata-containers/tests}" +export test_dir="${GOPATH}/src/${test_path}" # List of files to delete on exit files_to_remove=() @@ -38,6 +38,7 @@ typeset -r check_func_regex="^static_check_" typeset -r arch_func_regex="_arch_specific$" repo="" +repo_path="" specific_branch="false" force="false" branch=${branch:-main} @@ -248,6 +249,8 @@ static_check_go_arch_specific() local submodule_packages local all_packages + pushd $repo_path + # List of all golang packages found in all submodules # # These will be ignored: since they are references to other @@ -271,7 +274,7 @@ static_check_go_arch_specific() go_packages=$(skip_paths "${go_packages[@]}") # No packages to test - [ -z "$go_packages" ] && return + [ -z "$go_packages" ] && popd && return local linter="golangci-lint" @@ -280,11 +283,11 @@ static_check_go_arch_specific() then info "Installing ${linter}" - local linter_url=$(get_test_version "externals.golangci-lint.url") - local linter_version=$(get_test_version "externals.golangci-lint.version") + local linter_url=$(get_test_version "languages.golangci-lint.url") + local linter_version=$(get_test_version "languages.golangci-lint.version") info "Forcing ${linter} version ${linter_version}" - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin "${linter_version}" + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin "v${linter_version}" command -v $linter &>/dev/null || \ die "$linter command not found. Ensure that \"\$GOPATH/bin\" is in your \$PATH." fi @@ -320,6 +323,7 @@ static_check_go_arch_specific() info "Running $linter on $d" (cd $d && GO111MODULE=auto eval "$linter" "${linter_args}" ".") done + popd } @@ -357,13 +361,17 @@ static_check_versions() install_yamllint fi - [ ! -e "$db" ] && return + pushd $repo_path + + [ ! -e "$db" ] && popd && return if [ -n "$have_yamllint_cmd" ]; then eval "$yamllint_cmd" "$db" else info "Cannot check versions as $yamllint_cmd not available" fi + + popd } static_check_labels() @@ -375,7 +383,7 @@ static_check_labels() # Since this script is called from another repositories directory, # ensure the utility is built before the script below (which uses it) is run. - (cd "${tests_repo_dir}" && make github-labels) + (cd "${test_dir}" && make -C cmd/github-labels) tmp=$(mktemp) @@ -383,7 +391,7 @@ static_check_labels() info "Checking labels for repo ${repo} using temporary combined database ${tmp}" - bash -f "${tests_repo_dir}/cmd/github-labels/github-labels.sh" "generate" "${repo}" "${tmp}" + bash -f "${test_dir}/cmd/github-labels/github-labels.sh" "generate" "${repo}" "${tmp}" } # Ensure all files (where possible) contain an SPDX license header @@ -403,13 +411,15 @@ static_check_license_headers() header_checks+=("SPDX license header::${license_pattern}") header_checks+=("Copyright header:-i:${copyright_pattern}") + pushd $repo_path + files=$(get_pr_changed_file_details || true) # Strip off status files=$(echo "$files"|awk '{print $NF}') # no files were changed - [ -z "$files" ] && info "No files found" && return + [ -z "$files" ] && info "No files found" && popd && return local header_check @@ -443,7 +453,7 @@ static_check_license_headers() --exclude="*.drawio" \ --exclude="*.toml" \ --exclude="*.txt" \ - --exclude="*.dtd" \ + --exclude="*.dtd" \ --exclude="vendor/*" \ --exclude="VERSION" \ --exclude="kata_config_version" \ @@ -465,6 +475,7 @@ static_check_license_headers() --exclude="src/libs/protocols/protos/gogo/*.proto" \ --exclude="src/libs/protocols/protos/google/*.proto" \ --exclude="src/libs/*/test/texture/*" \ + --exclude="*.dic" \ -EL $extra_args "\<${pattern}\>" \ $files || true) @@ -478,6 +489,7 @@ EOF exit 1 fi done + popd } check_url() @@ -551,6 +563,8 @@ static_check_docs() { local cmd="xurls" + pushd $repo_path + if [ ! "$(command -v $cmd)" ] then info "Installing $cmd utility" @@ -578,6 +592,8 @@ static_check_docs() local new_urls local url + pushd $repo_path + all_docs=$(git ls-files "*.md" | grep -Ev "(grpc-rs|target)/" | sort || true) all_docs=$(skip_paths "${all_docs[@]}") @@ -631,7 +647,7 @@ static_check_docs() # is necessary to guarantee that all docs are referenced. md_docs_to_check="$all_docs" - (cd "${tests_repo_dir}" && make check-markdown) + (cd "${test_dir}" && make -C cmd/check-markdown) command -v kata-check-markdown &>/dev/null || \ die 'kata-check-markdown command not found. Ensure that "$GOPATH/bin" is in your $PATH.' @@ -668,6 +684,9 @@ static_check_docs() # since it displayed by default when visiting the repo. exclude_doc_regexs+=(^README\.md$) + # Exclude READMEs for test integration + exclude_doc_regexs+=(^\tests/cmd/.*/README\.md$) + local exclude_pattern # Convert the list of files into an egrep(1) alternation pattern. @@ -779,7 +798,7 @@ static_check_docs() fi # Now, spell check the docs - cmd="${tests_repo_dir}/cmd/check-spelling/kata-spell-check.sh" + cmd="${test_dir}/cmd/check-spelling/kata-spell-check.sh" local docs_failed=0 for doc in $docs @@ -789,6 +808,8 @@ static_check_docs() static_check_eof "$doc" done + popd + [ $docs_failed -eq 0 ] || die "spell check failed, See https://github.com/kata-containers/kata-containers/blob/main/docs/Documentation-Requirements.md#spelling for more information." } @@ -853,6 +874,8 @@ static_check_files() local matches="" + pushd $repo_path + for file in $files do local match @@ -879,6 +902,8 @@ static_check_files() matches+=" $match" done + popd + [ -z "$matches" ] && return echo >&2 -n \ @@ -907,20 +932,35 @@ static_check_files() # - Ensure vendor metadata is valid. static_check_vendor() { - local files - local vendor_files - local result + pushd $repo_path - # Check if repo has been changed to use go modules - if [ -f "go.mod" ]; then - info "go.mod file found, running go mod verify instead" - # This verifies the integrity of modules in the local cache. - # This does not really verify the integrity of vendored code: - # https://github.com/golang/go/issues/27348 - # Once that is added we need to add an extra step to verify vendored code. - go mod verify - return - fi + local files + local files_arr=() + + files=$(find . -type f -name "go.mod") + + while IFS= read -r line; do + files_arr+=("$line") + done <<< "$files" + + for file in "${files_arr[@]}"; do + local dir=$(echo $file | sed 's/go\.mod//') + + pushd $dir + + # Check if directory has been changed to use go modules + if [ -f "go.mod" ]; then + info "go.mod file found in $dir, running go mod verify instead" + # This verifies the integrity of modules in the local cache. + # This does not really verify the integrity of vendored code: + # https://github.com/golang/go/issues/27348 + # Once that is added we need to add an extra step to verify vendored code. + go mod verify + fi + popd + done + + popd } static_check_xml() @@ -928,6 +968,8 @@ static_check_xml() local all_xml local files + pushd $repo_path + need_chronic all_xml=$(git ls-files "*.xml" | grep -Ev "/(vendor|grpc-rs|target)/" | sort || true) @@ -947,7 +989,7 @@ static_check_xml() files=$(echo "$xml_status" | awk '{print $NF}') fi - [ -z "$files" ] && info "No XML files to check" && return + [ -z "$files" ] && info "No XML files to check" && popd && return local file @@ -975,6 +1017,8 @@ static_check_xml() [ "$ret" -eq 0 ] || die "failed to check XML file '$file'" done + + popd } static_check_shell() @@ -982,6 +1026,8 @@ static_check_shell() local all_scripts local scripts + pushd $repo_path + need_chronic all_scripts=$(git ls-files "*.sh" "*.bash" | grep -Ev "/(vendor|grpc-rs|target)/" | sort || true) @@ -1000,7 +1046,7 @@ static_check_shell() scripts=$(echo "$scripts_status" | awk '{print $NF}') fi - [ -z "$scripts" ] && info "No scripts to check" && return 0 + [ -z "$scripts" ] && info "No scripts to check" && popd && return 0 local script @@ -1016,6 +1062,8 @@ static_check_shell() static_check_eof "$script" done + + popd } static_check_json() @@ -1023,6 +1071,8 @@ static_check_json() local all_json local json_files + pushd $repo_path + need_chronic all_json=$(git ls-files "*.json" | grep -Ev "/(vendor|grpc-rs|target)/" | sort || true) @@ -1041,7 +1091,7 @@ static_check_json() json_files=$(echo "$json_status" | awk '{print $NF}') fi - [ -z "$json_files" ] && info "No JSON files to check" && return 0 + [ -z "$json_files" ] && info "No JSON files to check" && popd && return 0 local json @@ -1055,6 +1105,8 @@ static_check_json() [ "$ret" -eq 0 ] || die "failed to check JSON file '$json'" done + + popd } # The dockerfile checker relies on the hadolint tool. This function handle its @@ -1102,6 +1154,9 @@ static_check_dockerfiles() # Put here a list of files which should be ignored. local ignore_files=( ) + + pushd $repo_path + local linter_cmd="hadolint" all_files=$(git ls-files "*/Dockerfile*" | grep -Ev "/(vendor|grpc-rs|target)/" | sort || true) @@ -1119,11 +1174,12 @@ static_check_dockerfiles() files=$(echo "$files_status" | awk '{print $NF}') fi - [ -z "$files" ] && info "No Dockerfiles to check" && return 0 + [ -z "$files" ] && info "No Dockerfiles to check" && popd && return 0 # As of this writing hadolint is only distributed for x86_64 if [ "$(uname -m)" != "x86_64" ]; then info "Skip checking as $linter_cmd is not available for $(uname -m)" + popd return 0 fi has_hadolint_or_install @@ -1151,6 +1207,10 @@ static_check_dockerfiles() # DL3037 warning: Specify version with `zypper install -y =`. linter_cmd+=" --ignore DL3037" + # Temporary add to prevent failure for test migration + # DL3040 warning: `dnf clean all` missing after dnf command. + linter_cmd+=" --ignore DL3040" + local file for file in $files; do if echo "${ignore_files[@]}" | grep -q $file ; then @@ -1190,6 +1250,7 @@ static_check_dockerfiles() [ "$ret" -eq 0 ] || die "failed to check Dockerfile '$file'" done + popd } # Run the specified function (after first checking it is compatible with the @@ -1316,6 +1377,8 @@ main() fi fi + repo_path=$GOPATH/src/$repo + local all_check_funcs=$(typeset -F|awk '{print $3}'|grep "${check_func_regex}"|sort) # Run user-specified check and quit diff --git a/versions.yaml b/versions.yaml index 5dc0084c9a..4c7c4fedd0 100644 --- a/versions.yaml +++ b/versions.yaml @@ -249,6 +249,11 @@ externals: url: "http://ftp.gnu.org/pub/gnu/gperf/" version: "3.1" + hadolint: + description: "the dockerfile linter used by static-checks" + url: "https://github.com/hadolint/hadolint" + version: "2.12.0" + lvm2: description: "LVM2 and device-mapper tools and libraries" url: "https://github.com/lvmteam/lvm2" @@ -343,6 +348,12 @@ externals: # yamllint disable-line rule:line-length binary: "https://gitlab.com/virtio-fs/virtiofsd/uploads/9ec473efd0203219d016e66aac4190aa/virtiofsd-v1.8.0.zip" + xurls: + description: | + Tool used by the CI to check URLs in documents and code comments. + url: "mvdan.cc/xurls/v2/cmd/xurls" + version: "v2.5.0" + languages: description: | Details of programming languages required to build system @@ -371,6 +382,7 @@ languages: golangci-lint: description: "golangci-lint" notes: "'version' is the default minimum version used by this project." + url: "github.com/golangci/golangci-lint" version: "1.50.1" meta: description: | From 05efb232610b1921a694b7c0c0e82039aeb160b0 Mon Sep 17 00:00:00 2001 From: Chelsea Mafrica Date: Tue, 28 Nov 2023 17:40:41 -0800 Subject: [PATCH 6/6] tests: update go.mod and go.sum Generate a go.sum file for tests. Fixes #8187 Signed-off-by: Chelsea Mafrica --- tests/go.mod | 2 -- tests/go.sum | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 tests/go.sum diff --git a/tests/go.mod b/tests/go.mod index 61a26671c1..e033523e37 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -3,8 +3,6 @@ module github.com/kata-containers/tests go 1.19 require ( - github.com/BurntSushi/toml v0.3.1 - github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb github.com/olekukonko/tablewriter v0.0.6-0.20210304033056-74c60be0ef68 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.1 diff --git a/tests/go.sum b/tests/go.sum new file mode 100644 index 0000000000..02b3f32d4e --- /dev/null +++ b/tests/go.sum @@ -0,0 +1,40 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/olekukonko/tablewriter v0.0.6-0.20210304033056-74c60be0ef68 h1:sB6FDvBA1aVDINTWnVSrcJ95fV/QkN6fTJgksZOT8vY= +github.com/olekukonko/tablewriter v0.0.6-0.20210304033056-74c60be0ef68/go.mod h1:8Hf+pH6thup1sPZPD+NLg7d6vbpsdilu9CPIeikvgMQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= +github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/urfave/cli v1.22.0 h1:8nz/RUUotroXnOpYzT/Fy3sBp+2XEbXaY641/s3nbFI= +github.com/urfave/cli v1.22.0/go.mod h1:b3D7uWrF2GilkNgYpgcg6J+JMUw7ehmNkE8sZdliGLc= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32 h1:Js08h5hqB5xyWR789+QqueR6sDE8mk+YvpETZ+F6X9Y= +golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=