From 73eabec039a1324ab6a30df514e57ff38ad7f605 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 30 Dec 2018 08:02:17 -0500 Subject: [PATCH 1/5] feat: add junit support to shellcheck output This requires us to run shellcheck twice, as the first pass doesn't output anything human readable - both because of the redirect and because shellcheck doesn't directly support it - which increases lint runtime but allows us to have both the junit information and the normal human-readable output. Note that the checkstyle output does not contain timing info. --- tests.mk | 38 ++++++++++++++++++++++---------- tests/checkstyle2junit.xslt | 44 +++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 12 deletions(-) create mode 100644 tests/checkstyle2junit.xslt diff --git a/tests.mk b/tests.mk index 9bce5a385..b3c47d4db 100644 --- a/tests.mk +++ b/tests.mk @@ -1,5 +1,16 @@ SYSTEM := $(shell sh -c 'uname -s 2>/dev/null') +bats: +ifeq ($(SYSTEM),Darwin) +ifneq ($(shell bats --version > /dev/null 2>&1 ; echo $$?),0) + brew install bats-core +endif +else + git clone https://github.com/josegonzalez/bats-core.git /tmp/bats + cd /tmp/bats && sudo ./install.sh /usr/local + rm -rf /tmp/bats +endif + shellcheck: ifneq ($(shell shellcheck --version > /dev/null 2>&1 ; echo $$?),0) ifeq ($(SYSTEM),Darwin) @@ -11,7 +22,16 @@ else endif endif -ci-dependencies: shellcheck bats +xmlstarlet: +ifneq ($(shell xmlstarlet --version > /dev/null 2>&1 ; echo $$?),0) +ifeq ($(SYSTEM),Darwin) + brew install xmlstarlet +else + sudo apt-get update -qq && sudo apt-get install -qq -y xmlstarlet +endif +endif + +ci-dependencies: shellcheck bats xmlstarlet setup-deploy-tests: mkdir -p /home/dokku @@ -57,20 +77,14 @@ ifeq ($(shell grep dokku.me /home/dokku/VHOST 2>/dev/null),) echo "dokku.me" > /home/dokku/VHOST endif -bats: -ifeq ($(SYSTEM),Darwin) -ifneq ($(shell bats --version > /dev/null 2>&1 ; echo $$?),0) - brew install bats-core -endif -else - git clone https://github.com/josegonzalez/bats-core.git /tmp/bats - cd /tmp/bats && sudo ./install.sh /usr/local - rm -rf /tmp/bats -endif - lint: # these are disabled due to their expansive existence in the codebase. we should clean it up though # SC2034: VAR appears unused - https://github.com/koalaman/shellcheck/wiki/SC2034 +ifeq ($(CIRCLECI),true) + @echo creating junit output... + @mkdir -p test-results/shellcheck + @$(QUIET) find . -not -path '*/\.*' -not -path './debian/*' -type f | xargs file | grep text | awk -F ':' '{ print $$1 }' | xargs head -n1 | egrep -B1 "bash" | grep "==>" | awk '{ print $$2 }' | xargs shellcheck -e SC2034 -f checkstyle | xmlstarlet tr tests/checkstyle2junit.xslt > test-results/shellcheck/results.xml +endif @echo linting... @$(QUIET) find . -not -path '*/\.*' -not -path './debian/*' -type f | xargs file | grep text | awk -F ':' '{ print $$1 }' | xargs head -n1 | egrep -B1 "bash" | grep "==>" | awk '{ print $$2 }' | xargs shellcheck -e SC2034 diff --git a/tests/checkstyle2junit.xslt b/tests/checkstyle2junit.xslt new file mode 100644 index 000000000..0fea4528c --- /dev/null +++ b/tests/checkstyle2junit.xslt @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Line + + : + + See https://www.shellcheck.net/wiki/ + + + + From c4f1ef26c0a03428469773ce44b13c938284a657 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 5 Jan 2019 07:43:20 -0500 Subject: [PATCH 2/5] tests: move all shellcheck excludes into a single file This will allow us to add/remove shellcheck excludes more easily. --- tests.mk | 6 +++--- tests/shellcheck-exclude | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 tests/shellcheck-exclude diff --git a/tests.mk b/tests.mk index b3c47d4db..fb88ad16c 100644 --- a/tests.mk +++ b/tests.mk @@ -79,14 +79,14 @@ endif lint: # these are disabled due to their expansive existence in the codebase. we should clean it up though - # SC2034: VAR appears unused - https://github.com/koalaman/shellcheck/wiki/SC2034 + @cat tests/shellcheck-exclude | sed -n -e '/^# SC/p' ifeq ($(CIRCLECI),true) @echo creating junit output... @mkdir -p test-results/shellcheck - @$(QUIET) find . -not -path '*/\.*' -not -path './debian/*' -type f | xargs file | grep text | awk -F ':' '{ print $$1 }' | xargs head -n1 | egrep -B1 "bash" | grep "==>" | awk '{ print $$2 }' | xargs shellcheck -e SC2034 -f checkstyle | xmlstarlet tr tests/checkstyle2junit.xslt > test-results/shellcheck/results.xml + @$(QUIET) find . -not -path '*/\.*' -not -path './debian/*' -type f | xargs file | grep text | awk -F ':' '{ print $$1 }' | xargs head -n1 | egrep -B1 "bash" | grep "==>" | awk '{ print $$2 }' | xargs shellcheck -e $(shell cat tests/shellcheck-exclude | sed -n -e '/^# SC/p' | cut -d' ' -f2 | paste -d, -s) -f checkstyle | xmlstarlet tr tests/checkstyle2junit.xslt > test-results/shellcheck/results.xml endif @echo linting... - @$(QUIET) find . -not -path '*/\.*' -not -path './debian/*' -type f | xargs file | grep text | awk -F ':' '{ print $$1 }' | xargs head -n1 | egrep -B1 "bash" | grep "==>" | awk '{ print $$2 }' | xargs shellcheck -e SC2034 + @$(QUIET) find . -not -path '*/\.*' -not -path './debian/*' -type f | xargs file | grep text | awk -F ':' '{ print $$1 }' | xargs head -n1 | egrep -B1 "bash" | grep "==>" | awk '{ print $$2 }' | xargs shellcheck -e $(shell cat tests/shellcheck-exclude | sed -n -e '/^# SC/p' | cut -d' ' -f2 | paste -d, -s) ci-go-coverage: docker run --rm -ti \ diff --git a/tests/shellcheck-exclude b/tests/shellcheck-exclude new file mode 100644 index 000000000..57f97bf7d --- /dev/null +++ b/tests/shellcheck-exclude @@ -0,0 +1,29 @@ +# SC1001 - This \/ will be a regular '/' in this context - https://github.com/koalaman/shellcheck/wiki/SC1001 +# SC1003 - Want to escape a single quote? echo 'This is how it'\''s done' - https://github.com/koalaman/shellcheck/wiki/SC1003 +# SC1090 - Can't follow non-constant source. Use a directive to specify location - https://github.com/koalaman/shellcheck/wiki/SC1090 +# SC1091 - Not following: FILE: does not exist (No such file or directory) - https://github.com/koalaman/shellcheck/wiki/SC1091 +# SC1117 - Backslash is literal in "\n". Prefer explicit escaping: "\\n" - https://github.com/koalaman/shellcheck/wiki/SC1117 +# SC2029 - Note that, unescaped, this expands on the client side - https://github.com/koalaman/shellcheck/wiki/SC2029 +# SC2034 - VAR appears unused - https://github.com/koalaman/shellcheck/wiki/SC2034 +# SC2046 - Quote this to prevent word splitting - https://github.com/koalaman/shellcheck/wiki/SC2046 +# SC2068 - Double quote array expansions to avoid re-splitting elements - https://github.com/koalaman/shellcheck/wiki/SC2068 +# SC2086 - Double quote to prevent globbing and word splitting - https://github.com/koalaman/shellcheck/wiki/SC2086 +# SC2119 - Use 'function "$@"' if function's $1 should mean script's $1 - https://github.com/koalaman/shellcheck/wiki/SC2119 +# SC2120 - Function references arguments, but none are ever passed - https://github.com/koalaman/shellcheck/wiki/SC2120 +# SC2128 - Expanding an array without an index only gives the first element - https://github.com/koalaman/shellcheck/wiki/SC2128 +# SC2148 - Tips depend on target shell and yours is unknown. Add a shebang - https://github.com/koalaman/shellcheck/wiki/SC2148 +# SC2153 - Possible misspelling: DOKKU_ROOT may not be assigned, but DOKKU_PORT is - https://github.com/koalaman/shellcheck/wiki/SC2153 +# SC2154 - passed_docker_option is referenced but not assigned - https://github.com/koalaman/shellcheck/wiki/SC2154 +# SC2155 - Declare and assign separately to avoid masking return values - https://github.com/koalaman/shellcheck/wiki/SC2155 +# SC2162 - read without -r will mangle backslashes - https://github.com/koalaman/shellcheck/wiki/SC2162 +# SC2174 - When used with -p, -m only applies to the deepest directory - https://github.com/koalaman/shellcheck/wiki/SC2174 +# SC2179 - Use array+=("item") to append items to an array - https://github.com/koalaman/shellcheck/wiki/SC2179 +# SC2191 - The = here is literal. To assign by index, use ( [index]=value ) with no spaces. To keep as literal, quote it - https://github.com/koalaman/shellcheck/wiki/SC2191 +# SC2196 - egrep is non-standard and deprecated. Use grep -E instead - https://github.com/koalaman/shellcheck/wiki/SC2196 +# SC2199 - Arrays implicitly concatenate in [[ ]]. Use a loop (or explicit * instead of @) - https://github.com/koalaman/shellcheck/wiki/SC2199 +# SC2207 - Prefer mapfile or read -a to split command output (or quote to avoid splitting) - https://github.com/koalaman/shellcheck/wiki/SC2207 +# SC2219 - Instead of 'let expr', prefer (( expr )) - https://github.com/koalaman/shellcheck/wiki/SC2219 +# SC2220 - Invalid flags are not handled. Add a *) case - https://github.com/koalaman/shellcheck/wiki/SC2220 +# SC2230 - which is non-standard. Use builtin 'command -v' instead - https://github.com/koalaman/shellcheck/wiki/SC2230 +# SC2231 - Quote expansions in this for loop glob to prevent wordsplitting, e.g. "$dir"/*.txt - https://github.com/koalaman/shellcheck/wiki/SC2231 +# SC2235 - Use { ..; } instead of (..) to avoid subshell overhead - https://github.com/koalaman/shellcheck/wiki/SC2235 From d67f719fe49758d94140ef9127d3541f5c148859 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 5 Jan 2019 19:46:08 -0500 Subject: [PATCH 3/5] chore: lint python --- tests/organize-tests | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/organize-tests b/tests/organize-tests index 6e93d09be..63f94ebc6 100755 --- a/tests/organize-tests +++ b/tests/organize-tests @@ -4,6 +4,7 @@ import glob import os import xml.etree.ElementTree as ET + def mkdir_p(directory_name): try: os.makedirs(directory_name) @@ -11,6 +12,7 @@ def mkdir_p(directory_name): if exc.errno == errno.EEXIST and os.path.isdir(directory_name): pass + def main(): root_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') test_results_path = os.path.realpath(os.path.join(root_path, 'test-results')) @@ -27,5 +29,6 @@ def main(): mkdir_p(os.path.dirname(new_path)) os.rename(file, new_path) + if __name__ == "__main__": main() From 5f3b4640e82727a8ca680037ecb7f6d40e648f8e Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 5 Jan 2019 19:56:23 -0500 Subject: [PATCH 4/5] refactor: use a python script to translate from shellcheck output to junit The checkstyle xslt wasn't quite good enough, resulting in missing/malformatted output. Note that this now excludes _passing_ tests, which also isn't quite right, but a fair bit closer to the xsd here: https://github.com/windyroad/JUnit-Schema --- tests.mk | 10 +-- tests/checkstyle2junit.xslt | 44 ---------- tests/shellcheck-to-junit | 160 ++++++++++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 51 deletions(-) delete mode 100644 tests/checkstyle2junit.xslt create mode 100755 tests/shellcheck-to-junit diff --git a/tests.mk b/tests.mk index fb88ad16c..dd353e8aa 100644 --- a/tests.mk +++ b/tests.mk @@ -80,13 +80,9 @@ endif lint: # these are disabled due to their expansive existence in the codebase. we should clean it up though @cat tests/shellcheck-exclude | sed -n -e '/^# SC/p' -ifeq ($(CIRCLECI),true) - @echo creating junit output... - @mkdir -p test-results/shellcheck - @$(QUIET) find . -not -path '*/\.*' -not -path './debian/*' -type f | xargs file | grep text | awk -F ':' '{ print $$1 }' | xargs head -n1 | egrep -B1 "bash" | grep "==>" | awk '{ print $$2 }' | xargs shellcheck -e $(shell cat tests/shellcheck-exclude | sed -n -e '/^# SC/p' | cut -d' ' -f2 | paste -d, -s) -f checkstyle | xmlstarlet tr tests/checkstyle2junit.xslt > test-results/shellcheck/results.xml -endif @echo linting... - @$(QUIET) find . -not -path '*/\.*' -not -path './debian/*' -type f | xargs file | grep text | awk -F ':' '{ print $$1 }' | xargs head -n1 | egrep -B1 "bash" | grep "==>" | awk '{ print $$2 }' | xargs shellcheck -e $(shell cat tests/shellcheck-exclude | sed -n -e '/^# SC/p' | cut -d' ' -f2 | paste -d, -s) + @mkdir -p test-results/shellcheck + @$(QUIET) find . -not -path '*/\.*' -not -path './debian/*' -type f | xargs file | grep text | awk -F ':' '{ print $$1 }' | xargs head -n1 | egrep -B1 "bash" | grep "==>" | awk '{ print $$2 }' | xargs shellcheck -e $(shell cat tests/shellcheck-exclude | sed -n -e '/^# SC/p' | cut -d' ' -f2 | paste -d, -s) | tests/shellcheck-to-junit --output test-results/shellcheck/results.xml ci-go-coverage: docker run --rm -ti \ @@ -223,6 +219,6 @@ deploy-tests: test: setup-deploy-tests lint unit-tests deploy-tests test-ci: - mkdir -p test-results/bats + @mkdir -p test-results/bats @cd tests/unit && echo "executing tests: $(shell cd tests/unit ; circleci tests glob *.bats | circleci tests split --split-by=timings | xargs)" cd tests/unit && bats --formatter bats-format-junit -e -T -o ../../test-results/bats $(shell cd tests/unit ; circleci tests glob *.bats | circleci tests split --split-by=timings | xargs) diff --git a/tests/checkstyle2junit.xslt b/tests/checkstyle2junit.xslt deleted file mode 100644 index 0fea4528c..000000000 --- a/tests/checkstyle2junit.xslt +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Line - - : - - See https://www.shellcheck.net/wiki/ - - - - diff --git a/tests/shellcheck-to-junit b/tests/shellcheck-to-junit new file mode 100755 index 000000000..a1b62e547 --- /dev/null +++ b/tests/shellcheck-to-junit @@ -0,0 +1,160 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import argparse +import collections +import datetime +import re +import socket +import sys + +from xml.etree import ElementTree + + +def CDATA(text=None): + element = ElementTree.Element('![CDATA[') + element.text = text + return element + + +def _serialize_xml(write, elem, qnames, namespaces,short_empty_elements, **kwargs): + + if elem.tag == '![CDATA[': + write("\n<{}{}]]>\n".format(elem.tag, elem.text)) + if elem.tail: + write(ElementTree._escape_cdata(elem.tail)) + else: + return ElementTree._original_serialize_xml(write, elem, qnames, namespaces,short_empty_elements, **kwargs) + + +ElementTree._original_serialize_xml = ElementTree._serialize_xml +ElementTree._serialize_xml = ElementTree._serialize['xml'] = _serialize_xml + + +def read_in(): + lines = sys.stdin.readlines() + for i in range(len(lines)): + lines[i] = lines[i].rstrip() + return lines + + +def process_lines(lines): + files = {} + current_file = None + previous_line = None + line_no = None + new_issues = [] + code = None + + RE_VIOLATION = re.compile(r"\^-- (SC[\w]+): (.*)") + + for line in lines: + # start a new block + if line == '': + if current_file is not None: + file_data = files.get(current_file, {}) + files[current_file] = file_data + + issue_data = file_data.get(line_no, {}) + issue_data['code'] = code + files[current_file][line_no] = issue_data + + issues = issue_data.get('issues', []) + issues.extend(new_issues) + issue_data['issues'] = issues + + files[current_file][line_no] = issue_data + + code = None + current_file = None + line_no = None + elif line.startswith('In ./') and not previous_line: + current_file = line.split(' ')[1].replace('./', '') + line_no = line.split(' ')[3] + new_issues = [] + code = None + elif code is None and len(new_issues) == 0: + code = line + else: + match = RE_VIOLATION.match(line.strip()) + if not match: + print('Error: Issue parsing line "{0}"'.format(line.strip())) + else: + new_issues.append({ + 'shellcheck_id': match.group(1), + 'message': match.group(2), + 'original_message': line + }) + + previous_line = line + + return files + + +def output_junit(files, output=None): + timestamp = datetime.datetime.now().replace(microsecond=0).isoformat() + failures = 0 + for file, data in files.items(): + for line, issue_data in data.items(): + code = issue_data.get('code') + for issue in issue_data.get('issues', []): + failures += 1 + + root = ElementTree.Element("testsuite", + name="shellcheck", + tests="0", + failures="{0}".format(failures), + errors="0", + skipped="0", + timestamp=timestamp, + time="0", + hostname=socket.gethostname()) + + ElementTree.SubElement(root, "properties") + for file, data in files.items(): + for line, issue_data in data.items(): + code = issue_data.get('code') + for issue in issue_data.get('issues', []): + testcase = ElementTree.SubElement(root, + "testcase", + classname=file, + name=file, + time="0") + shellcheck_id = issue.get('shellcheck_id') + message = 'line {0}: {1}'.format(line, issue.get('message')) + original_message = issue.get('original_message') + e = ElementTree.Element("failure", + type=shellcheck_id, + message=message) + cdata = CDATA("\n".join([code, original_message])) + e.append(cdata) + testcase.append(e) + + ElementTree.SubElement(root, "system-out") + ElementTree.SubElement(root, "system-err") + + content = ElementTree.tostring(root, encoding='utf8', method='xml') + if output: + with open(output, 'w') as f: + f.write(content) + + +def main(): + parser = argparse.ArgumentParser( + description='Process shellcheck output to junit.') + parser.add_argument('--output', + dest='output', + action='store', + default=None, + help='file to write shellcheck output') + args = parser.parse_args() + + lines = read_in() + files = process_lines(lines) + files = collections.OrderedDict(sorted(files.items())) + output_junit(files, args.output) + for line in lines: + print(line) + + +if __name__ == '__main__': + main() From f33784cbc9e4b32f1d5e548e083998a309c55d22 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sat, 5 Jan 2019 20:42:42 -0500 Subject: [PATCH 5/5] fix: use correct encoding --- tests/shellcheck-to-junit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/shellcheck-to-junit b/tests/shellcheck-to-junit index a1b62e547..ed72d6633 100755 --- a/tests/shellcheck-to-junit +++ b/tests/shellcheck-to-junit @@ -132,7 +132,7 @@ def output_junit(files, output=None): ElementTree.SubElement(root, "system-out") ElementTree.SubElement(root, "system-err") - content = ElementTree.tostring(root, encoding='utf8', method='xml') + content = ElementTree.tostring(root, encoding='UTF-8', method='xml') if output: with open(output, 'w') as f: f.write(content)