Add GitHub workflows

tools/github/.gitignore

@@ -0,0 +1,2 @@
+cppcheck_and_clang_format.txt
+generated/*

tools/github/coverage_convert

@@ -0,0 +1,47 @@
+#!/bin/bash -e
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+project_dir="$( dirname "$( dirname "$script_dir" )" )"
+source_dir="$project_dir/src"
+unit_tests_dir="$project_dir/build/tests/unit"
+
+generated_dir="$script_dir/generated"
+html_dir="$generated_dir/html"
+data_file="$generated_dir/default.profdata"
+json_file="$generated_dir/report.json"
+coverage_file="$generated_dir/coverage.json"
+summary_file="$generated_dir/summary.rmu"
+
+# cleanup output directory
+if [ -d "$generated_dir" ]; then
+    rm -rf "$generated_dir"
+fi
+mkdir "$generated_dir"
+
+# merge raw coverage info
+raw_files="$( find "$unit_tests_dir" -name "*.profraw" | sort | tr '\n' ' ' )"
+llvm-profdata merge -sparse $raw_files -o "$data_file"
+
+# create list of binaries
+obj_files="$( echo "$raw_files" | sed "s/\.profraw//g" | sed -r 's/ +$//g' | sed "s/ / -object /g" )"
+
+# create list of source files
+src_files=$( find "$source_dir" \( -name '*.cpp' -o -name '*.hpp' \) -print | sort | tr '\n' ' ' )
+
+# generate html output
+llvm-cov show $obj_files \
+    -format html \
+    -instr-profile "$data_file" \
+    -o "$html_dir" \
+    -show-line-counts-or-regions \
+    -Xdemangler c++filt -Xdemangler -n \
+    $src_files
+
+# generate json output
+llvm-cov export $obj_files \
+    -instr-profile "$data_file" \
+    -Xdemangler c++filt -Xdemangler -n \
+    $src_files > "$json_file"
+
+# process json output
+$script_dir/coverage_parse_export "$json_file" "$coverage_file" "$summary_file" $src_files

tools/github/coverage_parse_export

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+import sys
+from collections import defaultdict
+
+def lines2phabricator(filename, lines):
+    ret = ""
+    numlines = 0
+    with open(filename) as f:
+        for row in f:
+            numlines += 1
+    for i in range(1, numlines + 1):
+        if not i in lines: ret += "N"
+        elif lines[i] == 0: ret += "U"
+        else: ret += "C"
+    return ret
+
+parser = argparse.ArgumentParser(description='Parse llvm-cov export data.')
+parser.add_argument('input', help='input file')
+parser.add_argument('coverage', help='coverage output file')
+parser.add_argument('summary', help='summary output file')
+parser.add_argument('files', nargs='+', help='files to process')
+args = parser.parse_args()
+
+# for specification of the format see:
+# https://github.com/llvm-mirror/llvm/blob/master/tools/llvm-cov/CoverageExporterJson.cpp
+
+data = json.load(open(args.input, "r"))
+
+totals = defaultdict(lambda: defaultdict(int))
+sources = defaultdict(lambda: defaultdict(int))
+for export in data["data"]:
+    for cfile in export["files"]:
+        for segment in cfile["segments"]:
+            filename = cfile["filename"]
+            if not filename in args.files: continue
+            line, col, count, has_count, is_region_entry = segment
+            sources[filename][line] += count
+    for function in export["functions"]:
+        for region in function["regions"]:
+            line_start, column_start, line_end, column_end, execution_count, \
+                    file_id, expanded_file_id, kind = region
+            filename = function["filenames"][file_id]
+            if filename not in args.files: continue
+            for i in range(line_start, line_end + 1):
+                sources[filename][i] += execution_count
+    for total, values in export["totals"].items():
+        for key, value in values.items():
+            totals[total][key] += value
+
+coverage = {}
+for filename, lines in sources.items():
+    path = "/".join(filename.split("/")[3:])
+    coverage[path] = lines2phabricator(filename, lines)
+
+with open(args.coverage, "w") as f:
+    json.dump(coverage, f)
+
+summary = """==== Code coverage: ====
+
+<table>
+<tr><th>Coverage</th><th>Total</th></tr>
+"""
+ROW = "<tr><td>{name}</td><td>{covered} / {count} ({percent:.2%})</td></tr>\n"
+for what in ["functions", "instantiations", "lines", "regions"]:
+    now = totals[what]
+    now["percent"] = now["covered"] / now["count"]
+    summary += ROW.format(**dict(totals[what], name=what.capitalize()))
+summary += "</table>\n"
+
+with open(args.summary, "w") as f:
+    f.write(summary)

tools/github/cppcheck_and_clang_format

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$DIR/../../"
+
+tmpfile="$DIR/cppcheck_and_clang_format.tmp"
+errfile="$DIR/cppcheck_and_clang_format.txt"
+
+mode=${1:-diff}
+threads=$( cat /proc/cpuinfo | grep processor | wc -l )
+
+if [ "$mode" == diff ]; then
+    files=$( git diff --name-only HEAD~1 HEAD | egrep '^(src|tests|poc)' | egrep '\.(hpp|h|cpp)$' )
+    flags=""
+else
+    files=src/
+    flags="-j$threads -Isrc"
+fi
+
+cat > .cppcheck_suppressions <<EOF
+// supress all explicit constructor warnings since we use the implicit conversion all over the codebase
+noExplicitConstructor:src/storage/property_value.hpp
+noExplicitConstructor:src/query/typed_value.hpp
+noExplicitConstructor:src/communication/bolt/v1/decoder/decoded_value.hpp
+
+// suppress antrl warnings
+variableScope:src/query/frontend/opencypher/generated/CypherParser.h
+variableScope:src/query/frontend/opencypher/generated/CypherLexer.h
+variableScope:src/query/frontend/opencypher/generated/CypherParser.cpp
+
+// supress all warnings of this type in the codebase
+missingInclude
+unusedFunction
+unusedStructMember
+useStlAlgorithm
+EOF
+
+cppcheck --enable=all --inline-suppr --force --suppressions-list=.cppcheck_suppressions $flags $files 2>"$tmpfile"
+rm .cppcheck_suppressions
+
+cat "$tmpfile" | grep -v "(information) Unmatched suppression" > "$errfile"
+rm $tmpfile
+
+cat "$errfile" >&2
+
+len="$( cat "$errfile" | wc -l )"
+if [ $len -gt 0 ]; then
+    echo -e "==== Cppcheck errors: ====\n\n\`\`\`\n$( cat "$errfile" )\n\`\`\`" > "$errfile"
+fi
+
+
+# check for clang-format errors
+
+format_list=""
+format_tmp="$DIR/.clang_format"
+
+if [ -f "$format_tmp" ]; then
+    rm "$format_tmp"
+fi
+
+for fname in $files; do
+    if [ ! -f "$fname" ]; then continue; fi
+    echo "Checking formatting errors for: $fname"
+    clang-format "$fname" > "$format_tmp"
+    if ! diff "$fname" "$format_tmp" >/dev/null; then
+        format_list+="\n$fname"
+    fi
+    rm "$format_tmp"
+done
+
+if [ "$format_list" != "" ]; then
+    if [ "$( cat "$errfile" | wc -l )" -gt 0 ]; then
+        echo "" >> "$errfile"
+    fi
+    echo -e "==== Clang-format should be applied on: ====\n\n\`\`\`$format_list\n\`\`\`" >> "$errfile"
+fi

tools/github/macro_benchmark_summary

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+import sys
+
+def load_file(fname):
+    with open(fname) as f:
+        data = f.read()
+    try:
+        return json.loads(data)
+    except json.decoder.JSONDecodeError:
+        return {"results": [], "headers": []}
+
+def strip_integers(row):
+    return {k: v for k, v in row.items() if type(v) == str}
+
+def find_item(results_prev, header_cur, row_cur):
+    row_cur = strip_integers(row_cur)
+    row_prev = None
+    for result in results_prev:
+        s = strip_integers(result)
+        if s == row_cur:
+            row_prev = result
+            break
+    if row_prev is None: return None
+    if not header_cur in row_prev: return None
+    return row_prev[header_cur]
+
+def compare_values(headers_cur, results_cur, headers_prev, results_prev):
+    ret = [list(map(lambda x: " ".join(x.split("_")).capitalize(),
+            headers_cur))]
+    for row_cur in results_cur:
+        ret.append([])
+        performance_change = False
+        for header in headers_cur:
+            item_cur = row_cur[header]
+            if type(item_cur) == str:
+                item = " ".join(item_cur.split("_")).capitalize()
+            else:
+                value_cur = item_cur["median"]
+                item_prev = find_item(results_prev, header, row_cur)
+                if header != "max_memory":
+                    fmt = "{:.3f}ms"
+                    scale = 1000.0
+                    treshold = 0.050
+                else:
+                    fmt = "{:.2f}MiB"
+                    scale = 1.0 / 1024.0
+                    treshold = 0.025
+                # TODO: add statistics check
+                if item_prev != None:
+                    value_prev = item_prev["median"]
+                    if value_prev != 0.0:
+                        diff = (value_cur - value_prev) / value_prev
+                    else:
+                        diff = 0.0
+                    if diff < -treshold and value_cur > 0.0005:
+                        performance_change = True
+                        sign = " {icon arrow-down color=green}"
+                    elif diff > treshold and value_cur > 0.0005:
+                        performance_change = True
+                        sign = " {icon arrow-up color=red}"
+                    else:
+                        sign = ""
+                    fmt += " //({:+.2%})//{}"
+                    item = fmt.format(value_cur * scale, diff, sign)
+                else:
+                    fmt += " //(new)// {{icon plus color=blue}}"
+                    item = fmt.format(value_cur * scale)
+                    performance_change = True
+            ret[-1].append(item)
+        if not performance_change: ret.pop()
+    return ret
+
+def generate_remarkup(data):
+    ret = "==== Macro benchmark summary: ====\n\n"
+    if len(data) > 1:
+        ret += "<table>\n"
+        for row in data:
+            ret += "  <tr>\n"
+            for item in row:
+                if row == data[0]:
+                    fmt = "    <th>{}</th>\n"
+                else:
+                    fmt = "    <td>{}</td>\n"
+                ret += fmt.format(item)
+            ret += "  </tr>\n"
+        ret += "</table>\n"
+    else:
+        ret += "No performance change detected.\n"
+    return ret
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Process macro benchmark summary.")
+    parser.add_argument("--current", nargs = "+", required = True,
+            help = "current summary files")
+    parser.add_argument("--previous", nargs = "+", required = True,
+            help = "previous summary files")
+    parser.add_argument("--output", default = "",
+            help = "output file, if not specified the script outputs to stdout")
+
+    args = parser.parse_args()
+
+    headers_cur, headers_prev = None, None
+    results_cur, results_prev = [], []
+    for current in args.current:
+        data = load_file(current)
+        if headers_cur is None:
+            headers_cur = data["headers"]
+        results_cur += data["results"]
+    for previous in args.previous:
+        data = load_file(previous)
+        if headers_prev is None:
+            headers_prev = data["headers"]
+        results_prev += data["results"]
+
+    markup = generate_remarkup(compare_values(headers_cur, results_cur,
+            headers_prev, results_prev))
+
+    if args.output == "":
+        sys.stdout.write(markup)
+        sys.exit(0)
+
+    with open(args.output, "w") as f:
+        f.write(markup)